{ "schema": 1, "status": "complete", "created_utc": "2026-09-26T07:41:44.086858+00:00", "arguments": { "command": "run", "data": "data/prepared/full.json", "output": "results/full", "models": [ "laya", "laya-multilingual" ], "device": "cuda", "batch_size": 8, "threads": 8, "offline": true }, "data_sha256": "9301766783b38ac56ca66c1a9ccd68797c9196a8f1dad0a4993077fa512f115f", "seed": 20260926, "sources": { "models": { "laya": { "repo": "convaiinnovations/laya", "revision": "55cf4c4ebb4ebe31b2550e8bdf3bd21b99753851" }, "laya-multilingual": { "repo": "convaiinnovations/laya-multilingual", "revision": "e4e9ddf21a7b1903b7acffd8814ad4307bf63a67" } }, "datasets": { "massive": { "repo": "AmazonScience/massive", "revision": "ed58ac423a2f4121720918bf5301577edce4ffd3", "source_revision": "ff6bd8e4b27c3543e4f8fe2108f32bb95a6f8740", "license": "CC-BY-4.0", "url": "https://huggingface.co/datasets/AmazonScience/massive" }, "norec": { "repo": "ltg/norec_sentence", "revision": "5326d626b35e6da3a503e5303e39adf11250ba3d", "license": "CC-BY-NC-4.0", "url": "https://huggingface.co/datasets/ltg/norec_sentence" } } }, "data_audit": { "massive_en": { "test": { "original": 2974, "kept": 2970, "removed_exact_text_duplicates": 4, "paired_pool": 2948 }, "validation": { "original": 2033, "kept": 2026, "removed_exact_text_duplicates": 7, "paired_pool": 1991 }, "train": { "original": 11514, "kept": 11436, "removed_exact_text_duplicates": 78 } }, "massive_nb": { "test": { "original": 2974, "kept": 2951, "removed_exact_text_duplicates": 23, "paired_pool": 2948 }, "validation": { "original": 2033, "kept": 1996, "removed_exact_text_duplicates": 37, "paired_pool": 1991 }, "train": { "original": 11514, "kept": 11035, "removed_exact_text_duplicates": 479 } }, "norec_no": { "test": { "original": 1181, "kept": 1173, "removed_exact_text_duplicates": 8 }, "validation": { "original": 1411, "kept": 1374, "removed_exact_text_duplicates": 37 }, "train": { "original": 7973, "kept": 7671, "removed_exact_text_duplicates": 302 } } }, "data_files": { "massive_en": [ { "file": "en-US/train/0000.parquet", "sha256": "9c8b3ce8a96ec3b0d4aee104d8778d9f6bacdbd18f2cea232f6d8333d24401b1", "rows": 11514 }, { "file": "en-US/validation/0000.parquet", "sha256": "cf22bf142963c86ee164c5bb1fe22c67c1261df5c52dfd7217ea87b977c7bb43", "rows": 2033 }, { "file": "en-US/test/0000.parquet", "sha256": "c418da9a5f3a7425b5cf44941bcea032a31040dbda2da85e0dd9323785c7731e", "rows": 2974 } ], "massive_nb": [ { "file": "nb-NO/train/0000.parquet", "sha256": "1d9997e199ac00114a62aaf2d306f0625ed888c797329c89b5e13a4dc9aedb29", "rows": 11514 }, { "file": "nb-NO/validation/0000.parquet", "sha256": "c88309d7b14d7a36f0bd50b9deb98338590cc30ba47ee28d72b4bf85504cf7cd", "rows": 2033 }, { "file": "nb-NO/test/0000.parquet", "sha256": "2abc9a5f3b98aa4802aeb6e7f858cc95642f134780ad5a457decb7b642b8eb76", "rows": 2974 } ], "norec_no": [ { "file": "ternary/train-00000-of-00001.parquet", "sha256": "edda5491ce8cd878ffe578f770bfc4bcb9237f724a718ddb583aad2c5e5dc31b", "rows": 7973 }, { "file": "ternary/validation-00000-of-00001.parquet", "sha256": "e576ace9fe5a9283b896ce8e011d1d88e505900d2af259752a521b47d5782499", "rows": 1411 }, { "file": "ternary/test-00000-of-00001.parquet", "sha256": "b0b436f31afc511cd0383906ae6c05643ca41a65de33cf41983740099aecdc57", "rows": 1181 } ] }, "dataset_counts": { "massive_en": { "test": { "play": 385, "calendar": 400, "news": 124, "general": 187, "iot": 214, "weather": 152, "qa": 287, "transport": 123, "social": 105, "audio": 61, "lists": 142, "alarm": 96, "email": 269, "cooking": 71, "takeaway": 57, "recommendation": 94, "music": 81, "datetime": 100 }, "validation": { "calendar": 277, "takeaway": 43, "iot": 115, "datetime": 66, "transport": 110, "email": 156, "play": 255, "news": 80, "weather": 119, "qa": 210, "lists": 110, "recommendation": 68, "audio": 35, "social": 67, "alarm": 63, "general": 119, "cooking": 43, "music": 55 }, "train": { "alarm": 390, "audio": 288, "iot": 765, "calendar": 1679, "play": 1370, "general": 647, "datetime": 395, "takeaway": 254, "news": 501, "music": 331, "weather": 567, "qa": 1175, "recommendation": 431, "social": 386, "transport": 568, "email": 946, "lists": 535, "cooking": 208 } }, "massive_nb": { "test": { "play": 385, "calendar": 400, "news": 124, "general": 187, "iot": 214, "weather": 152, "qa": 287, "transport": 123, "social": 105, "audio": 61, "lists": 142, "alarm": 96, "email": 269, "cooking": 71, "takeaway": 57, "recommendation": 94, "music": 81, "datetime": 100 }, "validation": { "calendar": 277, "takeaway": 43, "iot": 115, "datetime": 66, "transport": 110, "email": 156, "play": 255, "news": 80, "weather": 119, "qa": 210, "lists": 110, "recommendation": 68, "audio": 35, "social": 67, "alarm": 63, "general": 119, "cooking": 43, "music": 55 }, "train": { "alarm": 374, "audio": 254, "iot": 703, "calendar": 1658, "play": 1331, "general": 624, "datetime": 345, "takeaway": 255, "news": 486, "music": 314, "weather": 510, "qa": 1158, "recommendation": 423, "social": 386, "transport": 558, "email": 933, "lists": 515, "cooking": 208 } }, "norec_no": { "test": { "positive": 401, "negative": 182, "neutral": 590 }, "validation": { "neutral": 675, "positive": 488, "negative": 211 }, "train": { "neutral": 3785, "negative": 1270, "positive": 2616 } } }, "environment": { "python": "3.11.9", "platform": "Windows-10-10.0.26200-SP0", "processor": "Intel64 Family 6 Model 183 Stepping 1, GenuineIntel", "logical_cpus": 28, "ram_gb": 68.542492672, "torch": "2.8.0+cu128", "cuda": "12.8", "gpu": "NVIDIA GeForce RTX 5070 Ti", "threads": 8, "packages": { "aiohappyeyeballs": "2.7.1", "aiohttp": "3.14.3", "aiosignal": "1.4.0", "annotated-doc": "0.0.5", "anyio": "4.15.1", "attrs": "26.1.0", "certifi": "2026.7.22", "charset-normalizer": "3.5.1", "click": "8.5.0", "cloudpickle": "3.1.2", "colorama": "0.4.6", "datasets": "4.0.0", "dill": "0.3.8", "filelock": "4.0.3", "frozenlist": "1.8.0", "fsspec": "2025.3.0", "h11": "0.16.0", "hf-xet": "1.6.0", "httpcore": "1.0.9", "httpx": "0.28.1", "huggingface_hub": "0.36.2", "idna": "3.20", "iniconfig": "2.3.0", "Jinja2": "3.1.6", "joblib": "1.6.0", "laya": "0.3.20", "markdown-it-py": "4.2.0", "MarkupSafe": "3.0.3", "mdurl": "0.1.2", "mpmath": "1.3.0", "multidict": "6.9.1", "multiprocess": "0.70.16", "narwhals": "2.26.0", "networkx": "3.6.1", "numpy": "2.2.6", "packaging": "26.3", "pandas": "3.0.6", "pip": "24.0", "pluggy": "1.6.0", "propcache": "0.5.4", "psutil": "7.0.0", "pyarrow": "19.0.1", "Pygments": "2.21.0", "pytest": "8.4.2", "python-dateutil": "2.9.0.post0", "PyYAML": "6.0.3", "regex": "2026.9.10", "requests": "2.34.2", "rich": "15.0.0", "safetensors": "0.8.0", "scikit-learn": "1.7.2", "scipy": "1.15.3", "setuptools": "84.0.0", "shellingham": "1.5.4", "six": "1.17.0", "sympy": "1.14.0", "threadpoolctl": "3.7.0", "tokenizers": "0.22.2", "torch": "2.8.0+cu128", "tqdm": "4.70.1", "transformers": "4.57.6", "typer": "0.27.2", "typing_extensions": "4.16.0", "tzdata": "2026.4", "urllib3": "2.8.0", "xxhash": "4.0.1", "yarl": "1.25.1" } }, "source_hashes": { "adapter.py": "3dd7e592968b948cdccc8e97f0d092cee5fa53eab84d23e7dc46ffe13b4d79e7", "challenges.py": "8fa226b48defa715e6e3d9f524813ef025277d544972bdcc6a37c9744bc5c74d", "common.py": "71558997d110ab6187c1da79f8208a117d4860c77018d5286304a914b9438bd3", "data.py": "82104f9af119650ee8b36c361a64afcfb9344392a2a69270cb89e7040ce62256", "metrics.py": "1d791ebf169a0889cd3e802b9cedd8de6a20f5d8f31fe1e361de67f2e3998c85", "questions.py": "b9c885f5aa9446f1f9bbfcb80845fea3def680327959f578072577a2180958d8", "report.py": "129c2ec2321b2c24af2aba51e1dabe9bb6d6b95b64ab43be23675cae2f52ab3a", "runner.py": "53d2f9234711794f5f61e0e97a4c558fac4dbf6ea811edb5b58a808e6d5e58e9", "__init__.py": "52996f85871607985eb753470e1929e5e7e6147ddd3776d8640bee4595d38fcf", "__main__.py": "b860182ab1502409efcae427b11cb0b8e9278a8ce74131b6ac4455f3542845d1" }, "models": { "laya": { "metadata": { "repo": "convaiinnovations/laya", "revision": "55cf4c4ebb4ebe31b2550e8bdf3bd21b99753851", "load_seconds_excluding_download": 3.4223417999455705, "device": "cuda", "amp_dtype": "torch.bfloat16", "weight_dtype": "torch.float32", "config": { "encoder": "answerdotai/ModernBERT-large", "head_layers": 2, "max_len": 512, "head_max_len": 192, "max_prefixes": 6, "act_costs": { "escalate": 0.5 }, "cost_wrong_act": 3.0, "amp_dtype": "bf16", "model_name": "rl-agent", "temperature": [ 1.6369030475616455, 1.2514300346374512, 1.983399510383606 ], "temperature_by_options": { "choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793 }, "training": { "updates": 7313, "epochs_completed": 1, "hours": 1.96, "world_size": 1, "fine_tuned_from_checkpoint": true } }, "runtime_temperature": [ 1.6369030475616455, 1.2514300346374512, 1.983399510383606 ], "runtime_temperature_by_options": { "choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.5, "choice:2": 1.9063563346862793 }, "weights_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "peak_allocated_gpu_gb": 2.520393728 }, "suites": { "massive_en": { "n": 2948, "correct": 1962, "accuracy": 0.6655359565807327, "accuracy_wilson95": [ 0.6482990705972358, 0.6823419931643413 ], "accuracy_cluster95": [ 0.6492452510176391, 0.6828358208955224 ], "clusters": 2948, "macro_f1": 0.6904815123681389, "balanced_accuracy": 0.6969652033260797, "confusion_matrix": [ [ 94, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], [ 3, 24, 0, 0, 0, 0, 25, 4, 0, 4, 0, 0, 0, 0, 1, 0, 0, 0 ], [ 20, 0, 289, 2, 18, 2, 40, 5, 2, 2, 5, 0, 2, 5, 2, 5, 0, 1 ], [ 0, 0, 0, 65, 0, 0, 4, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0 ], [ 1, 0, 0, 0, 90, 0, 7, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0 ], [ 2, 0, 5, 0, 0, 210, 46, 3, 0, 0, 1, 0, 0, 0, 0, 0, 1, 1 ], [ 1, 1, 9, 4, 8, 0, 115, 7, 1, 1, 5, 3, 3, 21, 0, 1, 2, 5 ], [ 11, 4, 0, 6, 0, 0, 34, 156, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0 ], [ 0, 0, 8, 2, 0, 0, 14, 3, 105, 4, 0, 1, 0, 4, 0, 1, 0, 0 ], [ 0, 0, 0, 0, 0, 0, 14, 0, 1, 58, 1, 4, 0, 2, 0, 0, 1, 0 ], [ 0, 0, 0, 0, 1, 0, 18, 0, 0, 0, 100, 0, 1, 0, 0, 0, 3, 1 ], [ 3, 8, 0, 0, 0, 0, 59, 4, 1, 68, 1, 235, 1, 5, 0, 0, 0, 0 ], [ 0, 0, 2, 0, 4, 0, 187, 8, 0, 2, 18, 3, 48, 10, 0, 0, 4, 1 ], [ 0, 0, 8, 0, 0, 0, 11, 0, 0, 0, 11, 2, 0, 59, 0, 1, 2, 0 ], [ 0, 0, 0, 0, 0, 0, 22, 1, 1, 0, 3, 0, 0, 1, 75, 1, 1, 0 ], [ 0, 0, 0, 2, 3, 0, 10, 2, 0, 0, 0, 0, 0, 2, 0, 38, 0, 0 ], [ 1, 0, 4, 0, 2, 1, 15, 2, 0, 0, 0, 0, 0, 8, 0, 0, 90, 0 ], [ 3, 0, 0, 0, 1, 0, 23, 6, 0, 0, 0, 0, 3, 4, 0, 0, 1, 111 ] ], "labels": [ "alarm", "audio", "calendar", "cooking", "datetime", "email", "general", "iot", "lists", "music", "news", "play", "qa", "recommendation", "social", "takeaway", "transport", "weather" ], "per_class": { "alarm": { "precision": 0.6762589928057554, "recall": 0.9791666666666666, "f1-score": 0.8, "support": 96.0 }, "audio": { "precision": 0.6486486486486487, "recall": 0.39344262295081966, "f1-score": 0.4897959183673469, "support": 61.0 }, "calendar": { "precision": 0.8865030674846626, "recall": 0.7225, "f1-score": 0.7961432506887053, "support": 400.0 }, "cooking": { "precision": 0.8024691358024691, "recall": 0.9154929577464789, "f1-score": 0.8552631578947368, "support": 71.0 }, "datetime": { "precision": 0.703125, "recall": 0.9, "f1-score": 0.7894736842105263, "support": 100.0 }, "email": { "precision": 0.9859154929577465, "recall": 0.7806691449814126, "f1-score": 0.8713692946058091, "support": 269.0 }, "general": { "precision": 0.17857142857142858, "recall": 0.6149732620320856, "f1-score": 0.2767749699157641, "support": 187.0 }, "iot": { "precision": 0.7761194029850746, "recall": 0.7289719626168224, "f1-score": 0.7518072289156627, "support": 214.0 }, "lists": { "precision": 0.9459459459459459, "recall": 0.7394366197183099, "f1-score": 0.8300395256916996, "support": 142.0 }, "music": { "precision": 0.4172661870503597, "recall": 0.7160493827160493, "f1-score": 0.5272727272727272, "support": 81.0 }, "news": { "precision": 0.6896551724137931, "recall": 0.8064516129032258, "f1-score": 0.7434944237918215, "support": 124.0 }, "play": { "precision": 0.9437751004016064, "recall": 0.6103896103896104, "f1-score": 0.7413249211356467, "support": 385.0 }, "qa": { "precision": 0.8135593220338984, "recall": 0.1672473867595819, "f1-score": 0.2774566473988439, "support": 287.0 }, "recommendation": { "precision": 0.48360655737704916, "recall": 0.6276595744680851, "f1-score": 0.5462962962962963, "support": 94.0 }, "social": { "precision": 0.9615384615384616, "recall": 0.7142857142857143, "f1-score": 0.819672131147541, "support": 105.0 }, "takeaway": { "precision": 0.76, "recall": 0.6666666666666666, "f1-score": 0.7102803738317757, "support": 57.0 }, "transport": { "precision": 0.8490566037735849, "recall": 0.7317073170731707, "f1-score": 0.7860262008733624, "support": 123.0 }, "weather": { "precision": 0.925, "recall": 0.7302631578947368, "f1-score": 0.8161764705882353, "support": 152.0 }, "accuracy": 0.6655359565807327, "macro avg": { "precision": 0.7470563622105825, "recall": 0.6969652033260797, "f1-score": 0.6904815123681389, "support": 2948.0 }, "weighted avg": { "precision": 0.7925138149770184, "recall": 0.6655359565807327, "f1-score": 0.688025620167711, "support": 2948.0 } }, "raw_calibration": { "ece": 0.18098163460124742, "brier": 0.49848601060576375, "nll": 1.4279506736562824, "mean_confidence": 0.84651759118198, "reliability": [ { "low": 0.0, "high": 0.1, "n": 1, "accuracy": 0.0, "confidence": 0.09920992099209922 }, { "low": 0.1, "high": 0.2, "n": 0, "accuracy": null, "confidence": null }, { "low": 0.2, "high": 0.3, "n": 4, "accuracy": 0.0, "confidence": 0.2596135087145406 }, { "low": 0.3, "high": 0.4, "n": 79, "accuracy": 0.25316455696202533, "confidence": 0.35690375583125156 }, { "low": 0.4, "high": 0.5, "n": 152, "accuracy": 0.3092105263157895, "confidence": 0.45202315343914706 }, { "low": 0.5, "high": 0.6, "n": 229, "accuracy": 0.36681222707423583, "confidence": 0.5520549407159482 }, { "low": 0.6, "high": 0.7, "n": 262, "accuracy": 0.3511450381679389, "confidence": 0.6524417694455206 }, { "low": 0.7, "high": 0.8, "n": 255, "accuracy": 0.3568627450980392, "confidence": 0.7478857396801185 }, { "low": 0.8, "high": 0.9, "n": 227, "accuracy": 0.5198237885462555, "confidence": 0.853264928023201 }, { "low": 0.9, "high": 1.0, "n": 1739, "accuracy": 0.8683151236342725, "confidence": 0.9866192935691964 } ], "confident_errors": 229, "confident_count": 1739 }, "fitted_calibration": { "ece": 0.03481914718727714, "brier": 0.4445127817119739, "nll": 1.1455845400941616, "mean_confidence": 0.6854115320518188, "reliability": [ { "low": 0.0, "high": 0.1, "n": 1, "accuracy": 0.0, "confidence": 0.08757484111429246 }, { "low": 0.1, "high": 0.2, "n": 14, "accuracy": 0.0, "confidence": 0.18998091209785728 }, { "low": 0.2, "high": 0.3, "n": 339, "accuracy": 0.2536873156342183, "confidence": 0.261300825851421 }, { "low": 0.3, "high": 0.4, "n": 421, "accuracy": 0.3182897862232779, "confidence": 0.3454643647813007 }, { "low": 0.4, "high": 0.5, "n": 245, "accuracy": 0.47346938775510206, "confidence": 0.445409846085842 }, { "low": 0.5, "high": 0.6, "n": 176, "accuracy": 0.5852272727272727, "confidence": 0.5545602010627606 }, { "low": 0.6, "high": 0.7, "n": 179, "accuracy": 0.7039106145251397, "confidence": 0.6494143442025547 }, { "low": 0.7, "high": 0.8, "n": 204, "accuracy": 0.7009803921568627, "confidence": 0.7522688847759028 }, { "low": 0.8, "high": 0.9, "n": 254, "accuracy": 0.8228346456692913, "confidence": 0.855627721163195 }, { "low": 0.9, "high": 1.0, "n": 1115, "accuracy": 0.9372197309417041, "confidence": 0.9776313384588773 } ], "confident_errors": 70, "confident_count": 1115 }, "temperature": 1.8621186208975589, "calibration_n": 995, "policy_validation_n": 996, "policy_selection": null, "policy_test": { "threshold": null, "accepted": 0, "coverage": 0.0, "errors": 0, "accuracy": null, "accuracy_wilson95": [ null, null ], "errors_per_1000_total": 0.0, "review_per_1000": 1000.0 }, "fixed_thresholds_raw": [ { "threshold": 0.5, "accepted": 2712, "coverage": 0.9199457259158752, "errors": 817, "accuracy": 0.6987463126843658, "accuracy_wilson95": [ 0.6812076304621103, 0.7157227560664549 ], "errors_per_1000_total": 277.13704206241516, "review_per_1000": 80.0542740841248 }, { "threshold": 0.7, "accepted": 2221, "coverage": 0.7533921302578019, "errors": 502, "accuracy": 0.7739756866276453, "accuracy_wilson95": [ 0.7561165968486679, 0.7908886718526292 ], "errors_per_1000_total": 170.28493894165535, "review_per_1000": 246.60786974219806 }, { "threshold": 0.8, "accepted": 1966, "coverage": 0.6668928086838535, "errors": 338, "accuracy": 0.8280773143438453, "accuracy_wilson95": [ 0.8107629576879125, 0.8441120802109738 ], "errors_per_1000_total": 114.65400271370422, "review_per_1000": 333.1071913161465 }, { "threshold": 0.9, "accepted": 1739, "coverage": 0.5898914518317503, "errors": 229, "accuracy": 0.8683151236342725, "accuracy_wilson95": [ 0.8516071235230646, 0.8833994908288786 ], "errors_per_1000_total": 77.6797829036635, "review_per_1000": 410.1085481682497 }, { "threshold": 0.95, "accepted": 1562, "coverage": 0.5298507462686567, "errors": 162, "accuracy": 0.8962868117797695, "accuracy_wilson95": [ 0.8801820316040002, 0.9104471812234552 ], "errors_per_1000_total": 54.952510176390774, "review_per_1000": 470.1492537313433 }, { "threshold": 0.99, "accepted": 1219, "coverage": 0.41350067842605154, "errors": 76, "accuracy": 0.9376538146021329, "accuracy_wilson95": [ 0.9226578389162301, 0.9499000812486722 ], "errors_per_1000_total": 25.780189959294436, "review_per_1000": 586.4993215739485 } ], "fixed_thresholds_fitted": [ { "threshold": 0.5, "accepted": 1928, "coverage": 0.6540027137042063, "errors": 302, "accuracy": 0.8433609958506224, "accuracy_wilson95": [ 0.8264562356963416, 0.8589002121398306 ], "errors_per_1000_total": 102.44233378561738, "review_per_1000": 345.99728629579374 }, { "threshold": 0.7, "accepted": 1573, "coverage": 0.5335820895522388, "errors": 176, "accuracy": 0.8881118881118881, "accuracy_wilson95": [ 0.8715787419840604, 0.9027540187192098 ], "errors_per_1000_total": 59.701492537313435, "review_per_1000": 466.41791044776113 }, { "threshold": 0.8, "accepted": 1369, "coverage": 0.4643826322930801, "errors": 115, "accuracy": 0.9159970781592404, "accuracy_wilson95": [ 0.9001134977130559, 0.9295525884047323 ], "errors_per_1000_total": 39.009497964721845, "review_per_1000": 535.6173677069199 }, { "threshold": 0.9, "accepted": 1115, "coverage": 0.37822252374491183, "errors": 70, "accuracy": 0.9372197309417041, "accuracy_wilson95": [ 0.921426176823166, 0.9500109622936586 ], "errors_per_1000_total": 23.744911804613295, "review_per_1000": 621.7774762550881 }, { "threshold": 0.95, "accepted": 916, "coverage": 0.310719131614654, "errors": 51, "accuracy": 0.9443231441048034, "accuracy_wilson95": [ 0.9275338328778032, 0.9574012743564467 ], "errors_per_1000_total": 17.29986431478969, "review_per_1000": 689.2808683853459 }, { "threshold": 0.99, "accepted": 569, "coverage": 0.1930122116689281, "errors": 23, "accuracy": 0.9595782073813708, "accuracy_wilson95": [ 0.9400765261622535, 0.972916051359453 ], "errors_per_1000_total": 7.801899592944369, "review_per_1000": 806.9877883310719 } ], "audit": { "truncated_inputs": 0, "options_truncated": 0, "instruction_truncated": false }, "timing": { "batch_size": 8, "batch_p50_ms": 36.43689991440624, "batch_p95_ms": 46.866499981842935, "samples_per_second": 213.2850682317287, "measured_batches": 369, "single_p50_ms": 29.98019999358803, "single_p95_ms": 38.36899001034908, "single_n": 50 }, "option_order_audit": { "n": 100, "changed_predictions": 20, "original_accuracy": 0.62, "reversed_accuracy": 0.62 } }, "massive_nb": { "n": 2948, "correct": 621, "accuracy": 0.21065128900949798, "accuracy_wilson95": [ 0.19631283861441726, 0.22574284296626865 ], "accuracy_cluster95": [ 0.19538670284938942, 0.2252459294436906 ], "clusters": 2948, "macro_f1": 0.25585740738586793, "balanced_accuracy": 0.2451977055098419, "confusion_matrix": [ [ 77, 0, 0, 0, 1, 0, 14, 1, 2, 0, 0, 0, 0, 0, 1, 0, 0, 0 ], [ 1, 2, 0, 1, 2, 0, 48, 2, 1, 3, 0, 0, 1, 0, 0, 0, 0, 0 ], [ 16, 0, 34, 4, 7, 2, 294, 6, 19, 1, 0, 1, 0, 15, 0, 1, 0, 0 ], [ 0, 0, 0, 14, 0, 1, 45, 1, 4, 0, 0, 1, 0, 1, 0, 3, 0, 1 ], [ 3, 0, 1, 0, 15, 1, 70, 0, 0, 2, 6, 0, 0, 2, 0, 0, 0, 0 ], [ 3, 0, 2, 0, 2, 65, 184, 1, 1, 0, 3, 3, 0, 2, 2, 1, 0, 0 ], [ 1, 0, 0, 3, 4, 3, 142, 7, 6, 1, 7, 3, 0, 8, 0, 2, 0, 0 ], [ 3, 0, 1, 2, 1, 0, 159, 22, 10, 4, 0, 4, 0, 5, 0, 3, 0, 0 ], [ 0, 0, 0, 0, 0, 0, 76, 0, 61, 1, 0, 1, 0, 2, 0, 1, 0, 0 ], [ 0, 0, 0, 0, 0, 0, 49, 1, 0, 29, 0, 1, 0, 0, 0, 0, 1, 0 ], [ 0, 0, 0, 0, 2, 2, 101, 0, 0, 0, 13, 1, 0, 3, 0, 1, 1, 0 ], [ 0, 0, 4, 0, 0, 2, 260, 2, 4, 45, 1, 50, 0, 17, 0, 0, 0, 0 ], [ 1, 0, 0, 0, 2, 0, 255, 3, 1, 1, 8, 2, 1, 9, 0, 2, 0, 2 ], [ 0, 0, 0, 2, 0, 0, 65, 3, 1, 3, 1, 7, 0, 10, 0, 2, 0, 0 ], [ 1, 0, 0, 0, 1, 0, 43, 1, 1, 0, 0, 0, 0, 1, 57, 0, 0, 0 ], [ 1, 0, 0, 4, 2, 0, 29, 0, 2, 0, 0, 1, 0, 3, 0, 15, 0, 0 ], [ 0, 0, 1, 2, 1, 2, 85, 4, 2, 0, 0, 2, 0, 14, 0, 2, 8, 0 ], [ 3, 0, 2, 0, 13, 0, 113, 3, 1, 0, 3, 1, 1, 4, 0, 2, 0, 6 ] ], "labels": [ "alarm", "audio", "calendar", "cooking", "datetime", "email", "general", "iot", "lists", "music", "news", "play", "qa", "recommendation", "social", "takeaway", "transport", "weather" ], "per_class": { "alarm": { "precision": 0.7, "recall": 0.8020833333333334, "f1-score": 0.7475728155339806, "support": 96.0 }, "audio": { "precision": 1.0, "recall": 0.03278688524590164, "f1-score": 0.06349206349206349, "support": 61.0 }, "calendar": { "precision": 0.7555555555555555, "recall": 0.085, "f1-score": 0.15280898876404495, "support": 400.0 }, "cooking": { "precision": 0.4375, "recall": 0.19718309859154928, "f1-score": 0.27184466019417475, "support": 71.0 }, "datetime": { "precision": 0.2830188679245283, "recall": 0.15, "f1-score": 0.19607843137254902, "support": 100.0 }, "email": { "precision": 0.8333333333333334, "recall": 0.241635687732342, "f1-score": 0.3746397694524496, "support": 269.0 }, "general": { "precision": 0.06988188976377953, "recall": 0.7593582887700535, "f1-score": 0.12798557908968003, "support": 187.0 }, "iot": { "precision": 0.38596491228070173, "recall": 0.102803738317757, "f1-score": 0.16236162361623616, "support": 214.0 }, "lists": { "precision": 0.5258620689655172, "recall": 0.4295774647887324, "f1-score": 0.4728682170542636, "support": 142.0 }, "music": { "precision": 0.32222222222222224, "recall": 0.35802469135802467, "f1-score": 0.3391812865497076, "support": 81.0 }, "news": { "precision": 0.30952380952380953, "recall": 0.10483870967741936, "f1-score": 0.1566265060240964, "support": 124.0 }, "play": { "precision": 0.6410256410256411, "recall": 0.12987012987012986, "f1-score": 0.2159827213822894, "support": 385.0 }, "qa": { "precision": 0.3333333333333333, "recall": 0.003484320557491289, "f1-score": 0.006896551724137931, "support": 287.0 }, "recommendation": { "precision": 0.10416666666666667, "recall": 0.10638297872340426, "f1-score": 0.10526315789473684, "support": 94.0 }, "social": { "precision": 0.95, "recall": 0.5428571428571428, "f1-score": 0.6909090909090909, "support": 105.0 }, "takeaway": { "precision": 0.42857142857142855, "recall": 0.2631578947368421, "f1-score": 0.32608695652173914, "support": 57.0 }, "transport": { "precision": 0.8, "recall": 0.06504065040650407, "f1-score": 0.12030075187969924, "support": 123.0 }, "weather": { "precision": 0.6666666666666666, "recall": 0.039473684210526314, "f1-score": 0.07453416149068323, "support": 152.0 }, "accuracy": 0.21065128900949798, "macro avg": { "precision": 0.5303681331018435, "recall": 0.2451977055098419, "f1-score": 0.25585740738586793, "support": 2948.0 }, "weighted avg": { "precision": 0.5511995102983055, "recall": 0.21065128900949798, "f1-score": 0.22437511981024608, "support": 2948.0 } }, "raw_calibration": { "ece": 0.4694431218170974, "brier": 1.1120353437172776, "nll": 3.2688532148840945, "mean_confidence": 0.6800944108265954, "reliability": [ { "low": 0.0, "high": 0.1, "n": 0, "accuracy": null, "confidence": null }, { "low": 0.1, "high": 0.2, "n": 0, "accuracy": null, "confidence": null }, { "low": 0.2, "high": 0.3, "n": 27, "accuracy": 0.07407407407407407, "confidence": 0.27341687063221326 }, { "low": 0.3, "high": 0.4, "n": 173, "accuracy": 0.17341040462427745, "confidence": 0.3552174478958318 }, { "low": 0.4, "high": 0.5, "n": 315, "accuracy": 0.12698412698412698, "confidence": 0.4546355862792812 }, { "low": 0.5, "high": 0.6, "n": 457, "accuracy": 0.12035010940919037, "confidence": 0.5562599442288633 }, { "low": 0.6, "high": 0.7, "n": 653, "accuracy": 0.1332312404287902, "confidence": 0.6499004859601965 }, { "low": 0.7, "high": 0.8, "n": 587, "accuracy": 0.12095400340715502, "confidence": 0.7456366588966415 }, { "low": 0.8, "high": 0.9, "n": 298, "accuracy": 0.2181208053691275, "confidence": 0.8470829570015562 }, { "low": 0.9, "high": 1.0, "n": 438, "accuracy": 0.6187214611872146, "confidence": 0.9683972320602509 } ], "confident_errors": 167, "confident_count": 438 }, "fitted_calibration": { "ece": 0.06752436116597883, "brier": 0.8550105293503426, "nll": 2.4573281598409755, "mean_confidence": 0.2604609194864954, "reliability": [ { "low": 0.0, "high": 0.1, "n": 0, "accuracy": null, "confidence": null }, { "low": 0.1, "high": 0.2, "n": 1422, "accuracy": 0.10689170182841069, "confidence": 0.17467757823596822 }, { "low": 0.2, "high": 0.3, "n": 985, "accuracy": 0.15736040609137056, "confidence": 0.232025353865727 }, { "low": 0.3, "high": 0.4, "n": 212, "accuracy": 0.37264150943396224, "confidence": 0.34288258334575883 }, { "low": 0.4, "high": 0.5, "n": 92, "accuracy": 0.532608695652174, "confidence": 0.44681832279275996 }, { "low": 0.5, "high": 0.6, "n": 63, "accuracy": 0.6507936507936508, "confidence": 0.5445832379330428 }, { "low": 0.6, "high": 0.7, "n": 43, "accuracy": 0.7209302325581395, "confidence": 0.6450530236058541 }, { "low": 0.7, "high": 0.8, "n": 28, "accuracy": 0.6785714285714286, "confidence": 0.7440741426995315 }, { "low": 0.8, "high": 0.9, "n": 39, "accuracy": 0.8974358974358975, "confidence": 0.8472838357275189 }, { "low": 0.9, "high": 1.0, "n": 64, "accuracy": 0.9375, "confidence": 0.9559334043261286 } ], "confident_errors": 4, "confident_count": 64 }, "temperature": 2.8579759501843602, "calibration_n": 995, "policy_validation_n": 996, "policy_selection": null, "policy_test": { "threshold": null, "accepted": 0, "coverage": 0.0, "errors": 0, "accuracy": null, "accuracy_wilson95": [ null, null ], "errors_per_1000_total": 0.0, "review_per_1000": 1000.0 }, "fixed_thresholds_raw": [ { "threshold": 0.5, "accepted": 2433, "coverage": 0.8253052917232022, "errors": 1884, "accuracy": 0.2256473489519112, "accuracy_wilson95": [ 0.20947761287613656, 0.24268206900728342 ], "errors_per_1000_total": 639.0773405698778, "review_per_1000": 174.69470827679777 }, { "threshold": 0.7, "accepted": 1323, "coverage": 0.4487788331071913, "errors": 916, "accuracy": 0.3076341647770219, "accuracy_wilson95": [ 0.28335217140995905, 0.3330300296312997 ], "errors_per_1000_total": 310.719131614654, "review_per_1000": 551.2211668928087 }, { "threshold": 0.8, "accepted": 736, "coverage": 0.2496607869742198, "errors": 400, "accuracy": 0.4565217391304348, "accuracy_wilson95": [ 0.42085453888287694, 0.4926404413767912 ], "errors_per_1000_total": 135.68521031207598, "review_per_1000": 750.3392130257802 }, { "threshold": 0.9, "accepted": 438, "coverage": 0.1485753052917232, "errors": 167, "accuracy": 0.6187214611872146, "accuracy_wilson95": [ 0.5723894608976803, 0.6629890847860662 ], "errors_per_1000_total": 56.64857530529172, "review_per_1000": 851.4246947082769 }, { "threshold": 0.95, "accepted": 318, "coverage": 0.1078697421981004, "errors": 91, "accuracy": 0.7138364779874213, "accuracy_wilson95": [ 0.6618402206093915, 0.7607280852214896 ], "errors_per_1000_total": 30.868385345997286, "review_per_1000": 892.1302578018996 }, { "threshold": 0.99, "accepted": 172, "coverage": 0.05834464043419267, "errors": 27, "accuracy": 0.8430232558139534, "accuracy_wilson95": [ 0.781241699222133, 0.889817337659066 ], "errors_per_1000_total": 9.15875169606513, "review_per_1000": 941.6553595658073 } ], "fixed_thresholds_fitted": [ { "threshold": 0.5, "accepted": 237, "coverage": 0.08039348710990502, "errors": 51, "accuracy": 0.7848101265822784, "accuracy_wilson95": [ 0.7281679368191418, 0.8323667844910589 ], "errors_per_1000_total": 17.29986431478969, "review_per_1000": 919.606512890095 }, { "threshold": 0.7, "accepted": 131, "coverage": 0.044436906377204884, "errors": 17, "accuracy": 0.8702290076335878, "accuracy_wilson95": [ 0.8019885954053885, 0.9173747283701728 ], "errors_per_1000_total": 5.766621438263229, "review_per_1000": 955.5630936227951 }, { "threshold": 0.8, "accepted": 103, "coverage": 0.034938941655359566, "errors": 8, "accuracy": 0.9223300970873787, "accuracy_wilson95": [ 0.8541709980201915, 0.9601196430876495 ], "errors_per_1000_total": 2.7137042062415193, "review_per_1000": 965.0610583446404 }, { "threshold": 0.9, "accepted": 64, "coverage": 0.021709633649932156, "errors": 4, "accuracy": 0.9375, "accuracy_wilson95": [ 0.8500251490779133, 0.9754287986033819 ], "errors_per_1000_total": 1.3568521031207597, "review_per_1000": 978.2903663500679 }, { "threshold": 0.95, "accepted": 41, "coverage": 0.013907734056987787, "errors": 2, "accuracy": 0.9512195121951219, "accuracy_wilson95": [ 0.8386102112962914, 0.9865190546348308 ], "errors_per_1000_total": 0.6784260515603798, "review_per_1000": 986.0922659430122 }, { "threshold": 0.99, "accepted": 0, "coverage": 0.0, "errors": 0, "accuracy": null, "accuracy_wilson95": [ null, null ], "errors_per_1000_total": 0.0, "review_per_1000": 1000.0 } ], "audit": { "truncated_inputs": 0, "options_truncated": 0, "instruction_truncated": false }, "timing": { "batch_size": 8, "batch_p50_ms": 36.68599994853139, "batch_p95_ms": 47.06306005828082, "samples_per_second": 210.78134119051572, "measured_batches": 369, "single_p50_ms": 30.664850026369095, "single_p95_ms": 40.462439972907305, "single_n": 50 }, "option_order_audit": { "n": 100, "changed_predictions": 32, "original_accuracy": 0.21, "reversed_accuracy": 0.29 } }, "norec_no": { "n": 1173, "correct": 626, "accuracy": 0.5336743393009378, "accuracy_wilson95": [ 0.5050624307833703, 0.5620664075218041 ], "accuracy_cluster95": [ 0.5050656258639917, 0.5638733872461263 ], "clusters": 47, "macro_f1": 0.46971504794890356, "balanced_accuracy": 0.46666405012488954, "confusion_matrix": [ [ 52, 100, 30 ], [ 66, 397, 127 ], [ 48, 176, 177 ] ], "labels": [ "negative", "neutral", "positive" ], "per_class": { "negative": { "precision": 0.3132530120481928, "recall": 0.2857142857142857, "f1-score": 0.2988505747126437, "support": 182.0 }, "neutral": { "precision": 0.5898959881129272, "recall": 0.6728813559322034, "f1-score": 0.6286619160728424, "support": 590.0 }, "positive": { "precision": 0.5299401197604791, "recall": 0.44139650872817954, "f1-score": 0.4816326530612245, "support": 401.0 }, "accuracy": 0.5336743393009378, "macro avg": { "precision": 0.4776963733071997, "recall": 0.46666405012488954, "f1-score": 0.46971504794890356, "support": 1173.0 }, "weighted avg": { "precision": 0.5264762738306482, "recall": 0.5336743393009378, "f1-score": 0.5272259411408603, "support": 1173.0 } }, "raw_calibration": { "ece": 0.14813270665069111, "brier": 0.6216222629740592, "nll": 1.0526833731605918, "mean_confidence": 0.6811519734878607, "reliability": [ { "low": 0.0, "high": 0.1, "n": 0, "accuracy": null, "confidence": null }, { "low": 0.1, "high": 0.2, "n": 0, "accuracy": null, "confidence": null }, { "low": 0.2, "high": 0.3, "n": 0, "accuracy": null, "confidence": null }, { "low": 0.3, "high": 0.4, "n": 7, "accuracy": 0.42857142857142855, "confidence": 0.3736857142857143 }, { "low": 0.4, "high": 0.5, "n": 94, "accuracy": 0.425531914893617, "confidence": 0.47245580689498357 }, { "low": 0.5, "high": 0.6, "n": 258, "accuracy": 0.4418604651162791, "confidence": 0.5522073899235312 }, { "low": 0.6, "high": 0.7, "n": 277, "accuracy": 0.5306859205776173, "confidence": 0.6513200651806482 }, { "low": 0.7, "high": 0.8, "n": 293, "accuracy": 0.5802047781569966, "confidence": 0.74754294518502 }, { "low": 0.8, "high": 0.9, "n": 190, "accuracy": 0.6, "confidence": 0.8421735272365565 }, { "low": 0.9, "high": 1.0, "n": 54, "accuracy": 0.7037037037037037, "confidence": 0.9266000237715744 } ], "confident_errors": 16, "confident_count": 54 }, "fitted_calibration": { "ece": 0.019662883771310452, "brier": 0.5830176848238207, "nll": 0.9686250938850218, "mean_confidence": 0.5385036542849112, "reliability": [ { "low": 0.0, "high": 0.1, "n": 0, "accuracy": null, "confidence": null }, { "low": 0.1, "high": 0.2, "n": 0, "accuracy": null, "confidence": null }, { "low": 0.2, "high": 0.3, "n": 0, "accuracy": null, "confidence": null }, { "low": 0.3, "high": 0.4, "n": 25, "accuracy": 0.44, "confidence": 0.3786205344009143 }, { "low": 0.4, "high": 0.5, "n": 409, "accuracy": 0.4621026894865526, "confidence": 0.4601303455753912 }, { "low": 0.5, "high": 0.6, "n": 477, "accuracy": 0.559748427672956, "confidence": 0.5467551137777716 }, { "low": 0.6, "high": 0.7, "n": 202, "accuracy": 0.5792079207920792, "confidence": 0.637631146856257 }, { "low": 0.7, "high": 0.8, "n": 59, "accuracy": 0.6949152542372882, "confidence": 0.7383590322883076 }, { "low": 0.8, "high": 0.9, "n": 1, "accuracy": 1.0, "confidence": 0.8390979338719852 }, { "low": 0.9, "high": 1.0, "n": 0, "accuracy": null, "confidence": null } ], "confident_errors": 0, "confident_count": 0 }, "temperature": 1.9253076268212177, "calibration_n": 672, "policy_validation_n": 702, "policy_selection": null, "policy_test": { "threshold": null, "accepted": 0, "coverage": 0.0, "errors": 0, "accuracy": null, "accuracy_wilson95": [ null, null ], "errors_per_1000_total": 0.0, "review_per_1000": 1000.0 }, "fixed_thresholds_raw": [ { "threshold": 0.5, "accepted": 1072, "coverage": 0.9138959931798807, "errors": 489, "accuracy": 0.5438432835820896, "accuracy_wilson95": [ 0.5139239277412971, 0.573449540932275 ], "errors_per_1000_total": 416.87979539641947, "review_per_1000": 86.10400682011931 }, { "threshold": 0.7, "accepted": 537, "coverage": 0.4578005115089514, "errors": 215, "accuracy": 0.5996275605214152, "accuracy_wilson95": [ 0.55761996096556, 0.6402199018166198 ], "errors_per_1000_total": 183.29070758738277, "review_per_1000": 542.1994884910486 }, { "threshold": 0.8, "accepted": 244, "coverage": 0.20801364023870417, "errors": 92, "accuracy": 0.6229508196721312, "accuracy_wilson95": [ 0.5606775804769295, 0.6814126665927032 ], "errors_per_1000_total": 78.43137254901961, "review_per_1000": 791.9863597612958 }, { "threshold": 0.9, "accepted": 54, "coverage": 0.04603580562659847, "errors": 16, "accuracy": 0.7037037037037037, "accuracy_wilson95": [ 0.5717242709884697, 0.8086257486782021 ], "errors_per_1000_total": 13.640238704177323, "review_per_1000": 953.9641943734015 }, { "threshold": 0.95, "accepted": 7, "coverage": 0.005967604433077579, "errors": 1, "accuracy": 0.8571428571428572, "accuracy_wilson95": [ 0.4868721707256811, 0.9743203756552565 ], "errors_per_1000_total": 0.8525149190110827, "review_per_1000": 994.0323955669224 }, { "threshold": 0.99, "accepted": 0, "coverage": 0.0, "errors": 0, "accuracy": null, "accuracy_wilson95": [ null, null ], "errors_per_1000_total": 0.0, "review_per_1000": 1000.0 } ], "fixed_thresholds_fitted": [ { "threshold": 0.5, "accepted": 739, "coverage": 0.6300085251491901, "errors": 313, "accuracy": 0.5764546684709067, "accuracy_wilson95": [ 0.5405240431475044, 0.6115945531669694 ], "errors_per_1000_total": 266.83716965046887, "review_per_1000": 369.9914748508099 }, { "threshold": 0.7, "accepted": 60, "coverage": 0.05115089514066496, "errors": 18, "accuracy": 0.7, "accuracy_wilson95": [ 0.5749129205308552, 0.8010183386123089 ], "errors_per_1000_total": 15.34526854219949, "review_per_1000": 948.8491048593351 }, { "threshold": 0.8, "accepted": 1, "coverage": 0.0008525149190110827, "errors": 0, "accuracy": 1.0, "accuracy_wilson95": [ 0.20654931437723745, 1.0 ], "errors_per_1000_total": 0.0, "review_per_1000": 999.147485080989 }, { "threshold": 0.9, "accepted": 0, "coverage": 0.0, "errors": 0, "accuracy": null, "accuracy_wilson95": [ null, null ], "errors_per_1000_total": 0.0, "review_per_1000": 1000.0 }, { "threshold": 0.95, "accepted": 0, "coverage": 0.0, "errors": 0, "accuracy": null, "accuracy_wilson95": [ null, null ], "errors_per_1000_total": 0.0, "review_per_1000": 1000.0 }, { "threshold": 0.99, "accepted": 0, "coverage": 0.0, "errors": 0, "accuracy": null, "accuracy_wilson95": [ null, null ], "errors_per_1000_total": 0.0, "review_per_1000": 1000.0 } ], "audit": { "truncated_inputs": 0, "options_truncated": 0, "instruction_truncated": false }, "timing": { "batch_size": 8, "batch_p50_ms": 30.013600015081465, "batch_p95_ms": 41.18657002691178, "samples_per_second": 252.18062070138723, "measured_batches": 147, "single_p50_ms": 30.330349982250482, "single_p95_ms": 38.63657997571863, "single_n": 50 }, "option_order_audit": { "n": 100, "changed_predictions": 13, "original_accuracy": 0.59, "reversed_accuracy": 0.58 } } }, "diagnostics": { "support_en_prompt_en_route": { "n": 30, "correct": 24, "accuracy": 0.8, "accuracy_wilson95": [ 0.6269430358685175, 0.9049489282271013 ], "accuracy_cluster95": null, "clusters": 30, "macro_f1": 0.8046753246753248, "balanced_accuracy": 0.8, "confusion_matrix": [ [ 6, 0, 0, 0, 0 ], [ 1, 4, 0, 0, 1 ], [ 1, 0, 5, 0, 0 ], [ 0, 0, 0, 4, 2 ], [ 1, 0, 0, 0, 5 ] ], "labels": [ "billing", "technical", "sales", "account", "other" ], "per_class": { "billing": { "precision": 0.6666666666666666, "recall": 1.0, "f1-score": 0.8, "support": 6.0 }, "technical": { "precision": 1.0, "recall": 0.6666666666666666, "f1-score": 0.8, "support": 6.0 }, "sales": { "precision": 1.0, "recall": 0.8333333333333334, "f1-score": 0.9090909090909091, "support": 6.0 }, "account": { "precision": 1.0, "recall": 0.6666666666666666, "f1-score": 0.8, "support": 6.0 }, "other": { "precision": 0.625, "recall": 0.8333333333333334, "f1-score": 0.7142857142857143, "support": 6.0 }, "accuracy": 0.8, "macro avg": { "precision": 0.8583333333333332, "recall": 0.8, "f1-score": 0.8046753246753248, "support": 30.0 }, "weighted avg": { "precision": 0.8583333333333333, "recall": 0.8, "f1-score": 0.8046753246753247, "support": 30.0 } }, "truncated_inputs": 0 }, "support_en_prompt_en_refund": { "n": 30, "correct": 29, "accuracy": 0.9666666666666667, "accuracy_wilson95": [ 0.8332960900859082, 0.9940914096183874 ], "accuracy_cluster95": null, "clusters": 30, "macro_f1": 0.9191374663072776, "balanced_accuracy": 0.9814814814814814, "confusion_matrix": [ [ 26, 1 ], [ 0, 3 ] ], "labels": [ "0", "1" ], "per_class": { "0": { "precision": 1.0, "recall": 0.9629629629629629, "f1-score": 0.9811320754716981, "support": 27.0 }, "1": { "precision": 0.75, "recall": 1.0, "f1-score": 0.8571428571428571, "support": 3.0 }, "accuracy": 0.9666666666666667, "macro avg": { "precision": 0.875, "recall": 0.9814814814814814, "f1-score": 0.9191374663072776, "support": 30.0 }, "weighted avg": { "precision": 0.975, "recall": 0.9666666666666667, "f1-score": 0.9687331536388141, "support": 30.0 } }, "truncated_inputs": 0 }, "support_en_prompt_en_refund_choice": { "n": 30, "correct": 29, "accuracy": 0.9666666666666667, "accuracy_wilson95": [ 0.8332960900859082, 0.9940914096183874 ], "accuracy_cluster95": null, "clusters": 30, "macro_f1": 0.9191374663072776, "balanced_accuracy": 0.9814814814814814, "confusion_matrix": [ [ 26, 1 ], [ 0, 3 ] ], "labels": [ "A", "B" ], "per_class": { "A": { "precision": 1.0, "recall": 0.9629629629629629, "f1-score": 0.9811320754716981, "support": 27.0 }, "B": { "precision": 0.75, "recall": 1.0, "f1-score": 0.8571428571428571, "support": 3.0 }, "accuracy": 0.9666666666666667, "macro avg": { "precision": 0.875, "recall": 0.9814814814814814, "f1-score": 0.9191374663072776, "support": 30.0 }, "weighted avg": { "precision": 0.975, "recall": 0.9666666666666667, "f1-score": 0.9687331536388141, "support": 30.0 } }, "truncated_inputs": 0 }, "support_en_prompt_en_urgency": { "n": 30, "correct": 13, "accuracy": 0.43333333333333335, "accuracy_wilson95": [ 0.2737748557646652, 0.6080269299918639 ], "accuracy_cluster95": null, "clusters": 30, "macro_f1": 0.4523809523809524, "balanced_accuracy": 0.6944444444444443, "confusion_matrix": [ [ 8, 14, 2 ], [ 0, 3, 1 ], [ 0, 0, 2 ] ], "labels": [ "0", "1", "2" ], "per_class": { "0": { "precision": 1.0, "recall": 0.3333333333333333, "f1-score": 0.5, "support": 24.0 }, "1": { "precision": 0.17647058823529413, "recall": 0.75, "f1-score": 0.2857142857142857, "support": 4.0 }, "2": { "precision": 0.4, "recall": 1.0, "f1-score": 0.5714285714285714, "support": 2.0 }, "accuracy": 0.43333333333333335, "macro avg": { "precision": 0.5254901960784314, "recall": 0.6944444444444443, "f1-score": 0.4523809523809524, "support": 30.0 }, "weighted avg": { "precision": 0.8501960784313726, "recall": 0.43333333333333335, "f1-score": 0.47619047619047616, "support": 30.0 } }, "truncated_inputs": 0, "score_mae": 0.6325800000000001 }, "support_nb_prompt_en_route": { "n": 30, "correct": 20, "accuracy": 0.6666666666666666, "accuracy_wilson95": [ 0.4878005164454384, 0.8076950191632386 ], "accuracy_cluster95": null, "clusters": 30, "macro_f1": 0.6585314685314685, "balanced_accuracy": 0.6666666666666666, "confusion_matrix": [ [ 6, 0, 0, 0, 0 ], [ 1, 4, 0, 0, 1 ], [ 1, 1, 3, 1, 0 ], [ 1, 0, 0, 3, 2 ], [ 1, 0, 1, 0, 4 ] ], "labels": [ "billing", "technical", "sales", "account", "other" ], "per_class": { "billing": { "precision": 0.6, "recall": 1.0, "f1-score": 0.75, "support": 6.0 }, "technical": { "precision": 0.8, "recall": 0.6666666666666666, "f1-score": 0.7272727272727273, "support": 6.0 }, "sales": { "precision": 0.75, "recall": 0.5, "f1-score": 0.6, "support": 6.0 }, "account": { "precision": 0.75, "recall": 0.5, "f1-score": 0.6, "support": 6.0 }, "other": { "precision": 0.5714285714285714, "recall": 0.6666666666666666, "f1-score": 0.6153846153846154, "support": 6.0 }, "accuracy": 0.6666666666666666, "macro avg": { "precision": 0.6942857142857143, "recall": 0.6666666666666666, "f1-score": 0.6585314685314685, "support": 30.0 }, "weighted avg": { "precision": 0.6942857142857142, "recall": 0.6666666666666666, "f1-score": 0.6585314685314685, "support": 30.0 } }, "truncated_inputs": 0 }, "support_nb_prompt_en_refund": { "n": 30, "correct": 27, "accuracy": 0.9, "accuracy_wilson95": [ 0.7437891742081593, 0.9654001112526658 ], "accuracy_cluster95": null, "clusters": 30, "macro_f1": 0.6727272727272727, "balanced_accuracy": 0.6481481481481481, "confusion_matrix": [ [ 26, 1 ], [ 2, 1 ] ], "labels": [ "0", "1" ], "per_class": { "0": { "precision": 0.9285714285714286, "recall": 0.9629629629629629, "f1-score": 0.9454545454545454, "support": 27.0 }, "1": { "precision": 0.5, "recall": 0.3333333333333333, "f1-score": 0.4, "support": 3.0 }, "accuracy": 0.9, "macro avg": { "precision": 0.7142857142857143, "recall": 0.6481481481481481, "f1-score": 0.6727272727272727, "support": 30.0 }, "weighted avg": { "precision": 0.8857142857142858, "recall": 0.9, "f1-score": 0.8909090909090909, "support": 30.0 } }, "truncated_inputs": 0 }, "support_nb_prompt_en_refund_choice": { "n": 30, "correct": 27, "accuracy": 0.9, "accuracy_wilson95": [ 0.7437891742081593, 0.9654001112526658 ], "accuracy_cluster95": null, "clusters": 30, "macro_f1": 0.47368421052631576, "balanced_accuracy": 0.5, "confusion_matrix": [ [ 27, 0 ], [ 3, 0 ] ], "labels": [ "A", "B" ], "per_class": { "A": { "precision": 0.9, "recall": 1.0, "f1-score": 0.9473684210526315, "support": 27.0 }, "B": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 3.0 }, "accuracy": 0.9, "macro avg": { "precision": 0.45, "recall": 0.5, "f1-score": 0.47368421052631576, "support": 30.0 }, "weighted avg": { "precision": 0.81, "recall": 0.9, "f1-score": 0.8526315789473684, "support": 30.0 } }, "truncated_inputs": 0 }, "support_nb_prompt_en_urgency": { "n": 30, "correct": 10, "accuracy": 0.3333333333333333, "accuracy_wilson95": [ 0.19230498083676134, 0.5121994835545616 ], "accuracy_cluster95": null, "clusters": 30, "macro_f1": 0.3333333333333333, "balanced_accuracy": 0.5694444444444444, "confusion_matrix": [ [ 5, 16, 3 ], [ 0, 4, 0 ], [ 1, 0, 1 ] ], "labels": [ "0", "1", "2" ], "per_class": { "0": { "precision": 0.8333333333333334, "recall": 0.20833333333333334, "f1-score": 0.3333333333333333, "support": 24.0 }, "1": { "precision": 0.2, "recall": 1.0, "f1-score": 0.3333333333333333, "support": 4.0 }, "2": { "precision": 0.25, "recall": 0.5, "f1-score": 0.3333333333333333, "support": 2.0 }, "accuracy": 0.3333333333333333, "macro avg": { "precision": 0.4277777777777778, "recall": 0.5694444444444444, "f1-score": 0.3333333333333333, "support": 30.0 }, "weighted avg": { "precision": 0.7100000000000001, "recall": 0.3333333333333333, "f1-score": 0.3333333333333333, "support": 30.0 } }, "truncated_inputs": 0, "score_mae": 0.84976 }, "support_nn_prompt_en_route": { "n": 30, "correct": 16, "accuracy": 0.5333333333333333, "accuracy_wilson95": [ 0.36142299619873297, 0.6976761109230025 ], "accuracy_cluster95": null, "clusters": 30, "macro_f1": 0.5113308913308914, "balanced_accuracy": 0.5333333333333333, "confusion_matrix": [ [ 5, 1, 0, 0, 0 ], [ 2, 3, 0, 0, 1 ], [ 3, 1, 1, 1, 0 ], [ 1, 0, 0, 3, 2 ], [ 1, 1, 0, 0, 4 ] ], "labels": [ "billing", "technical", "sales", "account", "other" ], "per_class": { "billing": { "precision": 0.4166666666666667, "recall": 0.8333333333333334, "f1-score": 0.5555555555555556, "support": 6.0 }, "technical": { "precision": 0.5, "recall": 0.5, "f1-score": 0.5, "support": 6.0 }, "sales": { "precision": 1.0, "recall": 0.16666666666666666, "f1-score": 0.2857142857142857, "support": 6.0 }, "account": { "precision": 0.75, "recall": 0.5, "f1-score": 0.6, "support": 6.0 }, "other": { "precision": 0.5714285714285714, "recall": 0.6666666666666666, "f1-score": 0.6153846153846154, "support": 6.0 }, "accuracy": 0.5333333333333333, "macro avg": { "precision": 0.6476190476190476, "recall": 0.5333333333333333, "f1-score": 0.5113308913308914, "support": 30.0 }, "weighted avg": { "precision": 0.6476190476190475, "recall": 0.5333333333333333, "f1-score": 0.5113308913308913, "support": 30.0 } }, "truncated_inputs": 0 }, "support_nn_prompt_en_refund": { "n": 30, "correct": 26, "accuracy": 0.8666666666666667, "accuracy_wilson95": [ 0.703186733179637, 0.9469034451594526 ], "accuracy_cluster95": null, "clusters": 30, "macro_f1": 0.4642857142857143, "balanced_accuracy": 0.48148148148148145, "confusion_matrix": [ [ 26, 1 ], [ 3, 0 ] ], "labels": [ "0", "1" ], "per_class": { "0": { "precision": 0.896551724137931, "recall": 0.9629629629629629, "f1-score": 0.9285714285714286, "support": 27.0 }, "1": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 3.0 }, "accuracy": 0.8666666666666667, "macro avg": { "precision": 0.4482758620689655, "recall": 0.48148148148148145, "f1-score": 0.4642857142857143, "support": 30.0 }, "weighted avg": { "precision": 0.806896551724138, "recall": 0.8666666666666667, "f1-score": 0.8357142857142857, "support": 30.0 } }, "truncated_inputs": 0 }, "support_nn_prompt_en_refund_choice": { "n": 30, "correct": 27, "accuracy": 0.9, "accuracy_wilson95": [ 0.7437891742081593, 0.9654001112526658 ], "accuracy_cluster95": null, "clusters": 30, "macro_f1": 0.47368421052631576, "balanced_accuracy": 0.5, "confusion_matrix": [ [ 27, 0 ], [ 3, 0 ] ], "labels": [ "A", "B" ], "per_class": { "A": { "precision": 0.9, "recall": 1.0, "f1-score": 0.9473684210526315, "support": 27.0 }, "B": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 3.0 }, "accuracy": 0.9, "macro avg": { "precision": 0.45, "recall": 0.5, "f1-score": 0.47368421052631576, "support": 30.0 }, "weighted avg": { "precision": 0.81, "recall": 0.9, "f1-score": 0.8526315789473684, "support": 30.0 } }, "truncated_inputs": 0 }, "support_nn_prompt_en_urgency": { "n": 30, "correct": 10, "accuracy": 0.3333333333333333, "accuracy_wilson95": [ 0.19230498083676134, 0.5121994835545616 ], "accuracy_cluster95": null, "clusters": 30, "macro_f1": 0.30833333333333335, "balanced_accuracy": 0.5, "confusion_matrix": [ [ 6, 13, 5 ], [ 1, 3, 0 ], [ 1, 0, 1 ] ], "labels": [ "0", "1", "2" ], "per_class": { "0": { "precision": 0.75, "recall": 0.25, "f1-score": 0.375, "support": 24.0 }, "1": { "precision": 0.1875, "recall": 0.75, "f1-score": 0.3, "support": 4.0 }, "2": { "precision": 0.16666666666666666, "recall": 0.5, "f1-score": 0.25, "support": 2.0 }, "accuracy": 0.3333333333333333, "macro avg": { "precision": 0.3680555555555556, "recall": 0.5, "f1-score": 0.30833333333333335, "support": 30.0 }, "weighted avg": { "precision": 0.6361111111111111, "recall": 0.3333333333333333, "f1-score": 0.35666666666666663, "support": 30.0 } }, "truncated_inputs": 0, "score_mae": 0.9409733333333332 }, "support_nb_prompt_nb_route": { "n": 30, "correct": 11, "accuracy": 0.36666666666666664, "accuracy_wilson95": [ 0.21873920806100708, 0.5448643634520511 ], "accuracy_cluster95": null, "clusters": 30, "macro_f1": 0.3114285714285714, "balanced_accuracy": 0.3666666666666667, "confusion_matrix": [ [ 5, 1, 0, 0, 0 ], [ 3, 3, 0, 0, 0 ], [ 3, 0, 3, 0, 0 ], [ 6, 0, 0, 0, 0 ], [ 5, 0, 1, 0, 0 ] ], "labels": [ "billing", "technical", "sales", "account", "other" ], "per_class": { "billing": { "precision": 0.22727272727272727, "recall": 0.8333333333333334, "f1-score": 0.35714285714285715, "support": 6.0 }, "technical": { "precision": 0.75, "recall": 0.5, "f1-score": 0.6, "support": 6.0 }, "sales": { "precision": 0.75, "recall": 0.5, "f1-score": 0.6, "support": 6.0 }, "account": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 6.0 }, "other": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 6.0 }, "accuracy": 0.36666666666666664, "macro avg": { "precision": 0.34545454545454546, "recall": 0.3666666666666667, "f1-score": 0.3114285714285714, "support": 30.0 }, "weighted avg": { "precision": 0.34545454545454546, "recall": 0.36666666666666664, "f1-score": 0.3114285714285714, "support": 30.0 } }, "truncated_inputs": 0 }, "support_nb_prompt_nb_refund": { "n": 30, "correct": 25, "accuracy": 0.8333333333333334, "accuracy_wilson95": [ 0.6643564949358398, 0.9266345762815145 ], "accuracy_cluster95": null, "clusters": 30, "macro_f1": 0.5956873315363882, "balanced_accuracy": 0.611111111111111, "confusion_matrix": [ [ 24, 3 ], [ 2, 1 ] ], "labels": [ "0", "1" ], "per_class": { "0": { "precision": 0.9230769230769231, "recall": 0.8888888888888888, "f1-score": 0.9056603773584906, "support": 27.0 }, "1": { "precision": 0.25, "recall": 0.3333333333333333, "f1-score": 0.2857142857142857, "support": 3.0 }, "accuracy": 0.8333333333333334, "macro avg": { "precision": 0.5865384615384616, "recall": 0.611111111111111, "f1-score": 0.5956873315363882, "support": 30.0 }, "weighted avg": { "precision": 0.8557692307692307, "recall": 0.8333333333333334, "f1-score": 0.8436657681940701, "support": 30.0 } }, "truncated_inputs": 0 }, "support_nb_prompt_nb_refund_choice": { "n": 30, "correct": 21, "accuracy": 0.7, "accuracy_wilson95": [ 0.5212421254128504, 0.8333525173175619 ], "accuracy_cluster95": null, "clusters": 30, "macro_f1": 0.6000000000000001, "balanced_accuracy": 0.8333333333333333, "confusion_matrix": [ [ 18, 9 ], [ 0, 3 ] ], "labels": [ "A", "B" ], "per_class": { "A": { "precision": 1.0, "recall": 0.6666666666666666, "f1-score": 0.8, "support": 27.0 }, "B": { "precision": 0.25, "recall": 1.0, "f1-score": 0.4, "support": 3.0 }, "accuracy": 0.7, "macro avg": { "precision": 0.625, "recall": 0.8333333333333333, "f1-score": 0.6000000000000001, "support": 30.0 }, "weighted avg": { "precision": 0.925, "recall": 0.7, "f1-score": 0.76, "support": 30.0 } }, "truncated_inputs": 0 }, "support_nb_prompt_nb_urgency": { "n": 30, "correct": 24, "accuracy": 0.8, "accuracy_wilson95": [ 0.6269430358685175, 0.9049489282271013 ], "accuracy_cluster95": null, "clusters": 30, "macro_f1": 0.5138888888888888, "balanced_accuracy": 0.5416666666666666, "confusion_matrix": [ [ 21, 2, 1 ], [ 1, 3, 0 ], [ 2, 0, 0 ] ], "labels": [ "0", "1", "2" ], "per_class": { "0": { "precision": 0.875, "recall": 0.875, "f1-score": 0.875, "support": 24.0 }, "1": { "precision": 0.6, "recall": 0.75, "f1-score": 0.6666666666666666, "support": 4.0 }, "2": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 2.0 }, "accuracy": 0.8, "macro avg": { "precision": 0.4916666666666667, "recall": 0.5416666666666666, "f1-score": 0.5138888888888888, "support": 30.0 }, "weighted avg": { "precision": 0.7799999999999999, "recall": 0.8, "f1-score": 0.788888888888889, "support": 30.0 } }, "truncated_inputs": 0, "score_mae": 0.5750866666666667 }, "support_nn_prompt_nb_route": { "n": 30, "correct": 10, "accuracy": 0.3333333333333333, "accuracy_wilson95": [ 0.19230498083676134, 0.5121994835545616 ], "accuracy_cluster95": null, "clusters": 30, "macro_f1": 0.28296296296296297, "balanced_accuracy": 0.33333333333333337, "confusion_matrix": [ [ 5, 1, 0, 0, 0 ], [ 3, 2, 0, 0, 1 ], [ 3, 0, 3, 0, 0 ], [ 6, 0, 0, 0, 0 ], [ 4, 0, 1, 1, 0 ] ], "labels": [ "billing", "technical", "sales", "account", "other" ], "per_class": { "billing": { "precision": 0.23809523809523808, "recall": 0.8333333333333334, "f1-score": 0.37037037037037035, "support": 6.0 }, "technical": { "precision": 0.6666666666666666, "recall": 0.3333333333333333, "f1-score": 0.4444444444444444, "support": 6.0 }, "sales": { "precision": 0.75, "recall": 0.5, "f1-score": 0.6, "support": 6.0 }, "account": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 6.0 }, "other": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 6.0 }, "accuracy": 0.3333333333333333, "macro avg": { "precision": 0.33095238095238094, "recall": 0.33333333333333337, "f1-score": 0.28296296296296297, "support": 30.0 }, "weighted avg": { "precision": 0.33095238095238094, "recall": 0.3333333333333333, "f1-score": 0.28296296296296297, "support": 30.0 } }, "truncated_inputs": 0 }, "support_nn_prompt_nb_refund": { "n": 30, "correct": 27, "accuracy": 0.9, "accuracy_wilson95": [ 0.7437891742081593, 0.9654001112526658 ], "accuracy_cluster95": null, "clusters": 30, "macro_f1": 0.7574123989218329, "balanced_accuracy": 0.7962962962962963, "confusion_matrix": [ [ 25, 2 ], [ 1, 2 ] ], "labels": [ "0", "1" ], "per_class": { "0": { "precision": 0.9615384615384616, "recall": 0.9259259259259259, "f1-score": 0.9433962264150944, "support": 27.0 }, "1": { "precision": 0.5, "recall": 0.6666666666666666, "f1-score": 0.5714285714285714, "support": 3.0 }, "accuracy": 0.9, "macro avg": { "precision": 0.7307692307692308, "recall": 0.7962962962962963, "f1-score": 0.7574123989218329, "support": 30.0 }, "weighted avg": { "precision": 0.9153846153846155, "recall": 0.9, "f1-score": 0.9061994609164421, "support": 30.0 } }, "truncated_inputs": 0 }, "support_nn_prompt_nb_refund_choice": { "n": 30, "correct": 24, "accuracy": 0.8, "accuracy_wilson95": [ 0.6269430358685175, 0.9049489282271013 ], "accuracy_cluster95": null, "clusters": 30, "macro_f1": 0.6875, "balanced_accuracy": 0.8888888888888888, "confusion_matrix": [ [ 21, 6 ], [ 0, 3 ] ], "labels": [ "A", "B" ], "per_class": { "A": { "precision": 1.0, "recall": 0.7777777777777778, "f1-score": 0.875, "support": 27.0 }, "B": { "precision": 0.3333333333333333, "recall": 1.0, "f1-score": 0.5, "support": 3.0 }, "accuracy": 0.8, "macro avg": { "precision": 0.6666666666666666, "recall": 0.8888888888888888, "f1-score": 0.6875, "support": 30.0 }, "weighted avg": { "precision": 0.9333333333333333, "recall": 0.8, "f1-score": 0.8375, "support": 30.0 } }, "truncated_inputs": 0 }, "support_nn_prompt_nb_urgency": { "n": 30, "correct": 21, "accuracy": 0.7, "accuracy_wilson95": [ 0.5212421254128504, 0.8333525173175619 ], "accuracy_cluster95": null, "clusters": 30, "macro_f1": 0.441343669250646, "balanced_accuracy": 0.5694444444444445, "confusion_matrix": [ [ 17, 7, 0 ], [ 0, 4, 0 ], [ 2, 0, 0 ] ], "labels": [ "0", "1", "2" ], "per_class": { "0": { "precision": 0.8947368421052632, "recall": 0.7083333333333334, "f1-score": 0.7906976744186046, "support": 24.0 }, "1": { "precision": 0.36363636363636365, "recall": 1.0, "f1-score": 0.5333333333333333, "support": 4.0 }, "2": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 2.0 }, "accuracy": 0.7, "macro avg": { "precision": 0.4194577352472089, "recall": 0.5694444444444445, "f1-score": 0.441343669250646, "support": 30.0 }, "weighted avg": { "precision": 0.764274322169059, "recall": 0.7, "f1-score": 0.7036692506459947, "support": 30.0 } }, "truncated_inputs": 0, "score_mae": 0.6312499999999999 }, "minimal_pairs_en_prompt_en_refund": { "n": 12, "correct": 10, "accuracy": 0.8333333333333334, "accuracy_wilson95": [ 0.5519691377470266, 0.9530348578161463 ], "accuracy_cluster95": null, "clusters": 12, "macro_f1": 0.8285714285714285, "balanced_accuracy": 0.8333333333333333, "confusion_matrix": [ [ 4, 2 ], [ 0, 6 ] ], "labels": [ "0", "1" ], "per_class": { "0": { "precision": 1.0, "recall": 0.6666666666666666, "f1-score": 0.8, "support": 6.0 }, "1": { "precision": 0.75, "recall": 1.0, "f1-score": 0.8571428571428571, "support": 6.0 }, "accuracy": 0.8333333333333334, "macro avg": { "precision": 0.875, "recall": 0.8333333333333333, "f1-score": 0.8285714285714285, "support": 12.0 }, "weighted avg": { "precision": 0.875, "recall": 0.8333333333333334, "f1-score": 0.8285714285714286, "support": 12.0 } }, "truncated_inputs": 0 }, "minimal_pairs_en_prompt_en_refund_choice": { "n": 12, "correct": 8, "accuracy": 0.6666666666666666, "accuracy_wilson95": [ 0.39062208887279953, 0.8618799089087867 ], "accuracy_cluster95": null, "clusters": 12, "macro_f1": 0.6571428571428571, "balanced_accuracy": 0.6666666666666667, "confusion_matrix": [ [ 3, 3 ], [ 1, 5 ] ], "labels": [ "A", "B" ], "per_class": { "A": { "precision": 0.75, "recall": 0.5, "f1-score": 0.6, "support": 6.0 }, "B": { "precision": 0.625, "recall": 0.8333333333333334, "f1-score": 0.7142857142857143, "support": 6.0 }, "accuracy": 0.6666666666666666, "macro avg": { "precision": 0.6875, "recall": 0.6666666666666667, "f1-score": 0.6571428571428571, "support": 12.0 }, "weighted avg": { "precision": 0.6875, "recall": 0.6666666666666666, "f1-score": 0.6571428571428571, "support": 12.0 } }, "truncated_inputs": 0 }, "minimal_pairs_nb_prompt_en_refund": { "n": 12, "correct": 5, "accuracy": 0.4166666666666667, "accuracy_wilson95": [ 0.19326031365875665, 0.6804886874504503 ], "accuracy_cluster95": null, "clusters": 12, "macro_f1": 0.29411764705882354, "balanced_accuracy": 0.4166666666666667, "confusion_matrix": [ [ 5, 1 ], [ 6, 0 ] ], "labels": [ "0", "1" ], "per_class": { "0": { "precision": 0.45454545454545453, "recall": 0.8333333333333334, "f1-score": 0.5882352941176471, "support": 6.0 }, "1": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 6.0 }, "accuracy": 0.4166666666666667, "macro avg": { "precision": 0.22727272727272727, "recall": 0.4166666666666667, "f1-score": 0.29411764705882354, "support": 12.0 }, "weighted avg": { "precision": 0.22727272727272727, "recall": 0.4166666666666667, "f1-score": 0.29411764705882354, "support": 12.0 } }, "truncated_inputs": 0 }, "minimal_pairs_nb_prompt_en_refund_choice": { "n": 12, "correct": 6, "accuracy": 0.5, "accuracy_wilson95": [ 0.2537815976337061, 0.7462184023662939 ], "accuracy_cluster95": null, "clusters": 12, "macro_f1": 0.3333333333333333, "balanced_accuracy": 0.5, "confusion_matrix": [ [ 6, 0 ], [ 6, 0 ] ], "labels": [ "A", "B" ], "per_class": { "A": { "precision": 0.5, "recall": 1.0, "f1-score": 0.6666666666666666, "support": 6.0 }, "B": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 6.0 }, "accuracy": 0.5, "macro avg": { "precision": 0.25, "recall": 0.5, "f1-score": 0.3333333333333333, "support": 12.0 }, "weighted avg": { "precision": 0.25, "recall": 0.5, "f1-score": 0.3333333333333333, "support": 12.0 } }, "truncated_inputs": 0 }, "minimal_pairs_nn_prompt_en_refund": { "n": 12, "correct": 7, "accuracy": 0.5833333333333334, "accuracy_wilson95": [ 0.31951131254954973, 0.8067396863412435 ], "accuracy_cluster95": null, "clusters": 12, "macro_f1": 0.4957983193277311, "balanced_accuracy": 0.5833333333333334, "confusion_matrix": [ [ 6, 0 ], [ 5, 1 ] ], "labels": [ "0", "1" ], "per_class": { "0": { "precision": 0.5454545454545454, "recall": 1.0, "f1-score": 0.7058823529411765, "support": 6.0 }, "1": { "precision": 1.0, "recall": 0.16666666666666666, "f1-score": 0.2857142857142857, "support": 6.0 }, "accuracy": 0.5833333333333334, "macro avg": { "precision": 0.7727272727272727, "recall": 0.5833333333333334, "f1-score": 0.4957983193277311, "support": 12.0 }, "weighted avg": { "precision": 0.7727272727272728, "recall": 0.5833333333333334, "f1-score": 0.4957983193277311, "support": 12.0 } }, "truncated_inputs": 0 }, "minimal_pairs_nn_prompt_en_refund_choice": { "n": 12, "correct": 6, "accuracy": 0.5, "accuracy_wilson95": [ 0.2537815976337061, 0.7462184023662939 ], "accuracy_cluster95": null, "clusters": 12, "macro_f1": 0.3333333333333333, "balanced_accuracy": 0.5, "confusion_matrix": [ [ 6, 0 ], [ 6, 0 ] ], "labels": [ "A", "B" ], "per_class": { "A": { "precision": 0.5, "recall": 1.0, "f1-score": 0.6666666666666666, "support": 6.0 }, "B": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 6.0 }, "accuracy": 0.5, "macro avg": { "precision": 0.25, "recall": 0.5, "f1-score": 0.3333333333333333, "support": 12.0 }, "weighted avg": { "precision": 0.25, "recall": 0.5, "f1-score": 0.3333333333333333, "support": 12.0 } }, "truncated_inputs": 0 }, "minimal_pairs_nb_prompt_nb_refund": { "n": 12, "correct": 6, "accuracy": 0.5, "accuracy_wilson95": [ 0.2537815976337061, 0.7462184023662939 ], "accuracy_cluster95": null, "clusters": 12, "macro_f1": 0.4375, "balanced_accuracy": 0.5, "confusion_matrix": [ [ 5, 1 ], [ 5, 1 ] ], "labels": [ "0", "1" ], "per_class": { "0": { "precision": 0.5, "recall": 0.8333333333333334, "f1-score": 0.625, "support": 6.0 }, "1": { "precision": 0.5, "recall": 0.16666666666666666, "f1-score": 0.25, "support": 6.0 }, "accuracy": 0.5, "macro avg": { "precision": 0.5, "recall": 0.5, "f1-score": 0.4375, "support": 12.0 }, "weighted avg": { "precision": 0.5, "recall": 0.5, "f1-score": 0.4375, "support": 12.0 } }, "truncated_inputs": 0 }, "minimal_pairs_nb_prompt_nb_refund_choice": { "n": 12, "correct": 9, "accuracy": 0.75, "accuracy_wilson95": [ 0.46769466506643426, 0.9110583316059453 ], "accuracy_cluster95": null, "clusters": 12, "macro_f1": 0.7482517482517483, "balanced_accuracy": 0.75, "confusion_matrix": [ [ 4, 2 ], [ 1, 5 ] ], "labels": [ "A", "B" ], "per_class": { "A": { "precision": 0.8, "recall": 0.6666666666666666, "f1-score": 0.7272727272727273, "support": 6.0 }, "B": { "precision": 0.7142857142857143, "recall": 0.8333333333333334, "f1-score": 0.7692307692307693, "support": 6.0 }, "accuracy": 0.75, "macro avg": { "precision": 0.7571428571428571, "recall": 0.75, "f1-score": 0.7482517482517483, "support": 12.0 }, "weighted avg": { "precision": 0.7571428571428571, "recall": 0.75, "f1-score": 0.7482517482517483, "support": 12.0 } }, "truncated_inputs": 0 }, "minimal_pairs_nn_prompt_nb_refund": { "n": 12, "correct": 7, "accuracy": 0.5833333333333334, "accuracy_wilson95": [ 0.31951131254954973, 0.8067396863412435 ], "accuracy_cluster95": null, "clusters": 12, "macro_f1": 0.5555555555555556, "balanced_accuracy": 0.5833333333333334, "confusion_matrix": [ [ 5, 1 ], [ 4, 2 ] ], "labels": [ "0", "1" ], "per_class": { "0": { "precision": 0.5555555555555556, "recall": 0.8333333333333334, "f1-score": 0.6666666666666666, "support": 6.0 }, "1": { "precision": 0.6666666666666666, "recall": 0.3333333333333333, "f1-score": 0.4444444444444444, "support": 6.0 }, "accuracy": 0.5833333333333334, "macro avg": { "precision": 0.6111111111111112, "recall": 0.5833333333333334, "f1-score": 0.5555555555555556, "support": 12.0 }, "weighted avg": { "precision": 0.6111111111111112, "recall": 0.5833333333333334, "f1-score": 0.5555555555555555, "support": 12.0 } }, "truncated_inputs": 0 }, "minimal_pairs_nn_prompt_nb_refund_choice": { "n": 12, "correct": 5, "accuracy": 0.4166666666666667, "accuracy_wilson95": [ 0.19326031365875665, 0.6804886874504503 ], "accuracy_cluster95": null, "clusters": 12, "macro_f1": 0.37777777777777777, "balanced_accuracy": 0.41666666666666663, "confusion_matrix": [ [ 1, 5 ], [ 2, 4 ] ], "labels": [ "A", "B" ], "per_class": { "A": { "precision": 0.3333333333333333, "recall": 0.16666666666666666, "f1-score": 0.2222222222222222, "support": 6.0 }, "B": { "precision": 0.4444444444444444, "recall": 0.6666666666666666, "f1-score": 0.5333333333333333, "support": 6.0 }, "accuracy": 0.4166666666666667, "macro avg": { "precision": 0.38888888888888884, "recall": 0.41666666666666663, "f1-score": 0.37777777777777777, "support": 12.0 }, "weighted avg": { "precision": 0.38888888888888884, "recall": 0.4166666666666667, "f1-score": 0.37777777777777777, "support": 12.0 } }, "truncated_inputs": 0 }, "long_input_en_default": { "n": 6, "correct": 5, "accuracy": 0.8333333333333334, "accuracy_wilson95": [ 0.43649717781352987, 0.9699466302516935 ], "accuracy_cluster95": null, "clusters": 6, "macro_f1": 0.18181818181818182, "balanced_accuracy": 0.8333333333333334, "confusion_matrix": [ [ 5, 0, 0, 0, 1 ], [ 0, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 0 ] ], "labels": [ "billing", "technical", "sales", "account", "other" ], "per_class": { "billing": { "precision": 1.0, "recall": 0.8333333333333334, "f1-score": 0.9090909090909091, "support": 6.0 }, "technical": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 0.0 }, "sales": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 0.0 }, "account": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 0.0 }, "other": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 0.0 }, "accuracy": 0.8333333333333334, "macro avg": { "precision": 0.2, "recall": 0.16666666666666669, "f1-score": 0.18181818181818182, "support": 6.0 }, "weighted avg": { "precision": 1.0, "recall": 0.8333333333333334, "f1-score": 0.9090909090909091, "support": 6.0 } }, "truncated_inputs": 2, "cases": [ { "id": "en-0-start-default", "position": "start", "filler_repeats": 0, "prediction": "billing", "label": "billing", "audit": { "state_tokens": 7, "state_budget": 461, "state_truncated": false, "max_len": 512, "head_max_len": 192, "options_truncated": 0, "instruction_truncated": false }, "latency_ms": 30.678499955683947 }, { "id": "en-0-end-default", "position": "end", "filler_repeats": 0, "prediction": "billing", "label": "billing", "audit": { "state_tokens": 6, "state_budget": 461, "state_truncated": false, "max_len": 512, "head_max_len": 192, "options_truncated": 0, "instruction_truncated": false }, "latency_ms": 28.657800052314997 }, { "id": "en-40-start-default", "position": "start", "filler_repeats": 40, "prediction": "billing", "label": "billing", "audit": { "state_tokens": 287, "state_budget": 461, "state_truncated": false, "max_len": 512, "head_max_len": 192, "options_truncated": 0, "instruction_truncated": false }, "latency_ms": 28.457899927161634 }, { "id": "en-40-end-default", "position": "end", "filler_repeats": 40, "prediction": "billing", "label": "billing", "audit": { "state_tokens": 286, "state_budget": 461, "state_truncated": false, "max_len": 512, "head_max_len": 192, "options_truncated": 0, "instruction_truncated": false }, "latency_ms": 31.99960000347346 }, { "id": "en-160-start-default", "position": "start", "filler_repeats": 160, "prediction": "billing", "label": "billing", "audit": { "state_tokens": 1127, "state_budget": 461, "state_truncated": true, "max_len": 512, "head_max_len": 192, "options_truncated": 0, "instruction_truncated": false }, "latency_ms": 29.198999982327223 }, { "id": "en-160-end-default", "position": "end", "filler_repeats": 160, "prediction": "other", "label": "billing", "audit": { "state_tokens": 1126, "state_budget": 461, "state_truncated": true, "max_len": 512, "head_max_len": 192, "options_truncated": 0, "instruction_truncated": false }, "latency_ms": 26.844799984246492 } ] }, "long_input_nb_default": { "n": 6, "correct": 6, "accuracy": 1.0, "accuracy_wilson95": [ 0.6096657120978346, 1.0 ], "accuracy_cluster95": null, "clusters": 6, "macro_f1": 0.2, "balanced_accuracy": 1.0, "confusion_matrix": [ [ 6, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 0 ] ], "labels": [ "billing", "technical", "sales", "account", "other" ], "per_class": { "billing": { "precision": 1.0, "recall": 1.0, "f1-score": 1.0, "support": 6.0 }, "technical": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 0.0 }, "sales": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 0.0 }, "account": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 0.0 }, "other": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 0.0 }, "accuracy": 1.0, "macro avg": { "precision": 0.2, "recall": 0.2, "f1-score": 0.2, "support": 6.0 }, "weighted avg": { "precision": 1.0, "recall": 1.0, "f1-score": 1.0, "support": 6.0 } }, "truncated_inputs": 4, "cases": [ { "id": "nb-0-start-default", "position": "start", "filler_repeats": 0, "prediction": "billing", "label": "billing", "audit": { "state_tokens": 14, "state_budget": 461, "state_truncated": false, "max_len": 512, "head_max_len": 192, "options_truncated": 0, "instruction_truncated": false }, "latency_ms": 36.523799994029105 }, { "id": "nb-0-end-default", "position": "end", "filler_repeats": 0, "prediction": "billing", "label": "billing", "audit": { "state_tokens": 13, "state_budget": 461, "state_truncated": false, "max_len": 512, "head_max_len": 192, "options_truncated": 0, "instruction_truncated": false }, "latency_ms": 29.466599924489856 }, { "id": "nb-40-start-default", "position": "start", "filler_repeats": 40, "prediction": "billing", "label": "billing", "audit": { "state_tokens": 574, "state_budget": 461, "state_truncated": true, "max_len": 512, "head_max_len": 192, "options_truncated": 0, "instruction_truncated": false }, "latency_ms": 26.45420003682375 }, { "id": "nb-40-end-default", "position": "end", "filler_repeats": 40, "prediction": "billing", "label": "billing", "audit": { "state_tokens": 573, "state_budget": 461, "state_truncated": true, "max_len": 512, "head_max_len": 192, "options_truncated": 0, "instruction_truncated": false }, "latency_ms": 37.567899911664426 }, { "id": "nb-160-start-default", "position": "start", "filler_repeats": 160, "prediction": "billing", "label": "billing", "audit": { "state_tokens": 2254, "state_budget": 461, "state_truncated": true, "max_len": 512, "head_max_len": 192, "options_truncated": 0, "instruction_truncated": false }, "latency_ms": 28.040000004693866 }, { "id": "nb-160-end-default", "position": "end", "filler_repeats": 160, "prediction": "billing", "label": "billing", "audit": { "state_tokens": 2253, "state_budget": 461, "state_truncated": true, "max_len": 512, "head_max_len": 192, "options_truncated": 0, "instruction_truncated": false }, "latency_ms": 26.42580005340278 } ] }, "long_input_nn_default": { "n": 6, "correct": 6, "accuracy": 1.0, "accuracy_wilson95": [ 0.6096657120978346, 1.0 ], "accuracy_cluster95": null, "clusters": 6, "macro_f1": 0.2, "balanced_accuracy": 1.0, "confusion_matrix": [ [ 6, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 0 ] ], "labels": [ "billing", "technical", "sales", "account", "other" ], "per_class": { "billing": { "precision": 1.0, "recall": 1.0, "f1-score": 1.0, "support": 6.0 }, "technical": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 0.0 }, "sales": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 0.0 }, "account": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 0.0 }, "other": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 0.0 }, "accuracy": 1.0, "macro avg": { "precision": 0.2, "recall": 0.2, "f1-score": 0.2, "support": 6.0 }, "weighted avg": { "precision": 1.0, "recall": 1.0, "f1-score": 1.0, "support": 6.0 } }, "truncated_inputs": 4, "cases": [ { "id": "nn-0-start-default", "position": "start", "filler_repeats": 0, "prediction": "billing", "label": "billing", "audit": { "state_tokens": 14, "state_budget": 461, "state_truncated": false, "max_len": 512, "head_max_len": 192, "options_truncated": 0, "instruction_truncated": false }, "latency_ms": 25.967900059185922 }, { "id": "nn-0-end-default", "position": "end", "filler_repeats": 0, "prediction": "billing", "label": "billing", "audit": { "state_tokens": 13, "state_budget": 461, "state_truncated": false, "max_len": 512, "head_max_len": 192, "options_truncated": 0, "instruction_truncated": false }, "latency_ms": 27.612200006842613 }, { "id": "nn-40-start-default", "position": "start", "filler_repeats": 40, "prediction": "billing", "label": "billing", "audit": { "state_tokens": 574, "state_budget": 461, "state_truncated": true, "max_len": 512, "head_max_len": 192, "options_truncated": 0, "instruction_truncated": false }, "latency_ms": 26.40239999163896 }, { "id": "nn-40-end-default", "position": "end", "filler_repeats": 40, "prediction": "billing", "label": "billing", "audit": { "state_tokens": 573, "state_budget": 461, "state_truncated": true, "max_len": 512, "head_max_len": 192, "options_truncated": 0, "instruction_truncated": false }, "latency_ms": 25.918800034560263 }, { "id": "nn-160-start-default", "position": "start", "filler_repeats": 160, "prediction": "billing", "label": "billing", "audit": { "state_tokens": 2254, "state_budget": 461, "state_truncated": true, "max_len": 512, "head_max_len": 192, "options_truncated": 0, "instruction_truncated": false }, "latency_ms": 27.549999998882413 }, { "id": "nn-160-end-default", "position": "end", "filler_repeats": 160, "prediction": "billing", "label": "billing", "audit": { "state_tokens": 2253, "state_budget": 461, "state_truncated": true, "max_len": 512, "head_max_len": 192, "options_truncated": 0, "instruction_truncated": false }, "latency_ms": 35.41390004102141 } ] } } }, "laya-multilingual": { "metadata": { "repo": "convaiinnovations/laya-multilingual", "revision": "e4e9ddf21a7b1903b7acffd8814ad4307bf63a67", "load_seconds_excluding_download": 1.9105206999229267, "device": "cuda", "amp_dtype": "torch.bfloat16", "weight_dtype": "torch.float32", "config": { "encoder": "jhu-clsp/mmBERT-base", "head_layers": 2, "max_len": 1024, "head_max_len": 256, "max_prefixes": 6, "act_costs": { "escalate": 0.5 }, "cost_wrong_act": 3.0, "amp_dtype": "bf16", "model_name": "rl-agent", "temperature": [ 1.0, 1.0, 1.0 ], "temperature_by_options": {}, "training": { "updates": 15987, "epochs_completed": 4, "hours": 4.97, "world_size": 1, "fine_tuned_from_checkpoint": false } }, "runtime_temperature": [ 1.0, 1.0, 1.0 ], "runtime_temperature_by_options": {}, "weights_sha256": "9d628fd971b700382ac6f65920a86f149777b2e748e0c955fb3b19695aa8f204", "peak_allocated_gpu_gb": 1.650002432 }, "suites": { "massive_en": { "n": 2948, "correct": 1815, "accuracy": 0.6156716417910447, "accuracy_wilson95": [ 0.5979724741601596, 0.6330697445748766 ], "accuracy_cluster95": [ 0.5980325644504749, 0.6336584124830393 ], "clusters": 2948, "macro_f1": 0.6130195944163184, "balanced_accuracy": 0.635109835995963, "confusion_matrix": [ [ 94, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0 ], [ 6, 34, 0, 0, 0, 0, 4, 5, 0, 0, 0, 1, 10, 0, 0, 0, 0, 1 ], [ 23, 0, 281, 5, 25, 1, 14, 11, 1, 4, 2, 1, 21, 1, 1, 2, 3, 4 ], [ 0, 0, 0, 68, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0 ], [ 1, 0, 0, 0, 74, 0, 1, 1, 0, 0, 0, 0, 13, 0, 0, 0, 3, 7 ], [ 0, 0, 4, 0, 3, 211, 17, 1, 0, 0, 0, 0, 28, 0, 2, 1, 1, 1 ], [ 5, 1, 8, 4, 3, 0, 34, 8, 1, 4, 1, 15, 83, 9, 1, 3, 2, 5 ], [ 31, 1, 0, 21, 0, 0, 2, 145, 0, 0, 0, 1, 11, 0, 0, 1, 0, 1 ], [ 0, 0, 3, 1, 0, 0, 1, 4, 102, 6, 0, 1, 17, 0, 0, 7, 0, 0 ], [ 0, 9, 1, 0, 0, 0, 2, 0, 2, 53, 0, 3, 9, 1, 0, 0, 1, 0 ], [ 1, 0, 2, 0, 0, 1, 4, 0, 0, 0, 76, 1, 31, 1, 0, 0, 4, 3 ], [ 2, 119, 0, 1, 1, 1, 17, 15, 2, 44, 1, 141, 37, 1, 0, 0, 1, 2 ], [ 0, 0, 2, 1, 9, 0, 43, 11, 1, 4, 1, 7, 178, 11, 1, 2, 5, 11 ], [ 0, 0, 8, 3, 2, 0, 12, 1, 1, 1, 3, 3, 16, 33, 0, 1, 9, 1 ], [ 0, 2, 4, 1, 0, 1, 8, 8, 0, 1, 3, 0, 11, 1, 60, 3, 1, 1 ], [ 1, 0, 0, 11, 2, 0, 1, 3, 0, 0, 0, 0, 10, 0, 0, 27, 2, 0 ], [ 4, 0, 3, 0, 9, 0, 0, 1, 0, 0, 0, 1, 11, 1, 0, 0, 91, 2 ], [ 0, 0, 2, 0, 1, 0, 3, 21, 0, 0, 0, 0, 12, 0, 0, 0, 0, 113 ] ], "labels": [ "alarm", "audio", "calendar", "cooking", "datetime", "email", "general", "iot", "lists", "music", "news", "play", "qa", "recommendation", "social", "takeaway", "transport", "weather" ], "per_class": { "alarm": { "precision": 0.5595238095238095, "recall": 0.9791666666666666, "f1-score": 0.7121212121212122, "support": 96.0 }, "audio": { "precision": 0.20481927710843373, "recall": 0.5573770491803278, "f1-score": 0.29955947136563876, "support": 61.0 }, "calendar": { "precision": 0.8808777429467085, "recall": 0.7025, "f1-score": 0.7816411682892906, "support": 400.0 }, "cooking": { "precision": 0.5862068965517241, "recall": 0.9577464788732394, "f1-score": 0.7272727272727273, "support": 71.0 }, "datetime": { "precision": 0.5736434108527132, "recall": 0.74, "f1-score": 0.6462882096069869, "support": 100.0 }, "email": { "precision": 0.9813953488372092, "recall": 0.7843866171003717, "f1-score": 0.871900826446281, "support": 269.0 }, "general": { "precision": 0.2085889570552147, "recall": 0.18181818181818182, "f1-score": 0.19428571428571428, "support": 187.0 }, "iot": { "precision": 0.6170212765957447, "recall": 0.677570093457944, "f1-score": 0.6458797327394209, "support": 214.0 }, "lists": { "precision": 0.9272727272727272, "recall": 0.7183098591549296, "f1-score": 0.8095238095238095, "support": 142.0 }, "music": { "precision": 0.452991452991453, "recall": 0.654320987654321, "f1-score": 0.5353535353535354, "support": 81.0 }, "news": { "precision": 0.8735632183908046, "recall": 0.6129032258064516, "f1-score": 0.7203791469194313, "support": 124.0 }, "play": { "precision": 0.8057142857142857, "recall": 0.36623376623376624, "f1-score": 0.5035714285714286, "support": 385.0 }, "qa": { "precision": 0.35528942115768464, "recall": 0.6202090592334495, "f1-score": 0.4517766497461929, "support": 287.0 }, "recommendation": { "precision": 0.559322033898305, "recall": 0.35106382978723405, "f1-score": 0.43137254901960786, "support": 94.0 }, "social": { "precision": 0.9230769230769231, "recall": 0.5714285714285714, "f1-score": 0.7058823529411765, "support": 105.0 }, "takeaway": { "precision": 0.5625, "recall": 0.47368421052631576, "f1-score": 0.5142857142857142, "support": 57.0 }, "transport": { "precision": 0.7398373983739838, "recall": 0.7398373983739838, "f1-score": 0.7398373983739838, "support": 123.0 }, "weather": { "precision": 0.743421052631579, "recall": 0.743421052631579, "f1-score": 0.743421052631579, "support": 152.0 }, "accuracy": 0.6156716417910447, "macro avg": { "precision": 0.6419480684988502, "recall": 0.635109835995963, "f1-score": 0.6130195944163184, "support": 2948.0 }, "weighted avg": { "precision": 0.6875865714254813, "recall": 0.6156716417910447, "f1-score": 0.6254437965107088, "support": 2948.0 } }, "raw_calibration": { "ece": 0.1735927753489961, "brier": 0.5480325378313812, "nll": 1.5183683847589071, "mean_confidence": 0.7891008608209635, "reliability": [ { "low": 0.0, "high": 0.1, "n": 0, "accuracy": null, "confidence": null }, { "low": 0.1, "high": 0.2, "n": 4, "accuracy": 0.25, "confidence": 0.18972949642004494 }, { "low": 0.2, "high": 0.3, "n": 71, "accuracy": 0.2112676056338028, "confidence": 0.2641546616020235 }, { "low": 0.3, "high": 0.4, "n": 186, "accuracy": 0.24193548387096775, "confidence": 0.3576596952217773 }, { "low": 0.4, "high": 0.5, "n": 257, "accuracy": 0.26459143968871596, "confidence": 0.45325655591329705 }, { "low": 0.5, "high": 0.6, "n": 237, "accuracy": 0.33755274261603374, "confidence": 0.5490026756661441 }, { "low": 0.6, "high": 0.7, "n": 229, "accuracy": 0.38427947598253276, "confidence": 0.6484982920418644 }, { "low": 0.7, "high": 0.8, "n": 215, "accuracy": 0.5116279069767442, "confidence": 0.7487825703853708 }, { "low": 0.8, "high": 0.9, "n": 265, "accuracy": 0.5320754716981132, "confidence": 0.8550671285742255 }, { "low": 0.9, "high": 1.0, "n": 1484, "accuracy": 0.8537735849056604, "confidence": 0.9821718435608635 } ], "confident_errors": 217, "confident_count": 1484 }, "fitted_calibration": { "ece": 0.0366791015582661, "brier": 0.5071075716675325, "nll": 1.3071199392995911, "mean_confidence": 0.6390510329398897, "reliability": [ { "low": 0.0, "high": 0.1, "n": 0, "accuracy": null, "confidence": null }, { "low": 0.1, "high": 0.2, "n": 90, "accuracy": 0.2222222222222222, "confidence": 0.17759668374513776 }, { "low": 0.2, "high": 0.3, "n": 377, "accuracy": 0.21220159151193635, "confidence": 0.2548565135290901 }, { "low": 0.3, "high": 0.4, "n": 376, "accuracy": 0.3776595744680851, "confidence": 0.3475157533561864 }, { "low": 0.4, "high": 0.5, "n": 311, "accuracy": 0.4630225080385852, "confidence": 0.4493459873232305 }, { "low": 0.5, "high": 0.6, "n": 211, "accuracy": 0.5260663507109005, "confidence": 0.5446080267924254 }, { "low": 0.6, "high": 0.7, "n": 228, "accuracy": 0.618421052631579, "confidence": 0.6474038007468003 }, { "low": 0.7, "high": 0.8, "n": 226, "accuracy": 0.7212389380530974, "confidence": 0.7492873924256023 }, { "low": 0.8, "high": 0.9, "n": 262, "accuracy": 0.7862595419847328, "confidence": 0.8500690686065705 }, { "low": 0.9, "high": 1.0, "n": 867, "accuracy": 0.9319492502883506, "confidence": 0.9767795914224162 } ], "confident_errors": 59, "confident_count": 867 }, "temperature": 1.6577497523414595, "calibration_n": 995, "policy_validation_n": 996, "policy_selection": null, "policy_test": { "threshold": null, "accepted": 0, "coverage": 0.0, "errors": 0, "accuracy": null, "accuracy_wilson95": [ null, null ], "errors_per_1000_total": 0.0, "review_per_1000": 1000.0 }, "fixed_thresholds_raw": [ { "threshold": 0.5, "accepted": 2430, "coverage": 0.8242876526458616, "errors": 744, "accuracy": 0.6938271604938271, "accuracy_wilson95": [ 0.6752077253840292, 0.711834740585156 ], "errors_per_1000_total": 252.37449118046135, "review_per_1000": 175.71234735413844 }, { "threshold": 0.7, "accepted": 1964, "coverage": 0.666214382632293, "errors": 446, "accuracy": 0.7729124236252546, "accuracy_wilson95": [ 0.7538616409620628, 0.7908976917549325 ], "errors_per_1000_total": 151.2890094979647, "review_per_1000": 333.78561736770695 }, { "threshold": 0.8, "accepted": 1749, "coverage": 0.5932835820895522, "errors": 341, "accuracy": 0.8050314465408805, "accuracy_wilson95": [ 0.7858042576892499, 0.8229216454040476 ], "errors_per_1000_total": 115.67164179104478, "review_per_1000": 406.7164179104478 }, { "threshold": 0.9, "accepted": 1484, "coverage": 0.5033921302578019, "errors": 217, "accuracy": 0.8537735849056604, "accuracy_wilson95": [ 0.8348832535154459, 0.8708370998024809 ], "errors_per_1000_total": 73.60922659430122, "review_per_1000": 496.60786974219803 }, { "threshold": 0.95, "accepted": 1278, "coverage": 0.4335142469470828, "errors": 152, "accuracy": 0.8810641627543037, "accuracy_wilson95": [ 0.8621643050948778, 0.897680052643472 ], "errors_per_1000_total": 51.56037991858887, "review_per_1000": 566.4857530529173 }, { "threshold": 0.99, "accepted": 920, "coverage": 0.3120759837177748, "errors": 62, "accuracy": 0.932608695652174, "accuracy_wilson95": [ 0.9145441329056883, 0.9470755664484771 ], "errors_per_1000_total": 21.03120759837178, "review_per_1000": 687.9240162822252 } ], "fixed_thresholds_fitted": [ { "threshold": 0.5, "accepted": 1794, "coverage": 0.6085481682496607, "errors": 365, "accuracy": 0.7965440356744704, "accuracy_wilson95": [ 0.7772910750336253, 0.8145297414125243 ], "errors_per_1000_total": 123.81275440976934, "review_per_1000": 391.45183175033924 }, { "threshold": 0.7, "accepted": 1355, "coverage": 0.4596336499321574, "errors": 178, "accuracy": 0.8686346863468635, "accuracy_wilson95": [ 0.8496016696078471, 0.8855834349642552 ], "errors_per_1000_total": 60.379918588873814, "review_per_1000": 540.3663500678426 }, { "threshold": 0.8, "accepted": 1129, "coverage": 0.38297150610583447, "errors": 115, "accuracy": 0.8981399468556245, "accuracy_wilson95": [ 0.8791250205323253, 0.9144546926747401 ], "errors_per_1000_total": 39.009497964721845, "review_per_1000": 617.0284938941654 }, { "threshold": 0.9, "accepted": 867, "coverage": 0.2940976933514247, "errors": 59, "accuracy": 0.9319492502883506, "accuracy_wilson95": [ 0.9132096889765365, 0.9468779795806156 ], "errors_per_1000_total": 20.013568521031207, "review_per_1000": 705.9023066485752 }, { "threshold": 0.95, "accepted": 695, "coverage": 0.23575305291723203, "errors": 34, "accuracy": 0.9510791366906475, "accuracy_wilson95": [ 0.9324161056390966, 0.9647830975365025 ], "errors_per_1000_total": 11.533242876526458, "review_per_1000": 764.246947082768 }, { "threshold": 0.99, "accepted": 428, "coverage": 0.1451831750339213, "errors": 8, "accuracy": 0.9813084112149533, "accuracy_wilson95": [ 0.9635549131913127, 0.9904989220405366 ], "errors_per_1000_total": 2.7137042062415193, "review_per_1000": 854.8168249660787 } ], "audit": { "truncated_inputs": 0, "options_truncated": 0, "instruction_truncated": false }, "timing": { "batch_size": 8, "batch_p50_ms": 30.70779994595796, "batch_p95_ms": 41.059640003368244, "samples_per_second": 247.2223607610987, "measured_batches": 369, "single_p50_ms": 26.23760001733899, "single_p95_ms": 38.21431998512707, "single_n": 50 }, "option_order_audit": { "n": 100, "changed_predictions": 21, "original_accuracy": 0.57, "reversed_accuracy": 0.56 } }, "massive_nb": { "n": 2948, "correct": 1511, "accuracy": 0.512550881953867, "accuracy_wilson95": [ 0.494502922473108, 0.5305661745743885 ], "accuracy_cluster95": [ 0.49491180461329715, 0.5315468113975577 ], "clusters": 2948, "macro_f1": 0.5253281273250409, "balanced_accuracy": 0.5480212570537378, "confusion_matrix": [ [ 92, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0 ], [ 3, 26, 0, 0, 0, 0, 2, 16, 0, 0, 0, 1, 12, 0, 0, 0, 1, 0 ], [ 65, 0, 161, 9, 16, 2, 36, 28, 1, 8, 8, 2, 47, 3, 1, 1, 3, 9 ], [ 0, 0, 0, 64, 0, 0, 0, 1, 0, 0, 0, 0, 3, 0, 0, 3, 0, 0 ], [ 27, 0, 0, 0, 46, 0, 1, 1, 0, 0, 0, 0, 10, 0, 0, 0, 0, 15 ], [ 1, 0, 4, 2, 1, 164, 24, 19, 15, 0, 1, 0, 30, 1, 4, 0, 1, 2 ], [ 6, 1, 5, 9, 0, 0, 35, 11, 1, 5, 0, 7, 90, 5, 0, 2, 5, 5 ], [ 17, 1, 0, 20, 0, 0, 5, 145, 1, 1, 0, 3, 16, 0, 0, 3, 0, 2 ], [ 0, 5, 1, 9, 0, 0, 9, 9, 68, 7, 0, 3, 25, 1, 1, 4, 0, 0 ], [ 0, 6, 0, 0, 0, 0, 1, 1, 0, 62, 0, 6, 5, 0, 0, 0, 0, 0 ], [ 0, 0, 1, 2, 1, 0, 13, 0, 0, 0, 48, 1, 51, 2, 1, 0, 1, 3 ], [ 3, 65, 1, 0, 0, 0, 20, 29, 3, 77, 2, 127, 54, 2, 0, 0, 0, 2 ], [ 1, 0, 6, 1, 9, 0, 45, 22, 1, 7, 3, 3, 169, 9, 0, 0, 3, 8 ], [ 0, 0, 2, 3, 2, 0, 13, 0, 0, 4, 2, 6, 16, 34, 0, 4, 6, 2 ], [ 0, 2, 3, 2, 0, 0, 12, 15, 0, 1, 2, 0, 15, 1, 49, 2, 1, 0 ], [ 0, 0, 1, 10, 2, 0, 3, 8, 0, 0, 0, 0, 6, 1, 0, 25, 1, 0 ], [ 5, 0, 2, 0, 1, 1, 0, 3, 0, 0, 0, 0, 21, 1, 0, 2, 87, 0 ], [ 4, 0, 0, 1, 0, 0, 5, 22, 0, 0, 0, 1, 8, 1, 0, 0, 1, 109 ] ], "labels": [ "alarm", "audio", "calendar", "cooking", "datetime", "email", "general", "iot", "lists", "music", "news", "play", "qa", "recommendation", "social", "takeaway", "transport", "weather" ], "per_class": { "alarm": { "precision": 0.4107142857142857, "recall": 0.9583333333333334, "f1-score": 0.575, "support": 96.0 }, "audio": { "precision": 0.24528301886792453, "recall": 0.4262295081967213, "f1-score": 0.31137724550898205, "support": 61.0 }, "calendar": { "precision": 0.8609625668449198, "recall": 0.4025, "f1-score": 0.5485519591141397, "support": 400.0 }, "cooking": { "precision": 0.48484848484848486, "recall": 0.9014084507042254, "f1-score": 0.6305418719211823, "support": 71.0 }, "datetime": { "precision": 0.5897435897435898, "recall": 0.46, "f1-score": 0.5168539325842697, "support": 100.0 }, "email": { "precision": 0.9820359281437125, "recall": 0.6096654275092936, "f1-score": 0.7522935779816514, "support": 269.0 }, "general": { "precision": 0.15625, "recall": 0.18716577540106952, "f1-score": 0.170316301703163, "support": 187.0 }, "iot": { "precision": 0.4380664652567976, "recall": 0.677570093457944, "f1-score": 0.5321100917431193, "support": 214.0 }, "lists": { "precision": 0.7555555555555555, "recall": 0.4788732394366197, "f1-score": 0.5862068965517241, "support": 142.0 }, "music": { "precision": 0.36046511627906974, "recall": 0.7654320987654321, "f1-score": 0.4901185770750988, "support": 81.0 }, "news": { "precision": 0.7272727272727273, "recall": 0.3870967741935484, "f1-score": 0.5052631578947369, "support": 124.0 }, "play": { "precision": 0.79375, "recall": 0.32987012987012987, "f1-score": 0.46605504587155966, "support": 385.0 }, "qa": { "precision": 0.2913793103448276, "recall": 0.5888501742160279, "f1-score": 0.38985005767012687, "support": 287.0 }, "recommendation": { "precision": 0.5573770491803278, "recall": 0.3617021276595745, "f1-score": 0.43870967741935485, "support": 94.0 }, "social": { "precision": 0.8596491228070176, "recall": 0.4666666666666667, "f1-score": 0.6049382716049383, "support": 105.0 }, "takeaway": { "precision": 0.5434782608695652, "recall": 0.43859649122807015, "f1-score": 0.4854368932038835, "support": 57.0 }, "transport": { "precision": 0.7909090909090909, "recall": 0.7073170731707317, "f1-score": 0.7467811158798283, "support": 123.0 }, "weather": { "precision": 0.6942675159235668, "recall": 0.7171052631578947, "f1-score": 0.7055016181229773, "support": 152.0 }, "accuracy": 0.512550881953867, "macro avg": { "precision": 0.5856671160311924, "recall": 0.5480212570537378, "f1-score": 0.5253281273250409, "support": 2948.0 }, "weighted avg": { "precision": 0.6348848159767395, "recall": 0.512550881953867, "f1-score": 0.5246219713283041, "support": 2948.0 } }, "raw_calibration": { "ece": 0.25267516416760444, "brier": 0.7210398823718983, "nll": 2.0916191748042614, "mean_confidence": 0.7652260461214715, "reliability": [ { "low": 0.0, "high": 0.1, "n": 0, "accuracy": null, "confidence": null }, { "low": 0.1, "high": 0.2, "n": 6, "accuracy": 0.16666666666666666, "confidence": 0.18056880866818295 }, { "low": 0.2, "high": 0.3, "n": 79, "accuracy": 0.22784810126582278, "confidence": 0.25941197133239646 }, { "low": 0.3, "high": 0.4, "n": 215, "accuracy": 0.22325581395348837, "confidence": 0.3542422926169051 }, { "low": 0.4, "high": 0.5, "n": 259, "accuracy": 0.21235521235521235, "confidence": 0.4516124027544274 }, { "low": 0.5, "high": 0.6, "n": 267, "accuracy": 0.30711610486891383, "confidence": 0.5489878128297793 }, { "low": 0.6, "high": 0.7, "n": 253, "accuracy": 0.3715415019762846, "confidence": 0.6506273217034346 }, { "low": 0.7, "high": 0.8, "n": 261, "accuracy": 0.37547892720306514, "confidence": 0.749426223888052 }, { "low": 0.8, "high": 0.9, "n": 305, "accuracy": 0.45901639344262296, "confidence": 0.8512321328412782 }, { "low": 0.9, "high": 1.0, "n": 1303, "accuracy": 0.7482732156561781, "confidence": 0.9783309415079868 } ], "confident_errors": 328, "confident_count": 1303 }, "fitted_calibration": { "ece": 0.0337482760618665, "brier": 0.6301199436053887, "nll": 1.6548761769565377, "mean_confidence": 0.5203192133912973, "reliability": [ { "low": 0.0, "high": 0.1, "n": 0, "accuracy": null, "confidence": null }, { "low": 0.1, "high": 0.2, "n": 274, "accuracy": 0.19708029197080293, "confidence": 0.170093713357483 }, { "low": 0.2, "high": 0.3, "n": 579, "accuracy": 0.2694300518134715, "confidence": 0.24903106533412708 }, { "low": 0.3, "high": 0.4, "n": 437, "accuracy": 0.3890160183066362, "confidence": 0.3453338611788815 }, { "low": 0.4, "high": 0.5, "n": 311, "accuracy": 0.4437299035369775, "confidence": 0.4488221455093922 }, { "low": 0.5, "high": 0.6, "n": 273, "accuracy": 0.4725274725274725, "confidence": 0.5481968859671618 }, { "low": 0.6, "high": 0.7, "n": 245, "accuracy": 0.6326530612244898, "confidence": 0.6489098911473891 }, { "low": 0.7, "high": 0.8, "n": 173, "accuracy": 0.6936416184971098, "confidence": 0.7463921354131392 }, { "low": 0.8, "high": 0.9, "n": 182, "accuracy": 0.7912087912087912, "confidence": 0.8523078642568854 }, { "low": 0.9, "high": 1.0, "n": 474, "accuracy": 0.9388185654008439, "confidence": 0.9698844900404449 } ], "confident_errors": 29, "confident_count": 474 }, "temperature": 2.0772798123247496, "calibration_n": 995, "policy_validation_n": 996, "policy_selection": null, "policy_test": { "threshold": null, "accepted": 0, "coverage": 0.0, "errors": 0, "accuracy": null, "accuracy_wilson95": [ null, null ], "errors_per_1000_total": 0.0, "review_per_1000": 1000.0 }, "fixed_thresholds_raw": [ { "threshold": 0.5, "accepted": 2389, "coverage": 0.8103799185888738, "errors": 1000, "accuracy": 0.5814148179154458, "accuracy_wilson95": [ 0.5615173500065495, 0.6010508797297064 ], "errors_per_1000_total": 339.21302578019, "review_per_1000": 189.6200814111262 }, { "threshold": 0.7, "accepted": 1869, "coverage": 0.6339891451831751, "errors": 656, "accuracy": 0.6490101658640984, "accuracy_wilson95": [ 0.6270865999092121, 0.6703224505881729 ], "errors_per_1000_total": 222.5237449118046, "review_per_1000": 366.0108548168249 }, { "threshold": 0.8, "accepted": 1608, "coverage": 0.5454545454545454, "errors": 493, "accuracy": 0.693407960199005, "accuracy_wilson95": [ 0.6704330079741194, 0.7154610243428599 ], "errors_per_1000_total": 167.23202170963364, "review_per_1000": 454.54545454545456 }, { "threshold": 0.9, "accepted": 1303, "coverage": 0.44199457259158753, "errors": 328, "accuracy": 0.748273215656178, "accuracy_wilson95": [ 0.7240015860995028, 0.7710852476039971 ], "errors_per_1000_total": 111.2618724559023, "review_per_1000": 558.0054274084125 }, { "threshold": 0.95, "accepted": 1074, "coverage": 0.364314789687924, "errors": 219, "accuracy": 0.7960893854748603, "accuracy_wilson95": [ 0.7709578530157701, 0.8191103752892266 ], "errors_per_1000_total": 74.2876526458616, "review_per_1000": 635.685210312076 }, { "threshold": 0.99, "accepted": 697, "coverage": 0.2364314789687924, "errors": 79, "accuracy": 0.8866571018651364, "accuracy_wilson95": [ 0.8609722525156669, 0.9081032541003018 ], "errors_per_1000_total": 26.797829036635004, "review_per_1000": 763.5685210312075 } ], "fixed_thresholds_fitted": [ { "threshold": 0.5, "accepted": 1347, "coverage": 0.4569199457259159, "errors": 354, "accuracy": 0.7371937639198218, "accuracy_wilson95": [ 0.7130373124915013, 0.7600011746140523 ], "errors_per_1000_total": 120.08141112618723, "review_per_1000": 543.0800542740841 }, { "threshold": 0.7, "accepted": 829, "coverage": 0.2812075983717775, "errors": 120, "accuracy": 0.8552472858866104, "accuracy_wilson95": [ 0.8296565535310505, 0.8775608811461635 ], "errors_per_1000_total": 40.7055630936228, "review_per_1000": 718.7924016282226 }, { "threshold": 0.8, "accepted": 656, "coverage": 0.2225237449118046, "errors": 67, "accuracy": 0.8978658536585366, "accuracy_wilson95": [ 0.8723280312092496, 0.9187710927120447 ], "errors_per_1000_total": 22.727272727272727, "review_per_1000": 777.4762550881954 }, { "threshold": 0.9, "accepted": 474, "coverage": 0.16078697421981003, "errors": 29, "accuracy": 0.9388185654008439, "accuracy_wilson95": [ 0.9135146348777272, 0.9570670032804225 ], "errors_per_1000_total": 9.837177747625509, "review_per_1000": 839.2130257801899 }, { "threshold": 0.95, "accepted": 334, "coverage": 0.11329715061058344, "errors": 10, "accuracy": 0.9700598802395209, "accuracy_wilson95": [ 0.9457726901213542, 0.9836573481459511 ], "errors_per_1000_total": 3.3921302578018997, "review_per_1000": 886.7028493894165 }, { "threshold": 0.99, "accepted": 211, "coverage": 0.07157394843962007, "errors": 4, "accuracy": 0.981042654028436, "accuracy_wilson95": [ 0.9522790298343307, 0.9926037740448919 ], "errors_per_1000_total": 1.3568521031207597, "review_per_1000": 928.42605156038 } ], "audit": { "truncated_inputs": 0, "options_truncated": 0, "instruction_truncated": false }, "timing": { "batch_size": 8, "batch_p50_ms": 34.771200036630034, "batch_p95_ms": 78.81516003981227, "samples_per_second": 198.4142020731496, "measured_batches": 369, "single_p50_ms": 28.419699985533953, "single_p95_ms": 37.18851002631709, "single_n": 50 }, "option_order_audit": { "n": 100, "changed_predictions": 23, "original_accuracy": 0.5, "reversed_accuracy": 0.49 } }, "norec_no": { "n": 1173, "correct": 696, "accuracy": 0.5933503836317136, "accuracy_wilson95": [ 0.5649796692146967, 0.6211116673528123 ], "accuracy_cluster95": [ 0.5630512662579499, 0.6223901675044992 ], "clusters": 47, "macro_f1": 0.5337026667663335, "balanced_accuracy": 0.5305491625750302, "confusion_matrix": [ [ 75, 102, 5 ], [ 87, 462, 41 ], [ 32, 210, 159 ] ], "labels": [ "negative", "neutral", "positive" ], "per_class": { "negative": { "precision": 0.3865979381443299, "recall": 0.41208791208791207, "f1-score": 0.39893617021276595, "support": 182.0 }, "neutral": { "precision": 0.5968992248062015, "recall": 0.7830508474576271, "f1-score": 0.6774193548387096, "support": 590.0 }, "positive": { "precision": 0.775609756097561, "recall": 0.39650872817955113, "f1-score": 0.5247524752475248, "support": 401.0 }, "accuracy": 0.5933503836317136, "macro avg": { "precision": 0.5863689730160307, "recall": 0.5305491625750302, "f1-score": 0.5337026667663335, "support": 1173.0 }, "weighted avg": { "precision": 0.6253630686897262, "recall": 0.5933503836317136, "f1-score": 0.5820200723851828, "support": 1173.0 } }, "raw_calibration": { "ece": 0.21590891215293406, "brier": 0.6160334702788033, "nll": 1.2015411333167527, "mean_confidence": 0.8081435242586459, "reliability": [ { "low": 0.0, "high": 0.1, "n": 0, "accuracy": null, "confidence": null }, { "low": 0.1, "high": 0.2, "n": 0, "accuracy": null, "confidence": null }, { "low": 0.2, "high": 0.3, "n": 0, "accuracy": null, "confidence": null }, { "low": 0.3, "high": 0.4, "n": 1, "accuracy": 1.0, "confidence": 0.3456 }, { "low": 0.4, "high": 0.5, "n": 20, "accuracy": 0.4, "confidence": 0.47843322366213226 }, { "low": 0.5, "high": 0.6, "n": 126, "accuracy": 0.4365079365079365, "confidence": 0.5511819088863349 }, { "low": 0.6, "high": 0.7, "n": 137, "accuracy": 0.49635036496350365, "confidence": 0.6522560671108599 }, { "low": 0.7, "high": 0.8, "n": 182, "accuracy": 0.6043956043956044, "confidence": 0.7548053416424766 }, { "low": 0.8, "high": 0.9, "n": 313, "accuracy": 0.6325878594249201, "confidence": 0.8547603506050893 }, { "low": 0.9, "high": 1.0, "n": 394, "accuracy": 0.649746192893401, "confidence": 0.9500394057105566 } ], "confident_errors": 138, "confident_count": 394 }, "fitted_calibration": { "ece": 0.03540740365141427, "brier": 0.5399921507934391, "nll": 0.8993513465988737, "mean_confidence": 0.5853293744641144, "reliability": [ { "low": 0.0, "high": 0.1, "n": 0, "accuracy": null, "confidence": null }, { "low": 0.1, "high": 0.2, "n": 0, "accuracy": null, "confidence": null }, { "low": 0.2, "high": 0.3, "n": 0, "accuracy": null, "confidence": null }, { "low": 0.3, "high": 0.4, "n": 10, "accuracy": 0.5, "confidence": 0.3791413058431123 }, { "low": 0.4, "high": 0.5, "n": 222, "accuracy": 0.45045045045045046, "confidence": 0.46482529020721836 }, { "low": 0.5, "high": 0.6, "n": 456, "accuracy": 0.6052631578947368, "confidence": 0.5520566289982839 }, { "low": 0.6, "high": 0.7, "n": 335, "accuracy": 0.6358208955223881, "confidence": 0.6463278679215365 }, { "low": 0.7, "high": 0.8, "n": 137, "accuracy": 0.6861313868613139, "confidence": 0.734988325307795 }, { "low": 0.8, "high": 0.9, "n": 13, "accuracy": 0.6153846153846154, "confidence": 0.8198207398363386 }, { "low": 0.9, "high": 1.0, "n": 0, "accuracy": null, "confidence": null } ], "confident_errors": 0, "confident_count": 0 }, "temperature": 2.764245334212195, "calibration_n": 672, "policy_validation_n": 702, "policy_selection": null, "policy_test": { "threshold": null, "accepted": 0, "coverage": 0.0, "errors": 0, "accuracy": null, "accuracy_wilson95": [ null, null ], "errors_per_1000_total": 0.0, "review_per_1000": 1000.0 }, "fixed_thresholds_raw": [ { "threshold": 0.5, "accepted": 1152, "coverage": 0.9820971867007673, "errors": 465, "accuracy": 0.5963541666666667, "accuracy_wilson95": [ 0.5677474357209324, 0.6243204281766879 ], "errors_per_1000_total": 396.4194373401534, "review_per_1000": 17.90281329923271 }, { "threshold": 0.7, "accepted": 889, "coverage": 0.7578857630008525, "errors": 325, "accuracy": 0.6344206974128235, "accuracy_wilson95": [ 0.6022477291921602, 0.6654369729896605 ], "errors_per_1000_total": 277.0673486786019, "review_per_1000": 242.11423699914747 }, { "threshold": 0.8, "accepted": 707, "coverage": 0.6027280477408354, "errors": 253, "accuracy": 0.6421499292786421, "accuracy_wilson95": [ 0.6061338037700056, 0.6766296697244422 ], "errors_per_1000_total": 215.68627450980392, "review_per_1000": 397.27195225916455 }, { "threshold": 0.9, "accepted": 394, "coverage": 0.3358908780903666, "errors": 138, "accuracy": 0.649746192893401, "accuracy_wilson95": [ 0.6014013570540264, 0.6951992042594746 ], "errors_per_1000_total": 117.6470588235294, "review_per_1000": 664.1091219096334 }, { "threshold": 0.95, "accepted": 207, "coverage": 0.17647058823529413, "errors": 72, "accuracy": 0.6521739130434783, "accuracy_wilson95": [ 0.5850531610297626, 0.7137495524042071 ], "errors_per_1000_total": 61.38107416879796, "review_per_1000": 823.5294117647059 }, { "threshold": 0.99, "accepted": 17, "coverage": 0.014492753623188406, "errors": 6, "accuracy": 0.6470588235294117, "accuracy_wilson95": [ 0.41300363485218694, 0.8269027960275627 ], "errors_per_1000_total": 5.115089514066496, "review_per_1000": 985.5072463768116 } ], "fixed_thresholds_fitted": [ { "threshold": 0.5, "accepted": 941, "coverage": 0.8022165387894288, "errors": 350, "accuracy": 0.6280552603613178, "accuracy_wilson95": [ 0.5967120625179241, 0.6583571850792387 ], "errors_per_1000_total": 298.38022165387895, "review_per_1000": 197.78346121057123 }, { "threshold": 0.7, "accepted": 150, "coverage": 0.1278772378516624, "errors": 48, "accuracy": 0.6799999999999999, "accuracy_wilson95": [ 0.6016560026038712, 0.7493547094236214 ], "errors_per_1000_total": 40.92071611253197, "review_per_1000": 872.1227621483376 }, { "threshold": 0.8, "accepted": 13, "coverage": 0.011082693947144074, "errors": 5, "accuracy": 0.6153846153846154, "accuracy_wilson95": [ 0.355228915126657, 0.8229029220223743 ], "errors_per_1000_total": 4.262574595055414, "review_per_1000": 988.9173060528559 }, { "threshold": 0.9, "accepted": 0, "coverage": 0.0, "errors": 0, "accuracy": null, "accuracy_wilson95": [ null, null ], "errors_per_1000_total": 0.0, "review_per_1000": 1000.0 }, { "threshold": 0.95, "accepted": 0, "coverage": 0.0, "errors": 0, "accuracy": null, "accuracy_wilson95": [ null, null ], "errors_per_1000_total": 0.0, "review_per_1000": 1000.0 }, { "threshold": 0.99, "accepted": 0, "coverage": 0.0, "errors": 0, "accuracy": null, "accuracy_wilson95": [ null, null ], "errors_per_1000_total": 0.0, "review_per_1000": 1000.0 } ], "audit": { "truncated_inputs": 0, "options_truncated": 0, "instruction_truncated": false }, "timing": { "batch_size": 8, "batch_p50_ms": 29.84079997986555, "batch_p95_ms": 41.95168998558074, "samples_per_second": 251.44467614909357, "measured_batches": 147, "single_p50_ms": 29.477650008630008, "single_p95_ms": 37.96153999282978, "single_n": 50 }, "option_order_audit": { "n": 100, "changed_predictions": 9, "original_accuracy": 0.66, "reversed_accuracy": 0.67 } } }, "diagnostics": { "support_en_prompt_en_route": { "n": 30, "correct": 20, "accuracy": 0.6666666666666666, "accuracy_wilson95": [ 0.4878005164454384, 0.8076950191632386 ], "accuracy_cluster95": null, "clusters": 30, "macro_f1": 0.6575757575757575, "balanced_accuracy": 0.6666666666666667, "confusion_matrix": [ [ 6, 0, 0, 0, 0 ], [ 1, 4, 1, 0, 0 ], [ 3, 0, 3, 0, 0 ], [ 0, 1, 0, 5, 0 ], [ 2, 1, 1, 0, 2 ] ], "labels": [ "billing", "technical", "sales", "account", "other" ], "per_class": { "billing": { "precision": 0.5, "recall": 1.0, "f1-score": 0.6666666666666666, "support": 6.0 }, "technical": { "precision": 0.6666666666666666, "recall": 0.6666666666666666, "f1-score": 0.6666666666666666, "support": 6.0 }, "sales": { "precision": 0.6, "recall": 0.5, "f1-score": 0.5454545454545454, "support": 6.0 }, "account": { "precision": 1.0, "recall": 0.8333333333333334, "f1-score": 0.9090909090909091, "support": 6.0 }, "other": { "precision": 1.0, "recall": 0.3333333333333333, "f1-score": 0.5, "support": 6.0 }, "accuracy": 0.6666666666666666, "macro avg": { "precision": 0.7533333333333333, "recall": 0.6666666666666667, "f1-score": 0.6575757575757575, "support": 30.0 }, "weighted avg": { "precision": 0.7533333333333334, "recall": 0.6666666666666666, "f1-score": 0.6575757575757576, "support": 30.0 } }, "truncated_inputs": 0 }, "support_en_prompt_en_refund": { "n": 30, "correct": 26, "accuracy": 0.8666666666666667, "accuracy_wilson95": [ 0.703186733179637, 0.9469034451594526 ], "accuracy_cluster95": null, "clusters": 30, "macro_f1": 0.76, "balanced_accuracy": 0.9259259259259259, "confusion_matrix": [ [ 23, 4 ], [ 0, 3 ] ], "labels": [ "0", "1" ], "per_class": { "0": { "precision": 1.0, "recall": 0.8518518518518519, "f1-score": 0.92, "support": 27.0 }, "1": { "precision": 0.42857142857142855, "recall": 1.0, "f1-score": 0.6, "support": 3.0 }, "accuracy": 0.8666666666666667, "macro avg": { "precision": 0.7142857142857143, "recall": 0.9259259259259259, "f1-score": 0.76, "support": 30.0 }, "weighted avg": { "precision": 0.9428571428571428, "recall": 0.8666666666666667, "f1-score": 0.888, "support": 30.0 } }, "truncated_inputs": 0 }, "support_en_prompt_en_refund_choice": { "n": 30, "correct": 27, "accuracy": 0.9, "accuracy_wilson95": [ 0.7437891742081593, 0.9654001112526658 ], "accuracy_cluster95": null, "clusters": 30, "macro_f1": 0.803921568627451, "balanced_accuracy": 0.9444444444444444, "confusion_matrix": [ [ 24, 3 ], [ 0, 3 ] ], "labels": [ "A", "B" ], "per_class": { "A": { "precision": 1.0, "recall": 0.8888888888888888, "f1-score": 0.9411764705882353, "support": 27.0 }, "B": { "precision": 0.5, "recall": 1.0, "f1-score": 0.6666666666666666, "support": 3.0 }, "accuracy": 0.9, "macro avg": { "precision": 0.75, "recall": 0.9444444444444444, "f1-score": 0.803921568627451, "support": 30.0 }, "weighted avg": { "precision": 0.95, "recall": 0.9, "f1-score": 0.9137254901960784, "support": 30.0 } }, "truncated_inputs": 0 }, "support_en_prompt_en_urgency": { "n": 30, "correct": 3, "accuracy": 0.1, "accuracy_wilson95": [ 0.03459988874733419, 0.25621082579184085 ], "accuracy_cluster95": null, "clusters": 30, "macro_f1": 0.10833333333333334, "balanced_accuracy": 0.4166666666666667, "confusion_matrix": [ [ 0, 11, 13 ], [ 0, 1, 3 ], [ 0, 0, 2 ] ], "labels": [ "0", "1", "2" ], "per_class": { "0": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 24.0 }, "1": { "precision": 0.08333333333333333, "recall": 0.25, "f1-score": 0.125, "support": 4.0 }, "2": { "precision": 0.1111111111111111, "recall": 1.0, "f1-score": 0.2, "support": 2.0 }, "accuracy": 0.1, "macro avg": { "precision": 0.06481481481481481, "recall": 0.4166666666666667, "f1-score": 0.10833333333333334, "support": 30.0 }, "weighted avg": { "precision": 0.01851851851851852, "recall": 0.1, "f1-score": 0.030000000000000002, "support": 30.0 } }, "truncated_inputs": 0, "score_mae": 1.2210133333333333 }, "support_nb_prompt_en_route": { "n": 30, "correct": 21, "accuracy": 0.7, "accuracy_wilson95": [ 0.5212421254128504, 0.8333525173175619 ], "accuracy_cluster95": null, "clusters": 30, "macro_f1": 0.6690476190476191, "balanced_accuracy": 0.7, "confusion_matrix": [ [ 5, 0, 0, 0, 1 ], [ 0, 5, 0, 1, 0 ], [ 1, 0, 5, 0, 0 ], [ 0, 1, 0, 5, 0 ], [ 2, 2, 1, 0, 1 ] ], "labels": [ "billing", "technical", "sales", "account", "other" ], "per_class": { "billing": { "precision": 0.625, "recall": 0.8333333333333334, "f1-score": 0.7142857142857143, "support": 6.0 }, "technical": { "precision": 0.625, "recall": 0.8333333333333334, "f1-score": 0.7142857142857143, "support": 6.0 }, "sales": { "precision": 0.8333333333333334, "recall": 0.8333333333333334, "f1-score": 0.8333333333333334, "support": 6.0 }, "account": { "precision": 0.8333333333333334, "recall": 0.8333333333333334, "f1-score": 0.8333333333333334, "support": 6.0 }, "other": { "precision": 0.5, "recall": 0.16666666666666666, "f1-score": 0.25, "support": 6.0 }, "accuracy": 0.7, "macro avg": { "precision": 0.6833333333333333, "recall": 0.7, "f1-score": 0.6690476190476191, "support": 30.0 }, "weighted avg": { "precision": 0.6833333333333333, "recall": 0.7, "f1-score": 0.669047619047619, "support": 30.0 } }, "truncated_inputs": 0 }, "support_nb_prompt_en_refund": { "n": 30, "correct": 28, "accuracy": 0.9333333333333333, "accuracy_wilson95": [ 0.7867654163738308, 0.9815229762087295 ], "accuracy_cluster95": null, "clusters": 30, "macro_f1": 0.8557692307692308, "balanced_accuracy": 0.962962962962963, "confusion_matrix": [ [ 25, 2 ], [ 0, 3 ] ], "labels": [ "0", "1" ], "per_class": { "0": { "precision": 1.0, "recall": 0.9259259259259259, "f1-score": 0.9615384615384616, "support": 27.0 }, "1": { "precision": 0.6, "recall": 1.0, "f1-score": 0.75, "support": 3.0 }, "accuracy": 0.9333333333333333, "macro avg": { "precision": 0.8, "recall": 0.962962962962963, "f1-score": 0.8557692307692308, "support": 30.0 }, "weighted avg": { "precision": 0.9600000000000001, "recall": 0.9333333333333333, "f1-score": 0.9403846153846155, "support": 30.0 } }, "truncated_inputs": 0 }, "support_nb_prompt_en_refund_choice": { "n": 30, "correct": 28, "accuracy": 0.9333333333333333, "accuracy_wilson95": [ 0.7867654163738308, 0.9815229762087295 ], "accuracy_cluster95": null, "clusters": 30, "macro_f1": 0.8557692307692308, "balanced_accuracy": 0.962962962962963, "confusion_matrix": [ [ 25, 2 ], [ 0, 3 ] ], "labels": [ "A", "B" ], "per_class": { "A": { "precision": 1.0, "recall": 0.9259259259259259, "f1-score": 0.9615384615384616, "support": 27.0 }, "B": { "precision": 0.6, "recall": 1.0, "f1-score": 0.75, "support": 3.0 }, "accuracy": 0.9333333333333333, "macro avg": { "precision": 0.8, "recall": 0.962962962962963, "f1-score": 0.8557692307692308, "support": 30.0 }, "weighted avg": { "precision": 0.9600000000000001, "recall": 0.9333333333333333, "f1-score": 0.9403846153846155, "support": 30.0 } }, "truncated_inputs": 0 }, "support_nb_prompt_en_urgency": { "n": 30, "correct": 3, "accuracy": 0.1, "accuracy_wilson95": [ 0.03459988874733419, 0.25621082579184085 ], "accuracy_cluster95": null, "clusters": 30, "macro_f1": 0.10939112487100104, "balanced_accuracy": 0.4166666666666667, "confusion_matrix": [ [ 0, 12, 12 ], [ 0, 1, 3 ], [ 0, 0, 2 ] ], "labels": [ "0", "1", "2" ], "per_class": { "0": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 24.0 }, "1": { "precision": 0.07692307692307693, "recall": 0.25, "f1-score": 0.11764705882352941, "support": 4.0 }, "2": { "precision": 0.11764705882352941, "recall": 1.0, "f1-score": 0.21052631578947367, "support": 2.0 }, "accuracy": 0.1, "macro avg": { "precision": 0.06485671191553545, "recall": 0.4166666666666667, "f1-score": 0.10939112487100104, "support": 30.0 }, "weighted avg": { "precision": 0.01809954751131222, "recall": 0.1, "f1-score": 0.029721362229102165, "support": 30.0 } }, "truncated_inputs": 0, "score_mae": 1.2131766666666668 }, "support_nn_prompt_en_route": { "n": 30, "correct": 21, "accuracy": 0.7, "accuracy_wilson95": [ 0.5212421254128504, 0.8333525173175619 ], "accuracy_cluster95": null, "clusters": 30, "macro_f1": 0.6746753246753248, "balanced_accuracy": 0.7, "confusion_matrix": [ [ 5, 0, 0, 0, 1 ], [ 1, 5, 0, 0, 0 ], [ 1, 0, 5, 0, 0 ], [ 0, 1, 0, 5, 0 ], [ 2, 2, 1, 0, 1 ] ], "labels": [ "billing", "technical", "sales", "account", "other" ], "per_class": { "billing": { "precision": 0.5555555555555556, "recall": 0.8333333333333334, "f1-score": 0.6666666666666666, "support": 6.0 }, "technical": { "precision": 0.625, "recall": 0.8333333333333334, "f1-score": 0.7142857142857143, "support": 6.0 }, "sales": { "precision": 0.8333333333333334, "recall": 0.8333333333333334, "f1-score": 0.8333333333333334, "support": 6.0 }, "account": { "precision": 1.0, "recall": 0.8333333333333334, "f1-score": 0.9090909090909091, "support": 6.0 }, "other": { "precision": 0.5, "recall": 0.16666666666666666, "f1-score": 0.25, "support": 6.0 }, "accuracy": 0.7, "macro avg": { "precision": 0.7027777777777777, "recall": 0.7, "f1-score": 0.6746753246753248, "support": 30.0 }, "weighted avg": { "precision": 0.7027777777777778, "recall": 0.7, "f1-score": 0.6746753246753247, "support": 30.0 } }, "truncated_inputs": 0 }, "support_nn_prompt_en_refund": { "n": 30, "correct": 29, "accuracy": 0.9666666666666667, "accuracy_wilson95": [ 0.8332960900859082, 0.9940914096183874 ], "accuracy_cluster95": null, "clusters": 30, "macro_f1": 0.9191374663072776, "balanced_accuracy": 0.9814814814814814, "confusion_matrix": [ [ 26, 1 ], [ 0, 3 ] ], "labels": [ "0", "1" ], "per_class": { "0": { "precision": 1.0, "recall": 0.9629629629629629, "f1-score": 0.9811320754716981, "support": 27.0 }, "1": { "precision": 0.75, "recall": 1.0, "f1-score": 0.8571428571428571, "support": 3.0 }, "accuracy": 0.9666666666666667, "macro avg": { "precision": 0.875, "recall": 0.9814814814814814, "f1-score": 0.9191374663072776, "support": 30.0 }, "weighted avg": { "precision": 0.975, "recall": 0.9666666666666667, "f1-score": 0.9687331536388141, "support": 30.0 } }, "truncated_inputs": 0 }, "support_nn_prompt_en_refund_choice": { "n": 30, "correct": 29, "accuracy": 0.9666666666666667, "accuracy_wilson95": [ 0.8332960900859082, 0.9940914096183874 ], "accuracy_cluster95": null, "clusters": 30, "macro_f1": 0.9191374663072776, "balanced_accuracy": 0.9814814814814814, "confusion_matrix": [ [ 26, 1 ], [ 0, 3 ] ], "labels": [ "A", "B" ], "per_class": { "A": { "precision": 1.0, "recall": 0.9629629629629629, "f1-score": 0.9811320754716981, "support": 27.0 }, "B": { "precision": 0.75, "recall": 1.0, "f1-score": 0.8571428571428571, "support": 3.0 }, "accuracy": 0.9666666666666667, "macro avg": { "precision": 0.875, "recall": 0.9814814814814814, "f1-score": 0.9191374663072776, "support": 30.0 }, "weighted avg": { "precision": 0.975, "recall": 0.9666666666666667, "f1-score": 0.9687331536388141, "support": 30.0 } }, "truncated_inputs": 0 }, "support_nn_prompt_en_urgency": { "n": 30, "correct": 3, "accuracy": 0.1, "accuracy_wilson95": [ 0.03459988874733419, 0.25621082579184085 ], "accuracy_cluster95": null, "clusters": 30, "macro_f1": 0.11351909184726522, "balanced_accuracy": 0.4166666666666667, "confusion_matrix": [ [ 0, 14, 10 ], [ 0, 1, 3 ], [ 0, 0, 2 ] ], "labels": [ "0", "1", "2" ], "per_class": { "0": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 24.0 }, "1": { "precision": 0.06666666666666667, "recall": 0.25, "f1-score": 0.10526315789473684, "support": 4.0 }, "2": { "precision": 0.13333333333333333, "recall": 1.0, "f1-score": 0.23529411764705882, "support": 2.0 }, "accuracy": 0.1, "macro avg": { "precision": 0.06666666666666667, "recall": 0.4166666666666667, "f1-score": 0.11351909184726522, "support": 30.0 }, "weighted avg": { "precision": 0.017777777777777778, "recall": 0.1, "f1-score": 0.029721362229102165, "support": 30.0 } }, "truncated_inputs": 0, "score_mae": 1.19367 }, "support_nb_prompt_nb_route": { "n": 30, "correct": 20, "accuracy": 0.6666666666666666, "accuracy_wilson95": [ 0.4878005164454384, 0.8076950191632386 ], "accuracy_cluster95": null, "clusters": 30, "macro_f1": 0.6508690140269088, "balanced_accuracy": 0.6666666666666667, "confusion_matrix": [ [ 6, 0, 0, 0, 0 ], [ 1, 5, 0, 0, 0 ], [ 4, 0, 2, 0, 0 ], [ 0, 1, 0, 5, 0 ], [ 2, 1, 1, 0, 2 ] ], "labels": [ "billing", "technical", "sales", "account", "other" ], "per_class": { "billing": { "precision": 0.46153846153846156, "recall": 1.0, "f1-score": 0.631578947368421, "support": 6.0 }, "technical": { "precision": 0.7142857142857143, "recall": 0.8333333333333334, "f1-score": 0.7692307692307693, "support": 6.0 }, "sales": { "precision": 0.6666666666666666, "recall": 0.3333333333333333, "f1-score": 0.4444444444444444, "support": 6.0 }, "account": { "precision": 1.0, "recall": 0.8333333333333334, "f1-score": 0.9090909090909091, "support": 6.0 }, "other": { "precision": 1.0, "recall": 0.3333333333333333, "f1-score": 0.5, "support": 6.0 }, "accuracy": 0.6666666666666666, "macro avg": { "precision": 0.7684981684981685, "recall": 0.6666666666666667, "f1-score": 0.6508690140269088, "support": 30.0 }, "weighted avg": { "precision": 0.7684981684981685, "recall": 0.6666666666666666, "f1-score": 0.6508690140269088, "support": 30.0 } }, "truncated_inputs": 0 }, "support_nb_prompt_nb_refund": { "n": 30, "correct": 26, "accuracy": 0.8666666666666667, "accuracy_wilson95": [ 0.703186733179637, 0.9469034451594526 ], "accuracy_cluster95": null, "clusters": 30, "macro_f1": 0.76, "balanced_accuracy": 0.9259259259259259, "confusion_matrix": [ [ 23, 4 ], [ 0, 3 ] ], "labels": [ "0", "1" ], "per_class": { "0": { "precision": 1.0, "recall": 0.8518518518518519, "f1-score": 0.92, "support": 27.0 }, "1": { "precision": 0.42857142857142855, "recall": 1.0, "f1-score": 0.6, "support": 3.0 }, "accuracy": 0.8666666666666667, "macro avg": { "precision": 0.7142857142857143, "recall": 0.9259259259259259, "f1-score": 0.76, "support": 30.0 }, "weighted avg": { "precision": 0.9428571428571428, "recall": 0.8666666666666667, "f1-score": 0.888, "support": 30.0 } }, "truncated_inputs": 0 }, "support_nb_prompt_nb_refund_choice": { "n": 30, "correct": 28, "accuracy": 0.9333333333333333, "accuracy_wilson95": [ 0.7867654163738308, 0.9815229762087295 ], "accuracy_cluster95": null, "clusters": 30, "macro_f1": 0.8557692307692308, "balanced_accuracy": 0.962962962962963, "confusion_matrix": [ [ 25, 2 ], [ 0, 3 ] ], "labels": [ "A", "B" ], "per_class": { "A": { "precision": 1.0, "recall": 0.9259259259259259, "f1-score": 0.9615384615384616, "support": 27.0 }, "B": { "precision": 0.6, "recall": 1.0, "f1-score": 0.75, "support": 3.0 }, "accuracy": 0.9333333333333333, "macro avg": { "precision": 0.8, "recall": 0.962962962962963, "f1-score": 0.8557692307692308, "support": 30.0 }, "weighted avg": { "precision": 0.9600000000000001, "recall": 0.9333333333333333, "f1-score": 0.9403846153846155, "support": 30.0 } }, "truncated_inputs": 0 }, "support_nb_prompt_nb_urgency": { "n": 30, "correct": 4, "accuracy": 0.13333333333333333, "accuracy_wilson95": [ 0.053096554840547455, 0.296813266820363 ], "accuracy_cluster95": null, "clusters": 30, "macro_f1": 0.15584415584415584, "balanced_accuracy": 0.5, "confusion_matrix": [ [ 0, 16, 8 ], [ 0, 2, 2 ], [ 0, 0, 2 ] ], "labels": [ "0", "1", "2" ], "per_class": { "0": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 24.0 }, "1": { "precision": 0.1111111111111111, "recall": 0.5, "f1-score": 0.18181818181818182, "support": 4.0 }, "2": { "precision": 0.16666666666666666, "recall": 1.0, "f1-score": 0.2857142857142857, "support": 2.0 }, "accuracy": 0.13333333333333333, "macro avg": { "precision": 0.0925925925925926, "recall": 0.5, "f1-score": 0.15584415584415584, "support": 30.0 }, "weighted avg": { "precision": 0.02592592592592592, "recall": 0.13333333333333333, "f1-score": 0.04329004329004329, "support": 30.0 } }, "truncated_inputs": 0, "score_mae": 1.1059533333333333 }, "support_nn_prompt_nb_route": { "n": 30, "correct": 20, "accuracy": 0.6666666666666666, "accuracy_wilson95": [ 0.4878005164454384, 0.8076950191632386 ], "accuracy_cluster95": null, "clusters": 30, "macro_f1": 0.6352314352314352, "balanced_accuracy": 0.6666666666666667, "confusion_matrix": [ [ 6, 0, 0, 0, 0 ], [ 1, 5, 0, 0, 0 ], [ 3, 0, 3, 0, 0 ], [ 0, 1, 0, 5, 0 ], [ 2, 1, 2, 0, 1 ] ], "labels": [ "billing", "technical", "sales", "account", "other" ], "per_class": { "billing": { "precision": 0.5, "recall": 1.0, "f1-score": 0.6666666666666666, "support": 6.0 }, "technical": { "precision": 0.7142857142857143, "recall": 0.8333333333333334, "f1-score": 0.7692307692307693, "support": 6.0 }, "sales": { "precision": 0.6, "recall": 0.5, "f1-score": 0.5454545454545454, "support": 6.0 }, "account": { "precision": 1.0, "recall": 0.8333333333333334, "f1-score": 0.9090909090909091, "support": 6.0 }, "other": { "precision": 1.0, "recall": 0.16666666666666666, "f1-score": 0.2857142857142857, "support": 6.0 }, "accuracy": 0.6666666666666666, "macro avg": { "precision": 0.7628571428571429, "recall": 0.6666666666666667, "f1-score": 0.6352314352314352, "support": 30.0 }, "weighted avg": { "precision": 0.7628571428571429, "recall": 0.6666666666666666, "f1-score": 0.6352314352314353, "support": 30.0 } }, "truncated_inputs": 0 }, "support_nn_prompt_nb_refund": { "n": 30, "correct": 27, "accuracy": 0.9, "accuracy_wilson95": [ 0.7437891742081593, 0.9654001112526658 ], "accuracy_cluster95": null, "clusters": 30, "macro_f1": 0.803921568627451, "balanced_accuracy": 0.9444444444444444, "confusion_matrix": [ [ 24, 3 ], [ 0, 3 ] ], "labels": [ "0", "1" ], "per_class": { "0": { "precision": 1.0, "recall": 0.8888888888888888, "f1-score": 0.9411764705882353, "support": 27.0 }, "1": { "precision": 0.5, "recall": 1.0, "f1-score": 0.6666666666666666, "support": 3.0 }, "accuracy": 0.9, "macro avg": { "precision": 0.75, "recall": 0.9444444444444444, "f1-score": 0.803921568627451, "support": 30.0 }, "weighted avg": { "precision": 0.95, "recall": 0.9, "f1-score": 0.9137254901960784, "support": 30.0 } }, "truncated_inputs": 0 }, "support_nn_prompt_nb_refund_choice": { "n": 30, "correct": 28, "accuracy": 0.9333333333333333, "accuracy_wilson95": [ 0.7867654163738308, 0.9815229762087295 ], "accuracy_cluster95": null, "clusters": 30, "macro_f1": 0.8557692307692308, "balanced_accuracy": 0.962962962962963, "confusion_matrix": [ [ 25, 2 ], [ 0, 3 ] ], "labels": [ "A", "B" ], "per_class": { "A": { "precision": 1.0, "recall": 0.9259259259259259, "f1-score": 0.9615384615384616, "support": 27.0 }, "B": { "precision": 0.6, "recall": 1.0, "f1-score": 0.75, "support": 3.0 }, "accuracy": 0.9333333333333333, "macro avg": { "precision": 0.8, "recall": 0.962962962962963, "f1-score": 0.8557692307692308, "support": 30.0 }, "weighted avg": { "precision": 0.9600000000000001, "recall": 0.9333333333333333, "f1-score": 0.9403846153846155, "support": 30.0 } }, "truncated_inputs": 0 }, "support_nn_prompt_nb_urgency": { "n": 30, "correct": 5, "accuracy": 0.16666666666666666, "accuracy_wilson95": [ 0.0733654237184855, 0.3356435050641603 ], "accuracy_cluster95": null, "clusters": 30, "macro_f1": 0.20343434343434344, "balanced_accuracy": 0.5138888888888888, "confusion_matrix": [ [ 1, 18, 5 ], [ 0, 2, 2 ], [ 0, 0, 2 ] ], "labels": [ "0", "1", "2" ], "per_class": { "0": { "precision": 1.0, "recall": 0.041666666666666664, "f1-score": 0.08, "support": 24.0 }, "1": { "precision": 0.1, "recall": 0.5, "f1-score": 0.16666666666666666, "support": 4.0 }, "2": { "precision": 0.2222222222222222, "recall": 1.0, "f1-score": 0.36363636363636365, "support": 2.0 }, "accuracy": 0.16666666666666666, "macro avg": { "precision": 0.4407407407407408, "recall": 0.5138888888888888, "f1-score": 0.20343434343434344, "support": 30.0 }, "weighted avg": { "precision": 0.8281481481481481, "recall": 0.16666666666666666, "f1-score": 0.11046464646464645, "support": 30.0 } }, "truncated_inputs": 0, "score_mae": 1.0730866666666665 }, "minimal_pairs_en_prompt_en_refund": { "n": 12, "correct": 6, "accuracy": 0.5, "accuracy_wilson95": [ 0.2537815976337061, 0.7462184023662939 ], "accuracy_cluster95": null, "clusters": 12, "macro_f1": 0.4375, "balanced_accuracy": 0.5, "confusion_matrix": [ [ 1, 5 ], [ 1, 5 ] ], "labels": [ "0", "1" ], "per_class": { "0": { "precision": 0.5, "recall": 0.16666666666666666, "f1-score": 0.25, "support": 6.0 }, "1": { "precision": 0.5, "recall": 0.8333333333333334, "f1-score": 0.625, "support": 6.0 }, "accuracy": 0.5, "macro avg": { "precision": 0.5, "recall": 0.5, "f1-score": 0.4375, "support": 12.0 }, "weighted avg": { "precision": 0.5, "recall": 0.5, "f1-score": 0.4375, "support": 12.0 } }, "truncated_inputs": 0 }, "minimal_pairs_en_prompt_en_refund_choice": { "n": 12, "correct": 8, "accuracy": 0.6666666666666666, "accuracy_wilson95": [ 0.39062208887279953, 0.8618799089087867 ], "accuracy_cluster95": null, "clusters": 12, "macro_f1": 0.6571428571428571, "balanced_accuracy": 0.6666666666666667, "confusion_matrix": [ [ 3, 3 ], [ 1, 5 ] ], "labels": [ "A", "B" ], "per_class": { "A": { "precision": 0.75, "recall": 0.5, "f1-score": 0.6, "support": 6.0 }, "B": { "precision": 0.625, "recall": 0.8333333333333334, "f1-score": 0.7142857142857143, "support": 6.0 }, "accuracy": 0.6666666666666666, "macro avg": { "precision": 0.6875, "recall": 0.6666666666666667, "f1-score": 0.6571428571428571, "support": 12.0 }, "weighted avg": { "precision": 0.6875, "recall": 0.6666666666666666, "f1-score": 0.6571428571428571, "support": 12.0 } }, "truncated_inputs": 0 }, "minimal_pairs_nb_prompt_en_refund": { "n": 12, "correct": 7, "accuracy": 0.5833333333333334, "accuracy_wilson95": [ 0.31951131254954973, 0.8067396863412435 ], "accuracy_cluster95": null, "clusters": 12, "macro_f1": 0.5804195804195804, "balanced_accuracy": 0.5833333333333333, "confusion_matrix": [ [ 3, 3 ], [ 2, 4 ] ], "labels": [ "0", "1" ], "per_class": { "0": { "precision": 0.6, "recall": 0.5, "f1-score": 0.5454545454545454, "support": 6.0 }, "1": { "precision": 0.5714285714285714, "recall": 0.6666666666666666, "f1-score": 0.6153846153846154, "support": 6.0 }, "accuracy": 0.5833333333333334, "macro avg": { "precision": 0.5857142857142856, "recall": 0.5833333333333333, "f1-score": 0.5804195804195804, "support": 12.0 }, "weighted avg": { "precision": 0.5857142857142857, "recall": 0.5833333333333334, "f1-score": 0.5804195804195804, "support": 12.0 } }, "truncated_inputs": 0 }, "minimal_pairs_nb_prompt_en_refund_choice": { "n": 12, "correct": 8, "accuracy": 0.6666666666666666, "accuracy_wilson95": [ 0.39062208887279953, 0.8618799089087867 ], "accuracy_cluster95": null, "clusters": 12, "macro_f1": 0.625, "balanced_accuracy": 0.6666666666666666, "confusion_matrix": [ [ 2, 4 ], [ 0, 6 ] ], "labels": [ "A", "B" ], "per_class": { "A": { "precision": 1.0, "recall": 0.3333333333333333, "f1-score": 0.5, "support": 6.0 }, "B": { "precision": 0.6, "recall": 1.0, "f1-score": 0.75, "support": 6.0 }, "accuracy": 0.6666666666666666, "macro avg": { "precision": 0.8, "recall": 0.6666666666666666, "f1-score": 0.625, "support": 12.0 }, "weighted avg": { "precision": 0.7999999999999999, "recall": 0.6666666666666666, "f1-score": 0.625, "support": 12.0 } }, "truncated_inputs": 0 }, "minimal_pairs_nn_prompt_en_refund": { "n": 12, "correct": 6, "accuracy": 0.5, "accuracy_wilson95": [ 0.2537815976337061, 0.7462184023662939 ], "accuracy_cluster95": null, "clusters": 12, "macro_f1": 0.5, "balanced_accuracy": 0.5, "confusion_matrix": [ [ 3, 3 ], [ 3, 3 ] ], "labels": [ "0", "1" ], "per_class": { "0": { "precision": 0.5, "recall": 0.5, "f1-score": 0.5, "support": 6.0 }, "1": { "precision": 0.5, "recall": 0.5, "f1-score": 0.5, "support": 6.0 }, "accuracy": 0.5, "macro avg": { "precision": 0.5, "recall": 0.5, "f1-score": 0.5, "support": 12.0 }, "weighted avg": { "precision": 0.5, "recall": 0.5, "f1-score": 0.5, "support": 12.0 } }, "truncated_inputs": 0 }, "minimal_pairs_nn_prompt_en_refund_choice": { "n": 12, "correct": 8, "accuracy": 0.6666666666666666, "accuracy_wilson95": [ 0.39062208887279953, 0.8618799089087867 ], "accuracy_cluster95": null, "clusters": 12, "macro_f1": 0.625, "balanced_accuracy": 0.6666666666666666, "confusion_matrix": [ [ 2, 4 ], [ 0, 6 ] ], "labels": [ "A", "B" ], "per_class": { "A": { "precision": 1.0, "recall": 0.3333333333333333, "f1-score": 0.5, "support": 6.0 }, "B": { "precision": 0.6, "recall": 1.0, "f1-score": 0.75, "support": 6.0 }, "accuracy": 0.6666666666666666, "macro avg": { "precision": 0.8, "recall": 0.6666666666666666, "f1-score": 0.625, "support": 12.0 }, "weighted avg": { "precision": 0.7999999999999999, "recall": 0.6666666666666666, "f1-score": 0.625, "support": 12.0 } }, "truncated_inputs": 0 }, "minimal_pairs_nb_prompt_nb_refund": { "n": 12, "correct": 9, "accuracy": 0.75, "accuracy_wilson95": [ 0.46769466506643426, 0.9110583316059453 ], "accuracy_cluster95": null, "clusters": 12, "macro_f1": 0.7333333333333334, "balanced_accuracy": 0.75, "confusion_matrix": [ [ 3, 3 ], [ 0, 6 ] ], "labels": [ "0", "1" ], "per_class": { "0": { "precision": 1.0, "recall": 0.5, "f1-score": 0.6666666666666666, "support": 6.0 }, "1": { "precision": 0.6666666666666666, "recall": 1.0, "f1-score": 0.8, "support": 6.0 }, "accuracy": 0.75, "macro avg": { "precision": 0.8333333333333333, "recall": 0.75, "f1-score": 0.7333333333333334, "support": 12.0 }, "weighted avg": { "precision": 0.8333333333333334, "recall": 0.75, "f1-score": 0.7333333333333334, "support": 12.0 } }, "truncated_inputs": 0 }, "minimal_pairs_nb_prompt_nb_refund_choice": { "n": 12, "correct": 10, "accuracy": 0.8333333333333334, "accuracy_wilson95": [ 0.5519691377470266, 0.9530348578161463 ], "accuracy_cluster95": null, "clusters": 12, "macro_f1": 0.8285714285714285, "balanced_accuracy": 0.8333333333333333, "confusion_matrix": [ [ 4, 2 ], [ 0, 6 ] ], "labels": [ "A", "B" ], "per_class": { "A": { "precision": 1.0, "recall": 0.6666666666666666, "f1-score": 0.8, "support": 6.0 }, "B": { "precision": 0.75, "recall": 1.0, "f1-score": 0.8571428571428571, "support": 6.0 }, "accuracy": 0.8333333333333334, "macro avg": { "precision": 0.875, "recall": 0.8333333333333333, "f1-score": 0.8285714285714285, "support": 12.0 }, "weighted avg": { "precision": 0.875, "recall": 0.8333333333333334, "f1-score": 0.8285714285714286, "support": 12.0 } }, "truncated_inputs": 0 }, "minimal_pairs_nn_prompt_nb_refund": { "n": 12, "correct": 9, "accuracy": 0.75, "accuracy_wilson95": [ 0.46769466506643426, 0.9110583316059453 ], "accuracy_cluster95": null, "clusters": 12, "macro_f1": 0.7333333333333334, "balanced_accuracy": 0.75, "confusion_matrix": [ [ 3, 3 ], [ 0, 6 ] ], "labels": [ "0", "1" ], "per_class": { "0": { "precision": 1.0, "recall": 0.5, "f1-score": 0.6666666666666666, "support": 6.0 }, "1": { "precision": 0.6666666666666666, "recall": 1.0, "f1-score": 0.8, "support": 6.0 }, "accuracy": 0.75, "macro avg": { "precision": 0.8333333333333333, "recall": 0.75, "f1-score": 0.7333333333333334, "support": 12.0 }, "weighted avg": { "precision": 0.8333333333333334, "recall": 0.75, "f1-score": 0.7333333333333334, "support": 12.0 } }, "truncated_inputs": 0 }, "minimal_pairs_nn_prompt_nb_refund_choice": { "n": 12, "correct": 11, "accuracy": 0.9166666666666666, "accuracy_wilson95": [ 0.646120088858883, 0.9851349055950829 ], "accuracy_cluster95": null, "clusters": 12, "macro_f1": 0.916083916083916, "balanced_accuracy": 0.9166666666666667, "confusion_matrix": [ [ 5, 1 ], [ 0, 6 ] ], "labels": [ "A", "B" ], "per_class": { "A": { "precision": 1.0, "recall": 0.8333333333333334, "f1-score": 0.9090909090909091, "support": 6.0 }, "B": { "precision": 0.8571428571428571, "recall": 1.0, "f1-score": 0.9230769230769231, "support": 6.0 }, "accuracy": 0.9166666666666666, "macro avg": { "precision": 0.9285714285714286, "recall": 0.9166666666666667, "f1-score": 0.916083916083916, "support": 12.0 }, "weighted avg": { "precision": 0.9285714285714285, "recall": 0.9166666666666666, "f1-score": 0.9160839160839161, "support": 12.0 } }, "truncated_inputs": 0 }, "long_input_en_default": { "n": 6, "correct": 6, "accuracy": 1.0, "accuracy_wilson95": [ 0.6096657120978346, 1.0 ], "accuracy_cluster95": null, "clusters": 6, "macro_f1": 0.2, "balanced_accuracy": 1.0, "confusion_matrix": [ [ 6, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 0 ] ], "labels": [ "billing", "technical", "sales", "account", "other" ], "per_class": { "billing": { "precision": 1.0, "recall": 1.0, "f1-score": 1.0, "support": 6.0 }, "technical": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 0.0 }, "sales": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 0.0 }, "account": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 0.0 }, "other": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 0.0 }, "accuracy": 1.0, "macro avg": { "precision": 0.2, "recall": 0.2, "f1-score": 0.2, "support": 6.0 }, "weighted avg": { "precision": 1.0, "recall": 1.0, "f1-score": 1.0, "support": 6.0 } }, "truncated_inputs": 0, "cases": [ { "id": "en-0-start-default", "position": "start", "filler_repeats": 0, "prediction": "billing", "label": "billing", "audit": { "state_tokens": 7, "state_budget": 975, "state_truncated": false, "max_len": 1024, "head_max_len": 256, "options_truncated": 0, "instruction_truncated": false }, "latency_ms": 23.47909996751696 }, { "id": "en-0-end-default", "position": "end", "filler_repeats": 0, "prediction": "billing", "label": "billing", "audit": { "state_tokens": 6, "state_budget": 975, "state_truncated": false, "max_len": 1024, "head_max_len": 256, "options_truncated": 0, "instruction_truncated": false }, "latency_ms": 34.55240000039339 }, { "id": "en-40-start-default", "position": "start", "filler_repeats": 40, "prediction": "billing", "label": "billing", "audit": { "state_tokens": 247, "state_budget": 975, "state_truncated": false, "max_len": 1024, "head_max_len": 256, "options_truncated": 0, "instruction_truncated": false }, "latency_ms": 28.704599943012 }, { "id": "en-40-end-default", "position": "end", "filler_repeats": 40, "prediction": "billing", "label": "billing", "audit": { "state_tokens": 246, "state_budget": 975, "state_truncated": false, "max_len": 1024, "head_max_len": 256, "options_truncated": 0, "instruction_truncated": false }, "latency_ms": 25.721199926920235 }, { "id": "en-160-start-default", "position": "start", "filler_repeats": 160, "prediction": "billing", "label": "billing", "audit": { "state_tokens": 967, "state_budget": 975, "state_truncated": false, "max_len": 1024, "head_max_len": 256, "options_truncated": 0, "instruction_truncated": false }, "latency_ms": 32.28230006061494 }, { "id": "en-160-end-default", "position": "end", "filler_repeats": 160, "prediction": "billing", "label": "billing", "audit": { "state_tokens": 966, "state_budget": 975, "state_truncated": false, "max_len": 1024, "head_max_len": 256, "options_truncated": 0, "instruction_truncated": false }, "latency_ms": 31.7480000667274 } ] }, "long_input_en_2048": { "n": 6, "correct": 6, "accuracy": 1.0, "accuracy_wilson95": [ 0.6096657120978346, 1.0 ], "accuracy_cluster95": null, "clusters": 6, "macro_f1": 0.2, "balanced_accuracy": 1.0, "confusion_matrix": [ [ 6, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 0 ] ], "labels": [ "billing", "technical", "sales", "account", "other" ], "per_class": { "billing": { "precision": 1.0, "recall": 1.0, "f1-score": 1.0, "support": 6.0 }, "technical": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 0.0 }, "sales": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 0.0 }, "account": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 0.0 }, "other": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 0.0 }, "accuracy": 1.0, "macro avg": { "precision": 0.2, "recall": 0.2, "f1-score": 0.2, "support": 6.0 }, "weighted avg": { "precision": 1.0, "recall": 1.0, "f1-score": 1.0, "support": 6.0 } }, "truncated_inputs": 0, "cases": [ { "id": "en-0-start-2048", "position": "start", "filler_repeats": 0, "prediction": "billing", "label": "billing", "audit": { "state_tokens": 7, "state_budget": 1999, "state_truncated": false, "max_len": 2048, "head_max_len": 256, "options_truncated": 0, "instruction_truncated": false }, "latency_ms": 21.59740007482469 }, { "id": "en-0-end-2048", "position": "end", "filler_repeats": 0, "prediction": "billing", "label": "billing", "audit": { "state_tokens": 6, "state_budget": 1999, "state_truncated": false, "max_len": 2048, "head_max_len": 256, "options_truncated": 0, "instruction_truncated": false }, "latency_ms": 23.3051999239251 }, { "id": "en-40-start-2048", "position": "start", "filler_repeats": 40, "prediction": "billing", "label": "billing", "audit": { "state_tokens": 247, "state_budget": 1999, "state_truncated": false, "max_len": 2048, "head_max_len": 256, "options_truncated": 0, "instruction_truncated": false }, "latency_ms": 25.028400006704032 }, { "id": "en-40-end-2048", "position": "end", "filler_repeats": 40, "prediction": "billing", "label": "billing", "audit": { "state_tokens": 246, "state_budget": 1999, "state_truncated": false, "max_len": 2048, "head_max_len": 256, "options_truncated": 0, "instruction_truncated": false }, "latency_ms": 28.475900064222515 }, { "id": "en-160-start-2048", "position": "start", "filler_repeats": 160, "prediction": "billing", "label": "billing", "audit": { "state_tokens": 967, "state_budget": 1999, "state_truncated": false, "max_len": 2048, "head_max_len": 256, "options_truncated": 0, "instruction_truncated": false }, "latency_ms": 31.085200025700033 }, { "id": "en-160-end-2048", "position": "end", "filler_repeats": 160, "prediction": "billing", "label": "billing", "audit": { "state_tokens": 966, "state_budget": 1999, "state_truncated": false, "max_len": 2048, "head_max_len": 256, "options_truncated": 0, "instruction_truncated": false }, "latency_ms": 28.268099995329976 } ] }, "long_input_nb_default": { "n": 6, "correct": 5, "accuracy": 0.8333333333333334, "accuracy_wilson95": [ 0.43649717781352987, 0.9699466302516935 ], "accuracy_cluster95": null, "clusters": 6, "macro_f1": 0.18181818181818182, "balanced_accuracy": 0.8333333333333334, "confusion_matrix": [ [ 5, 1, 0, 0, 0 ], [ 0, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 0 ] ], "labels": [ "billing", "technical", "sales", "account", "other" ], "per_class": { "billing": { "precision": 1.0, "recall": 0.8333333333333334, "f1-score": 0.9090909090909091, "support": 6.0 }, "technical": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 0.0 }, "sales": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 0.0 }, "account": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 0.0 }, "other": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 0.0 }, "accuracy": 0.8333333333333334, "macro avg": { "precision": 0.2, "recall": 0.16666666666666669, "f1-score": 0.18181818181818182, "support": 6.0 }, "weighted avg": { "precision": 1.0, "recall": 0.8333333333333334, "f1-score": 0.9090909090909091, "support": 6.0 } }, "truncated_inputs": 2, "cases": [ { "id": "nb-0-start-default", "position": "start", "filler_repeats": 0, "prediction": "billing", "label": "billing", "audit": { "state_tokens": 11, "state_budget": 975, "state_truncated": false, "max_len": 1024, "head_max_len": 256, "options_truncated": 0, "instruction_truncated": false }, "latency_ms": 26.15329995751381 }, { "id": "nb-0-end-default", "position": "end", "filler_repeats": 0, "prediction": "billing", "label": "billing", "audit": { "state_tokens": 10, "state_budget": 975, "state_truncated": false, "max_len": 1024, "head_max_len": 256, "options_truncated": 0, "instruction_truncated": false }, "latency_ms": 36.717799957841635 }, { "id": "nb-40-start-default", "position": "start", "filler_repeats": 40, "prediction": "billing", "label": "billing", "audit": { "state_tokens": 411, "state_budget": 975, "state_truncated": false, "max_len": 1024, "head_max_len": 256, "options_truncated": 0, "instruction_truncated": false }, "latency_ms": 39.985100040212274 }, { "id": "nb-40-end-default", "position": "end", "filler_repeats": 40, "prediction": "billing", "label": "billing", "audit": { "state_tokens": 410, "state_budget": 975, "state_truncated": false, "max_len": 1024, "head_max_len": 256, "options_truncated": 0, "instruction_truncated": false }, "latency_ms": 26.955399895086884 }, { "id": "nb-160-start-default", "position": "start", "filler_repeats": 160, "prediction": "billing", "label": "billing", "audit": { "state_tokens": 1611, "state_budget": 975, "state_truncated": true, "max_len": 1024, "head_max_len": 256, "options_truncated": 0, "instruction_truncated": false }, "latency_ms": 28.048700070939958 }, { "id": "nb-160-end-default", "position": "end", "filler_repeats": 160, "prediction": "technical", "label": "billing", "audit": { "state_tokens": 1610, "state_budget": 975, "state_truncated": true, "max_len": 1024, "head_max_len": 256, "options_truncated": 0, "instruction_truncated": false }, "latency_ms": 24.396799970418215 } ] }, "long_input_nb_2048": { "n": 6, "correct": 6, "accuracy": 1.0, "accuracy_wilson95": [ 0.6096657120978346, 1.0 ], "accuracy_cluster95": null, "clusters": 6, "macro_f1": 0.2, "balanced_accuracy": 1.0, "confusion_matrix": [ [ 6, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 0 ] ], "labels": [ "billing", "technical", "sales", "account", "other" ], "per_class": { "billing": { "precision": 1.0, "recall": 1.0, "f1-score": 1.0, "support": 6.0 }, "technical": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 0.0 }, "sales": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 0.0 }, "account": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 0.0 }, "other": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 0.0 }, "accuracy": 1.0, "macro avg": { "precision": 0.2, "recall": 0.2, "f1-score": 0.2, "support": 6.0 }, "weighted avg": { "precision": 1.0, "recall": 1.0, "f1-score": 1.0, "support": 6.0 } }, "truncated_inputs": 0, "cases": [ { "id": "nb-0-start-2048", "position": "start", "filler_repeats": 0, "prediction": "billing", "label": "billing", "audit": { "state_tokens": 11, "state_budget": 1999, "state_truncated": false, "max_len": 2048, "head_max_len": 256, "options_truncated": 0, "instruction_truncated": false }, "latency_ms": 39.89969997201115 }, { "id": "nb-0-end-2048", "position": "end", "filler_repeats": 0, "prediction": "billing", "label": "billing", "audit": { "state_tokens": 10, "state_budget": 1999, "state_truncated": false, "max_len": 2048, "head_max_len": 256, "options_truncated": 0, "instruction_truncated": false }, "latency_ms": 24.633800028823316 }, { "id": "nb-40-start-2048", "position": "start", "filler_repeats": 40, "prediction": "billing", "label": "billing", "audit": { "state_tokens": 411, "state_budget": 1999, "state_truncated": false, "max_len": 2048, "head_max_len": 256, "options_truncated": 0, "instruction_truncated": false }, "latency_ms": 26.27320005558431 }, { "id": "nb-40-end-2048", "position": "end", "filler_repeats": 40, "prediction": "billing", "label": "billing", "audit": { "state_tokens": 410, "state_budget": 1999, "state_truncated": false, "max_len": 2048, "head_max_len": 256, "options_truncated": 0, "instruction_truncated": false }, "latency_ms": 32.03690005466342 }, { "id": "nb-160-start-2048", "position": "start", "filler_repeats": 160, "prediction": "billing", "label": "billing", "audit": { "state_tokens": 1611, "state_budget": 1999, "state_truncated": false, "max_len": 2048, "head_max_len": 256, "options_truncated": 0, "instruction_truncated": false }, "latency_ms": 33.899499918334186 }, { "id": "nb-160-end-2048", "position": "end", "filler_repeats": 160, "prediction": "billing", "label": "billing", "audit": { "state_tokens": 1610, "state_budget": 1999, "state_truncated": false, "max_len": 2048, "head_max_len": 256, "options_truncated": 0, "instruction_truncated": false }, "latency_ms": 40.183300036005676 } ] }, "long_input_nn_default": { "n": 6, "correct": 5, "accuracy": 0.8333333333333334, "accuracy_wilson95": [ 0.43649717781352987, 0.9699466302516935 ], "accuracy_cluster95": null, "clusters": 6, "macro_f1": 0.18181818181818182, "balanced_accuracy": 0.8333333333333334, "confusion_matrix": [ [ 5, 1, 0, 0, 0 ], [ 0, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 0 ] ], "labels": [ "billing", "technical", "sales", "account", "other" ], "per_class": { "billing": { "precision": 1.0, "recall": 0.8333333333333334, "f1-score": 0.9090909090909091, "support": 6.0 }, "technical": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 0.0 }, "sales": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 0.0 }, "account": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 0.0 }, "other": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 0.0 }, "accuracy": 0.8333333333333334, "macro avg": { "precision": 0.2, "recall": 0.16666666666666669, "f1-score": 0.18181818181818182, "support": 6.0 }, "weighted avg": { "precision": 1.0, "recall": 0.8333333333333334, "f1-score": 0.9090909090909091, "support": 6.0 } }, "truncated_inputs": 2, "cases": [ { "id": "nn-0-start-default", "position": "start", "filler_repeats": 0, "prediction": "billing", "label": "billing", "audit": { "state_tokens": 12, "state_budget": 975, "state_truncated": false, "max_len": 1024, "head_max_len": 256, "options_truncated": 0, "instruction_truncated": false }, "latency_ms": 31.32710000500083 }, { "id": "nn-0-end-default", "position": "end", "filler_repeats": 0, "prediction": "billing", "label": "billing", "audit": { "state_tokens": 11, "state_budget": 975, "state_truncated": false, "max_len": 1024, "head_max_len": 256, "options_truncated": 0, "instruction_truncated": false }, "latency_ms": 27.919100015424192 }, { "id": "nn-40-start-default", "position": "start", "filler_repeats": 40, "prediction": "billing", "label": "billing", "audit": { "state_tokens": 412, "state_budget": 975, "state_truncated": false, "max_len": 1024, "head_max_len": 256, "options_truncated": 0, "instruction_truncated": false }, "latency_ms": 31.007800018414855 }, { "id": "nn-40-end-default", "position": "end", "filler_repeats": 40, "prediction": "billing", "label": "billing", "audit": { "state_tokens": 411, "state_budget": 975, "state_truncated": false, "max_len": 1024, "head_max_len": 256, "options_truncated": 0, "instruction_truncated": false }, "latency_ms": 29.024400049820542 }, { "id": "nn-160-start-default", "position": "start", "filler_repeats": 160, "prediction": "billing", "label": "billing", "audit": { "state_tokens": 1612, "state_budget": 975, "state_truncated": true, "max_len": 1024, "head_max_len": 256, "options_truncated": 0, "instruction_truncated": false }, "latency_ms": 31.32989990990609 }, { "id": "nn-160-end-default", "position": "end", "filler_repeats": 160, "prediction": "technical", "label": "billing", "audit": { "state_tokens": 1611, "state_budget": 975, "state_truncated": true, "max_len": 1024, "head_max_len": 256, "options_truncated": 0, "instruction_truncated": false }, "latency_ms": 23.744700010865927 } ] }, "long_input_nn_2048": { "n": 6, "correct": 6, "accuracy": 1.0, "accuracy_wilson95": [ 0.6096657120978346, 1.0 ], "accuracy_cluster95": null, "clusters": 6, "macro_f1": 0.2, "balanced_accuracy": 1.0, "confusion_matrix": [ [ 6, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 0 ] ], "labels": [ "billing", "technical", "sales", "account", "other" ], "per_class": { "billing": { "precision": 1.0, "recall": 1.0, "f1-score": 1.0, "support": 6.0 }, "technical": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 0.0 }, "sales": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 0.0 }, "account": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 0.0 }, "other": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 0.0 }, "accuracy": 1.0, "macro avg": { "precision": 0.2, "recall": 0.2, "f1-score": 0.2, "support": 6.0 }, "weighted avg": { "precision": 1.0, "recall": 1.0, "f1-score": 1.0, "support": 6.0 } }, "truncated_inputs": 0, "cases": [ { "id": "nn-0-start-2048", "position": "start", "filler_repeats": 0, "prediction": "billing", "label": "billing", "audit": { "state_tokens": 12, "state_budget": 1999, "state_truncated": false, "max_len": 2048, "head_max_len": 256, "options_truncated": 0, "instruction_truncated": false }, "latency_ms": 37.25349996238947 }, { "id": "nn-0-end-2048", "position": "end", "filler_repeats": 0, "prediction": "billing", "label": "billing", "audit": { "state_tokens": 11, "state_budget": 1999, "state_truncated": false, "max_len": 2048, "head_max_len": 256, "options_truncated": 0, "instruction_truncated": false }, "latency_ms": 31.508099986240268 }, { "id": "nn-40-start-2048", "position": "start", "filler_repeats": 40, "prediction": "billing", "label": "billing", "audit": { "state_tokens": 412, "state_budget": 1999, "state_truncated": false, "max_len": 2048, "head_max_len": 256, "options_truncated": 0, "instruction_truncated": false }, "latency_ms": 38.747800048440695 }, { "id": "nn-40-end-2048", "position": "end", "filler_repeats": 40, "prediction": "billing", "label": "billing", "audit": { "state_tokens": 411, "state_budget": 1999, "state_truncated": false, "max_len": 2048, "head_max_len": 256, "options_truncated": 0, "instruction_truncated": false }, "latency_ms": 27.46930008288473 }, { "id": "nn-160-start-2048", "position": "start", "filler_repeats": 160, "prediction": "billing", "label": "billing", "audit": { "state_tokens": 1612, "state_budget": 1999, "state_truncated": false, "max_len": 2048, "head_max_len": 256, "options_truncated": 0, "instruction_truncated": false }, "latency_ms": 36.248099990189075 }, { "id": "nn-160-end-2048", "position": "end", "filler_repeats": 160, "prediction": "billing", "label": "billing", "audit": { "state_tokens": 1611, "state_budget": 1999, "state_truncated": false, "max_len": 2048, "head_max_len": 256, "options_truncated": 0, "instruction_truncated": false }, "latency_ms": 30.207999981939793 } ] } } } }, "baselines": { "massive_en": { "majority": { "n": 2948, "correct": 400, "accuracy": 0.13568521031207598, "accuracy_wilson95": [ 0.12379633485221375, 0.1485223076069382 ], "accuracy_cluster95": [ 0.12313432835820895, 0.14755766621438263 ], "clusters": 2948, "macro_f1": 0.013274923669188901, "balanced_accuracy": 0.05555555555555555, "confusion_matrix": [ [ 0, 0, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 269, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 214, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 385, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 287, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] ], "labels": [ "alarm", "audio", "calendar", "cooking", "datetime", "email", "general", "iot", "lists", "music", "news", "play", "qa", "recommendation", "social", "takeaway", "transport", "weather" ], "per_class": { "alarm": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 96.0 }, "audio": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 61.0 }, "calendar": { "precision": 0.13568521031207598, "recall": 1.0, "f1-score": 0.23894862604540024, "support": 400.0 }, "cooking": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 71.0 }, "datetime": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 100.0 }, "email": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 269.0 }, "general": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 187.0 }, "iot": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 214.0 }, "lists": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 142.0 }, "music": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 81.0 }, "news": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 124.0 }, "play": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 385.0 }, "qa": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 287.0 }, "recommendation": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 94.0 }, "social": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 105.0 }, "takeaway": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 57.0 }, "transport": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 123.0 }, "weather": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 152.0 }, "accuracy": 0.13568521031207598, "macro avg": { "precision": 0.007538067239559776, "recall": 0.05555555555555555, "f1-score": 0.013274923669188901, "support": 2948.0 }, "weighted avg": { "precision": 0.01841047629743229, "recall": 0.13568521031207598, "f1-score": 0.03242179457875173, "support": 2948.0 } } }, "tfidf_linear_svm": { "n": 2948, "correct": 2643, "accuracy": 0.8965400271370421, "accuracy_wilson95": [ 0.8850250314888487, 0.9070229265793556 ], "accuracy_cluster95": [ 0.8853459972862958, 0.9073948439620081 ], "clusters": 2948, "macro_f1": 0.8909396939981217, "balanced_accuracy": 0.8872442350943942, "confusion_matrix": [ [ 93, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], [ 1, 56, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0 ], [ 0, 0, 380, 0, 2, 1, 4, 0, 1, 0, 4, 0, 0, 4, 2, 0, 0, 2 ], [ 0, 0, 0, 63, 0, 0, 2, 0, 2, 0, 0, 0, 4, 0, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 95, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 3 ], [ 0, 0, 3, 0, 0, 258, 2, 0, 0, 0, 1, 0, 2, 0, 1, 0, 1, 1 ], [ 0, 0, 13, 2, 0, 2, 103, 1, 2, 4, 8, 6, 30, 7, 1, 0, 2, 6 ], [ 0, 0, 0, 0, 0, 1, 3, 207, 1, 0, 0, 1, 0, 0, 0, 0, 0, 1 ], [ 0, 0, 6, 1, 0, 2, 2, 0, 128, 0, 1, 1, 0, 0, 0, 1, 0, 0 ], [ 0, 0, 0, 0, 0, 0, 3, 0, 2, 68, 0, 6, 1, 1, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 1, 1, 4, 2, 0, 0, 105, 1, 5, 3, 2, 0, 0, 0 ], [ 0, 3, 0, 0, 0, 0, 3, 1, 0, 7, 0, 365, 5, 0, 0, 1, 0, 0 ], [ 0, 1, 3, 0, 0, 3, 18, 0, 2, 0, 2, 1, 248, 1, 1, 0, 5, 2 ], [ 0, 0, 4, 0, 0, 0, 5, 0, 1, 0, 1, 0, 3, 77, 0, 1, 1, 1 ], [ 0, 0, 2, 0, 0, 4, 3, 0, 0, 0, 2, 0, 1, 0, 92, 1, 0, 0 ], [ 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 2, 1, 1, 0, 0, 50, 0, 0 ], [ 0, 0, 2, 0, 2, 0, 2, 0, 0, 0, 1, 1, 5, 1, 0, 0, 109, 0 ], [ 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 1, 2, 146 ] ], "labels": [ "alarm", "audio", "calendar", "cooking", "datetime", "email", "general", "iot", "lists", "music", "news", "play", "qa", "recommendation", "social", "takeaway", "transport", "weather" ], "per_class": { "alarm": { "precision": 0.9893617021276596, "recall": 0.96875, "f1-score": 0.9789473684210527, "support": 96.0 }, "audio": { "precision": 0.9180327868852459, "recall": 0.9180327868852459, "f1-score": 0.9180327868852459, "support": 61.0 }, "calendar": { "precision": 0.9112709832134293, "recall": 0.95, "f1-score": 0.9302325581395349, "support": 400.0 }, "cooking": { "precision": 0.9545454545454546, "recall": 0.8873239436619719, "f1-score": 0.9197080291970803, "support": 71.0 }, "datetime": { "precision": 0.95, "recall": 0.95, "f1-score": 0.95, "support": 100.0 }, "email": { "precision": 0.9485294117647058, "recall": 0.9591078066914498, "f1-score": 0.9537892791127541, "support": 269.0 }, "general": { "precision": 0.6518987341772152, "recall": 0.5508021390374331, "f1-score": 0.5971014492753624, "support": 187.0 }, "iot": { "precision": 0.9764150943396226, "recall": 0.9672897196261683, "f1-score": 0.971830985915493, "support": 214.0 }, "lists": { "precision": 0.920863309352518, "recall": 0.9014084507042254, "f1-score": 0.9110320284697508, "support": 142.0 }, "music": { "precision": 0.8607594936708861, "recall": 0.8395061728395061, "f1-score": 0.85, "support": 81.0 }, "news": { "precision": 0.8267716535433071, "recall": 0.8467741935483871, "f1-score": 0.8366533864541833, "support": 124.0 }, "play": { "precision": 0.9505208333333334, "recall": 0.948051948051948, "f1-score": 0.9492847854356307, "support": 385.0 }, "qa": { "precision": 0.8051948051948052, "recall": 0.8641114982578397, "f1-score": 0.8336134453781513, "support": 287.0 }, "recommendation": { "precision": 0.8105263157894737, "recall": 0.8191489361702128, "f1-score": 0.8148148148148148, "support": 94.0 }, "social": { "precision": 0.9292929292929293, "recall": 0.8761904761904762, "f1-score": 0.9019607843137255, "support": 105.0 }, "takeaway": { "precision": 0.9090909090909091, "recall": 0.8771929824561403, "f1-score": 0.8928571428571429, "support": 57.0 }, "transport": { "precision": 0.9083333333333333, "recall": 0.8861788617886179, "f1-score": 0.897119341563786, "support": 123.0 }, "weather": { "precision": 0.9012345679012346, "recall": 0.9605263157894737, "f1-score": 0.9299363057324841, "support": 152.0 }, "accuracy": 0.8965400271370421, "macro avg": { "precision": 0.8957023509753369, "recall": 0.8872442350943942, "f1-score": 0.8909396939981217, "support": 2948.0 }, "weighted avg": { "precision": 0.8950520490112034, "recall": 0.8965400271370421, "f1-score": 0.8952309896642847, "support": 2948.0 } }, "train_n": 11436, "training_seconds": 0.6509341000346467, "note": "Supervised comparator trained only on deduplicated official training split. Laya is zero-shot." }, "uniform_random_expected_accuracy": 0.05555555555555555 }, "massive_nb": { "majority": { "n": 2948, "correct": 400, "accuracy": 0.13568521031207598, "accuracy_wilson95": [ 0.12379633485221375, 0.1485223076069382 ], "accuracy_cluster95": [ 0.12313432835820895, 0.14755766621438263 ], "clusters": 2948, "macro_f1": 0.013274923669188901, "balanced_accuracy": 0.05555555555555555, "confusion_matrix": [ [ 0, 0, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 269, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 214, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 385, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 287, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] ], "labels": [ "alarm", "audio", "calendar", "cooking", "datetime", "email", "general", "iot", "lists", "music", "news", "play", "qa", "recommendation", "social", "takeaway", "transport", "weather" ], "per_class": { "alarm": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 96.0 }, "audio": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 61.0 }, "calendar": { "precision": 0.13568521031207598, "recall": 1.0, "f1-score": 0.23894862604540024, "support": 400.0 }, "cooking": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 71.0 }, "datetime": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 100.0 }, "email": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 269.0 }, "general": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 187.0 }, "iot": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 214.0 }, "lists": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 142.0 }, "music": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 81.0 }, "news": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 124.0 }, "play": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 385.0 }, "qa": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 287.0 }, "recommendation": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 94.0 }, "social": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 105.0 }, "takeaway": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 57.0 }, "transport": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 123.0 }, "weather": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 152.0 }, "accuracy": 0.13568521031207598, "macro avg": { "precision": 0.007538067239559776, "recall": 0.05555555555555555, "f1-score": 0.013274923669188901, "support": 2948.0 }, "weighted avg": { "precision": 0.01841047629743229, "recall": 0.13568521031207598, "f1-score": 0.03242179457875173, "support": 2948.0 } } }, "tfidf_linear_svm": { "n": 2948, "correct": 2628, "accuracy": 0.8914518317503393, "accuracy_wilson95": [ 0.8797090677985181, 0.9021757428179985 ], "accuracy_cluster95": [ 0.8802578018995929, 0.9029850746268657 ], "clusters": 2948, "macro_f1": 0.8835450159933101, "balanced_accuracy": 0.8747791654379592, "confusion_matrix": [ [ 91, 0, 3, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], [ 1, 54, 1, 0, 0, 0, 0, 3, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0 ], [ 1, 0, 383, 0, 1, 1, 2, 0, 2, 0, 3, 0, 3, 3, 1, 0, 0, 0 ], [ 0, 0, 0, 63, 0, 0, 3, 2, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0 ], [ 0, 0, 2, 1, 91, 0, 0, 0, 0, 0, 0, 1, 3, 0, 0, 0, 0, 2 ], [ 0, 0, 5, 0, 0, 257, 2, 0, 0, 0, 0, 0, 3, 0, 0, 0, 1, 1 ], [ 0, 3, 13, 5, 2, 3, 103, 2, 1, 2, 8, 10, 22, 7, 1, 0, 2, 3 ], [ 0, 1, 1, 0, 0, 1, 3, 205, 1, 0, 0, 2, 0, 0, 0, 0, 0, 0 ], [ 1, 0, 6, 2, 0, 2, 1, 0, 124, 0, 0, 1, 4, 0, 0, 1, 0, 0 ], [ 0, 0, 0, 0, 0, 0, 2, 0, 2, 62, 0, 13, 1, 1, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 106, 0, 11, 3, 2, 0, 0, 0 ], [ 0, 3, 1, 0, 0, 0, 3, 0, 0, 4, 0, 369, 4, 0, 0, 0, 0, 1 ], [ 0, 0, 5, 0, 2, 3, 16, 0, 1, 0, 1, 2, 250, 0, 1, 0, 3, 3 ], [ 0, 0, 4, 0, 0, 0, 3, 0, 0, 1, 1, 0, 4, 78, 0, 0, 2, 1 ], [ 1, 0, 4, 0, 0, 1, 2, 1, 0, 0, 2, 1, 2, 0, 89, 1, 0, 1 ], [ 0, 0, 1, 1, 0, 0, 3, 0, 0, 0, 1, 0, 2, 1, 0, 48, 0, 0 ], [ 0, 0, 2, 0, 1, 0, 2, 0, 0, 0, 0, 0, 4, 2, 0, 0, 112, 0 ], [ 0, 0, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0, 4, 1, 0, 0, 1, 143 ] ], "labels": [ "alarm", "audio", "calendar", "cooking", "datetime", "email", "general", "iot", "lists", "music", "news", "play", "qa", "recommendation", "social", "takeaway", "transport", "weather" ], "per_class": { "alarm": { "precision": 0.9578947368421052, "recall": 0.9479166666666666, "f1-score": 0.9528795811518325, "support": 96.0 }, "audio": { "precision": 0.8852459016393442, "recall": 0.8852459016393442, "f1-score": 0.8852459016393442, "support": 61.0 }, "calendar": { "precision": 0.8865740740740741, "recall": 0.9575, "f1-score": 0.9206730769230769, "support": 400.0 }, "cooking": { "precision": 0.875, "recall": 0.8873239436619719, "f1-score": 0.8811188811188811, "support": 71.0 }, "datetime": { "precision": 0.9285714285714286, "recall": 0.91, "f1-score": 0.9191919191919192, "support": 100.0 }, "email": { "precision": 0.9589552238805971, "recall": 0.9553903345724907, "f1-score": 0.957169459962756, "support": 269.0 }, "general": { "precision": 0.6912751677852349, "recall": 0.5508021390374331, "f1-score": 0.6130952380952381, "support": 187.0 }, "iot": { "precision": 0.9579439252336449, "recall": 0.9579439252336449, "f1-score": 0.9579439252336449, "support": 214.0 }, "lists": { "precision": 0.9393939393939394, "recall": 0.8732394366197183, "f1-score": 0.9051094890510949, "support": 142.0 }, "music": { "precision": 0.8985507246376812, "recall": 0.7654320987654321, "f1-score": 0.8266666666666667, "support": 81.0 }, "news": { "precision": 0.8688524590163934, "recall": 0.8548387096774194, "f1-score": 0.8617886178861789, "support": 124.0 }, "play": { "precision": 0.924812030075188, "recall": 0.9584415584415584, "f1-score": 0.9413265306122449, "support": 385.0 }, "qa": { "precision": 0.78125, "recall": 0.8710801393728222, "f1-score": 0.8237232289950577, "support": 287.0 }, "recommendation": { "precision": 0.8125, "recall": 0.8297872340425532, "f1-score": 0.8210526315789474, "support": 94.0 }, "social": { "precision": 0.9368421052631579, "recall": 0.8476190476190476, "f1-score": 0.89, "support": 105.0 }, "takeaway": { "precision": 0.96, "recall": 0.8421052631578947, "f1-score": 0.897196261682243, "support": 57.0 }, "transport": { "precision": 0.9256198347107438, "recall": 0.9105691056910569, "f1-score": 0.9180327868852459, "support": 123.0 }, "weather": { "precision": 0.9225806451612903, "recall": 0.9407894736842105, "f1-score": 0.9315960912052117, "support": 152.0 }, "accuracy": 0.8914518317503393, "macro avg": { "precision": 0.8951034553491569, "recall": 0.8747791654379592, "f1-score": 0.8835450159933101, "support": 2948.0 }, "weighted avg": { "precision": 0.8906138508129156, "recall": 0.8914518317503393, "f1-score": 0.8896906673327865, "support": 2948.0 } }, "train_n": 11035, "training_seconds": 0.6877627000212669, "note": "Supervised comparator trained only on deduplicated official training split. Laya is zero-shot." }, "uniform_random_expected_accuracy": 0.05555555555555555 }, "norec_no": { "majority": { "n": 1173, "correct": 590, "accuracy": 0.5029838022165388, "accuracy_wilson95": [ 0.4744079325073155, 0.5315401924058013 ], "accuracy_cluster95": [ 0.45114615037292216, 0.5514191376569767 ], "clusters": 47, "macro_f1": 0.2231045566269616, "balanced_accuracy": 0.3333333333333333, "confusion_matrix": [ [ 0, 182, 0 ], [ 0, 590, 0 ], [ 0, 401, 0 ] ], "labels": [ "negative", "neutral", "positive" ], "per_class": { "negative": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 182.0 }, "neutral": { "precision": 0.5029838022165388, "recall": 1.0, "f1-score": 0.6693136698808848, "support": 590.0 }, "positive": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 401.0 }, "accuracy": 0.5029838022165388, "macro avg": { "precision": 0.16766126740551293, "recall": 0.3333333333333333, "f1-score": 0.2231045566269616, "support": 1173.0 }, "weighted avg": { "precision": 0.25299270529220624, "recall": 0.5029838022165388, "f1-score": 0.3366539345521927, "support": 1173.0 } } }, "tfidf_linear_svm": { "n": 1173, "correct": 755, "accuracy": 0.6436487638533674, "accuracy_wilson95": [ 0.6158135392935419, 0.6705461886665062 ], "accuracy_cluster95": [ 0.604673709207281, 0.6812998678231339 ], "clusters": 47, "macro_f1": 0.5712651448139064, "balanced_accuracy": 0.5686419234709525, "confusion_matrix": [ [ 55, 75, 52 ], [ 51, 428, 111 ], [ 32, 97, 272 ] ], "labels": [ "negative", "neutral", "positive" ], "per_class": { "negative": { "precision": 0.39855072463768115, "recall": 0.3021978021978022, "f1-score": 0.34375, "support": 182.0 }, "neutral": { "precision": 0.7133333333333334, "recall": 0.7254237288135593, "f1-score": 0.719327731092437, "support": 590.0 }, "positive": { "precision": 0.6252873563218391, "recall": 0.6783042394014963, "f1-score": 0.6507177033492823, "support": 401.0 }, "accuracy": 0.6436487638533674, "macro avg": { "precision": 0.5790571380976178, "recall": 0.5686419234709525, "f1-score": 0.5712651448139064, "support": 1173.0 }, "weighted avg": { "precision": 0.6343931188710845, "recall": 0.6436487638533674, "f1-score": 0.6375990284634271, "support": 1173.0 } }, "train_n": 7671, "training_seconds": 0.9741447999840602, "note": "Supervised comparator trained only on deduplicated official training split. Laya is zero-shot." }, "uniform_random_expected_accuracy": 0.3333333333333333 } }, "comparisons": { "massive_en": { "n": 2948, "delta_accuracy_b_minus_a": -0.049864314789687926, "delta_cluster95": [ -0.06886024423337857, -0.030529172320217096 ], "b_only_correct": 344, "a_only_correct": 491, "mcnemar_exact_p": 4.0830327184207127e-07, "note": "Exploratory unadjusted p-value; clustered CI takes priority for multiple sentences per review." }, "massive_nb": { "n": 2948, "delta_accuracy_b_minus_a": 0.3018995929443691, "delta_cluster95": [ 0.2808683853459973, 0.3226000678426051 ], "b_only_correct": 1127, "a_only_correct": 237, "mcnemar_exact_p": 6.40329856283786e-139, "note": "Exploratory unadjusted p-value; clustered CI takes priority for multiple sentences per review." }, "norec_no": { "n": 1173, "delta_accuracy_b_minus_a": 0.059676044330775786, "delta_cluster95": [ 0.02019622704310428, 0.09372818686665473 ], "b_only_correct": 269, "a_only_correct": 199, "mcnemar_exact_p": 0.001399390793228242, "note": "Exploratory unadjusted p-value; clustered CI takes priority for multiple sentences per review." } }, "language_gaps": { "laya": { "n": 2948, "delta_accuracy_b_minus_a": -0.45488466757123475, "delta_cluster95": [ -0.4752374491180461, -0.43453188602442333 ], "b_only_correct": 97, "a_only_correct": 1438, "mcnemar_exact_p": 9.415013128299395e-307, "note": "Exploratory unadjusted p-value; clustered CI takes priority for multiple sentences per review." }, "laya-multilingual": { "n": 2948, "delta_accuracy_b_minus_a": -0.10312075983717775, "delta_cluster95": [ -0.12042062415196743, -0.08514246947082768 ], "b_only_correct": 239, "a_only_correct": 543, "mcnemar_exact_p": 4.901403575575856e-28, "note": "Exploratory unadjusted p-value; clustered CI takes priority for multiple sentences per review." } }, "finished_utc": "2026-09-26T07:44:42.507382+00:00" }