{ "created": "2026-10-05T00:53:20.007094+00:00", "protocol": { "created": "2026-10-01T14:11:56.138577+00:00", "source_sha256": "7726ab88c63977be98ded59aa3cd9d4e37aa7cc1a870c0fcf04e240b00667f05", "passes": 3, "selection": "Salted ID order independent of predictions. Existing 20 short messages/task for fresh tasks. 20 paired language IDs. 20 public JevBench cases stratified 4 easy, 6 original, 10 hard. 12 each for developer tools and industrial tasks. Full original inputs, no new truncation.", "timing": "Batch one, three serial passes, eight warm-ups per task excluded, GPU synchronized. Median over successful answered calls; failures, abstentions and native truncation retained. Incomplete blocks are never plotted.", "memory": "NVML device-used bytes sampled every 20 ms during each warmed task block, minus the median idle baseline measured before model load. Includes runtime and external native processes; excludes initial model-loading peak. An incremental whole-device estimate, not process-isolated VRAM. Desktop activity can affect it.", "cache": "CLM: candidate/action embeddings precomputed for this fixed workload; every request state re-encoded. Winnow: prefix reuse off. Other adapters retain their native runtime behavior, recorded in metadata.", "hardware": "RTX 5070 Ti 16 GB, Windows, existing pinned Python venvs, one GPU model at a time; PyTorch memory allocation cap 78%. Jev uses sequential persistent HTTPS; hosted memory unknown.", "groups": { "fresh/news": { "accuracy_n": 200, "timing_cases": 20 }, "fresh/emotion": { "accuracy_n": 200, "timing_cases": 20 }, "fresh/spam": { "accuracy_n": 200, "timing_cases": 20 }, "fresh/phishing": { "accuracy_n": 200, "timing_cases": 20 }, "massive_en": { "accuracy_n": 2948, "timing_cases": 20 }, "massive_nb": { "accuracy_n": 2948, "timing_cases": 20 }, "public_jevbench": { "accuracy_n": 231, "timing_cases": 20 }, "kev_claim/devtools-v1": { "accuracy_n": 900, "timing_cases": 12 }, "industry/routing/en": { "accuracy_n": 40, "timing_cases": 12 }, "industry/routing/nb": { "accuracy_n": 40, "timing_cases": 12 }, "industry/documents/en": { "accuracy_n": 28, "timing_cases": 12 }, "industry/documents/nb": { "accuracy_n": 28, "timing_cases": 12 } }, "models": { "jev-1.13.0": [ "fresh/news", "fresh/emotion", "fresh/spam", "fresh/phishing", "massive_en", "massive_nb", "public_jevbench", "kev_claim/devtools-v1", "industry/routing/en", "industry/routing/nb", "industry/documents/en", "industry/documents/nb" ], "clm-int8": [ "fresh/news", "fresh/emotion", "fresh/spam", "fresh/phishing", "massive_en", "massive_nb", "public_jevbench", "kev_claim/devtools-v1", "industry/routing/en", "industry/routing/nb", "industry/documents/en", "industry/documents/nb" ], "cygnet-12b-nf4": [ "fresh/news", "fresh/emotion", "fresh/spam", "fresh/phishing", "massive_en", "massive_nb", "public_jevbench", "kev_claim/devtools-v1", "industry/routing/en", "industry/routing/nb", "industry/documents/en", "industry/documents/nb" ], "decider-08b": [ "fresh/news", "fresh/emotion", "fresh/spam", "fresh/phishing", "massive_en", "massive_nb", "public_jevbench", "kev_claim/devtools-v1", "industry/routing/en", "industry/routing/nb", "industry/documents/en", "industry/documents/nb" ], "decider-2b": [ "fresh/news", "fresh/emotion", "fresh/spam", "fresh/phishing", "massive_en", "massive_nb", "public_jevbench", "kev_claim/devtools-v1", "industry/routing/en", "industry/routing/nb", "industry/documents/en", "industry/documents/nb" ], "decider-4b": [ "fresh/news", "fresh/emotion", "fresh/spam", "fresh/phishing", "massive_en", "massive_nb", "public_jevbench", "kev_claim/devtools-v1", "industry/routing/en", "industry/routing/nb", "industry/documents/en", "industry/documents/nb" ], "decision-4b-v12": [ "fresh/news", "fresh/emotion", "fresh/spam", "fresh/phishing", "massive_en", "massive_nb", "public_jevbench", "kev_claim/devtools-v1", "industry/routing/en", "industry/routing/nb", "industry/documents/en", "industry/documents/nb" ], "gliner-decide-1b": [ "fresh/news", "fresh/emotion", "fresh/spam", "fresh/phishing", "massive_en", "massive_nb", "public_jevbench", "kev_claim/devtools-v1", "industry/routing/en", "industry/routing/nb", "industry/documents/en", "industry/documents/nb" ], "gliner-decide-multi": [ "fresh/news", "fresh/emotion", "fresh/spam", "fresh/phishing", "massive_en", "massive_nb", "public_jevbench", "kev_claim/devtools-v1", "industry/routing/en", "industry/routing/nb", "industry/documents/en", "industry/documents/nb" ], "gliner-decide": [ "fresh/news", "fresh/emotion", "fresh/spam", "fresh/phishing", "massive_en", "massive_nb", "public_jevbench", "kev_claim/devtools-v1", "industry/routing/en", "industry/routing/nb", "industry/documents/en", "industry/documents/nb" ], "imajev-2b": [ "fresh/news", "fresh/emotion", "fresh/spam", "fresh/phishing", "massive_en", "massive_nb", "public_jevbench", "kev_claim/devtools-v1", "industry/routing/en", "industry/routing/nb", "industry/documents/en", "industry/documents/nb" ], "imajev-4b": [ "fresh/news", "fresh/emotion", "fresh/spam", "fresh/phishing", "massive_en", "massive_nb", "public_jevbench", "kev_claim/devtools-v1", "industry/routing/en", "industry/routing/nb", "industry/documents/en", "industry/documents/nb" ], "intern-decision-4b": [ "fresh/news", "fresh/emotion", "fresh/spam", "fresh/phishing", "massive_en", "massive_nb", "public_jevbench", "kev_claim/devtools-v1", "industry/routing/en", "industry/routing/nb", "industry/documents/en", "industry/documents/nb" ], "jev-omni-12b-nf4": [ "fresh/news", "fresh/emotion", "fresh/spam", "fresh/phishing", "massive_en", "massive_nb", "public_jevbench", "kev_claim/devtools-v1", "industry/routing/en", "industry/routing/nb", "industry/documents/en", "industry/documents/nb" ], "jevk5-2b": [ "fresh/news", "fresh/emotion", "fresh/spam", "fresh/phishing", "massive_en", "massive_nb", "public_jevbench", "kev_claim/devtools-v1", "industry/routing/en", "industry/routing/nb", "industry/documents/en", "industry/documents/nb" ], "jevk5": [ "fresh/news", "fresh/emotion", "fresh/spam", "fresh/phishing", "massive_en", "massive_nb", "public_jevbench", "kev_claim/devtools-v1", "industry/routing/en", "industry/routing/nb", "industry/documents/en", "industry/documents/nb" ], "julia": [ "fresh/news", "fresh/emotion", "fresh/spam", "fresh/phishing", "massive_en", "massive_nb", "public_jevbench", "kev_claim/devtools-v1", "industry/routing/en", "industry/routing/nb", "industry/documents/en", "industry/documents/nb" ], "kev-08b": [ "fresh/news", "fresh/emotion", "fresh/spam", "fresh/phishing", "massive_en", "massive_nb", "public_jevbench", "kev_claim/devtools-v1", "industry/routing/en", "industry/routing/nb", "industry/documents/en", "industry/documents/nb" ], "kev-4b": [ "fresh/news", "fresh/emotion", "fresh/spam", "fresh/phishing", "massive_en", "massive_nb", "public_jevbench", "kev_claim/devtools-v1", "industry/routing/en", "industry/routing/nb", "industry/documents/en", "industry/documents/nb" ], "laya-multilingual": [ "fresh/news", "fresh/emotion", "fresh/spam", "fresh/phishing", "massive_en", "massive_nb", "public_jevbench", "kev_claim/devtools-v1", "industry/routing/en", "industry/routing/nb", "industry/documents/en", "industry/documents/nb" ], "laya": [ "fresh/news", "fresh/emotion", "fresh/spam", "fresh/phishing", "massive_en", "massive_nb", "public_jevbench", "kev_claim/devtools-v1", "industry/routing/en", "industry/routing/nb", "industry/documents/en", "industry/documents/nb" ], "nev-2b": [ "fresh/news", "fresh/emotion", "fresh/spam", "fresh/phishing", "massive_en", "massive_nb", "public_jevbench", "kev_claim/devtools-v1", "industry/routing/en", "industry/routing/nb", "industry/documents/en", "industry/documents/nb" ], "nimble-9b": [ "fresh/news", "fresh/emotion", "fresh/spam", "fresh/phishing", "massive_en", "massive_nb", "public_jevbench", "kev_claim/devtools-v1", "industry/routing/en", "industry/routing/nb", "industry/documents/en", "industry/documents/nb" ], "plumb-4b": [ "fresh/news", "fresh/emotion", "fresh/spam", "fresh/phishing", "massive_en", "massive_nb", "public_jevbench", "kev_claim/devtools-v1", "industry/routing/en", "industry/routing/nb", "industry/documents/en", "industry/documents/nb" ], "tev1": [ "fresh/news", "fresh/emotion", "fresh/spam", "fresh/phishing", "massive_en", "massive_nb", "public_jevbench", "kev_claim/devtools-v1", "industry/routing/en", "industry/routing/nb", "industry/documents/en", "industry/documents/nb" ], "von": [ "fresh/news", "fresh/emotion", "fresh/spam", "fresh/phishing", "massive_en", "massive_nb", "public_jevbench", "kev_claim/devtools-v1", "industry/routing/en", "industry/routing/nb", "industry/documents/en", "industry/documents/nb" ], "wald-4b-v12": [ "fresh/news", "fresh/emotion", "fresh/spam", "fresh/phishing", "massive_en", "massive_nb", "public_jevbench", "kev_claim/devtools-v1", "industry/routing/en", "industry/routing/nb", "industry/documents/en", "industry/documents/nb" ], "winnow-12b-q8": [ "fresh/news", "fresh/emotion", "fresh/spam", "fresh/phishing", "massive_en", "massive_nb", "public_jevbench", "kev_claim/devtools-v1", "industry/routing/en", "industry/routing/nb", "industry/documents/en", "industry/documents/nb" ], "decision2-nox-4b": [ "fresh/news", "fresh/emotion", "fresh/spam", "fresh/phishing", "massive_en", "massive_nb", "public_jevbench", "kev_claim/devtools-v1", "industry/routing/en", "industry/routing/nb", "industry/documents/en", "industry/documents/nb" ] }, "fixture_sha256": "a2a7d8b690679591cd8d84deccc56a065f409f6083fe92259f06c447ef9d2098", "completion_updated": "2026-10-05T00:50:24.648048+00:00", "parent_protocol_sha256": "f45e8e64d7a321271097da2f86e682c200f92bf152045e75d51ddcbdb4698faf", "extension": "Identical fixed cases, warmups, passes and measurement policy. Eligibility expanded only when full accuracy groups complete. Historical raw timing files retained unchanged." }, "code_sha256": { "tradeoff_expansion.py": "3f148ae1676fb28bf87b82490b168ee832cad7095b6938893f0f14679f1ef8f7", "tradeoff_expansion_graphs.py": "0284728b86d7eab9fcb43a0becc960519ef2dce6790f7e65e30e1631c3b0075d" }, "accuracy_snapshot_utc": "2026-10-05T00:49:57.233828+00:00", "rows": [ { "model": "jev-1.13.0", "name": "Jev 1.13.0 API", "code": "J", "suite": "fresh/news", "n": 200, "correct": 178, "accuracy": 0.89, "interval": [ 0.839073090026985, 0.926227555398747 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 279.36069993302226, "p95_ms": 328.6575498525053, "added_device_peak_gib": null, "finished": "2026-10-01T14:15:26.462346+00:00", "predictions_sha256": "b4f9f5f54d5261614d97e94de7308707da0391c05e67ebe40907980b2a44e036", "timing_coverage": 1.0, "plottable": true, "config": "API" }, { "model": "jev-1.13.0", "name": "Jev 1.13.0 API", "code": "J", "suite": "fresh/emotion", "n": 200, "correct": 120, "accuracy": 0.6, "interval": [ 0.53083672039262, 0.6653942143319266 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 267.947250045836, "p95_ms": 325.6385649554431, "added_device_peak_gib": null, "finished": "2026-10-01T14:15:46.146850+00:00", "predictions_sha256": "d41af6925bfd312489fefebdbdcf8869e8e6a077c387b4ce165156671153d7b0", "timing_coverage": 1.0, "plottable": true, "config": "API" }, { "model": "jev-1.13.0", "name": "Jev 1.13.0 API", "code": "J", "suite": "fresh/spam", "n": 200, "correct": 196, "accuracy": 0.98, "interval": [ 0.9497129130941735, 0.9921955735836505 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 275.0707500381395, "p95_ms": 355.195070093032, "added_device_peak_gib": null, "finished": "2026-10-01T14:16:06.284822+00:00", "predictions_sha256": "be3d80ae76dd64285c05f074d980236fab4a26084b8803b00bd44b7e9156cfe2", "timing_coverage": 1.0, "plottable": true, "config": "API" }, { "model": "jev-1.13.0", "name": "Jev 1.13.0 API", "code": "J", "suite": "fresh/phishing", "n": 200, "correct": 179, "accuracy": 0.895, "interval": [ 0.8448196800887696, 0.9302925120731899 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 268.4892500983551, "p95_ms": 355.07174502126867, "added_device_peak_gib": null, "finished": "2026-10-01T14:16:26.541852+00:00", "predictions_sha256": "d0299ff5be366310edf846f41890f91fb76c0436547b19e8dbda7260b4eced05", "timing_coverage": 1.0, "plottable": true, "config": "API" }, { "model": "jev-1.13.0", "name": "Jev 1.13.0 API", "code": "J", "suite": "massive_en", "n": 2948, "correct": 2550, "accuracy": 0.8649932157394844, "interval": [ 0.8521812704161608, 0.8768551734515789 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 261.2835499458015, "p95_ms": 308.08460497064516, "added_device_peak_gib": null, "finished": "2026-10-01T14:16:45.619441+00:00", "predictions_sha256": "e31fc2b8826174b3b8f65c9178beb9f97c1dbfac085593c2fc83ea518bf39b79", "timing_coverage": 1.0, "plottable": true, "config": "API" }, { "model": "jev-1.13.0", "name": "Jev 1.13.0 API", "code": "J", "suite": "massive_nb", "n": 2948, "correct": 2412, "accuracy": 0.8181818181818182, "interval": [ 0.8038478031942561, 0.8316876841179558 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 271.0956499213353, "p95_ms": 348.64468002924696, "added_device_peak_gib": null, "finished": "2026-10-01T14:17:05.530704+00:00", "predictions_sha256": "0978b346946efa8389555831f67267da1eca20ed6dc428a51ecfbd9b593433d8", "timing_coverage": 1.0, "plottable": true, "config": "API" }, { "model": "jev-1.13.0", "name": "Jev 1.13.0 API", "code": "J", "suite": "public_jevbench", "n": 231, "correct": 199, "accuracy": 0.8614718614718615, "interval": [ 0.8109826332087084, 0.9001354157023556 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 275.7654001470655, "p95_ms": 328.4529149415903, "added_device_peak_gib": null, "finished": "2026-10-01T14:17:25.469486+00:00", "predictions_sha256": "76630190a3735d309427ed47928c6cb577297baa2c13c85fd7884510d7005529", "timing_coverage": 1.0, "plottable": true, "config": "API" }, { "model": "jev-1.13.0", "name": "Jev 1.13.0 API", "code": "J", "suite": "kev_claim/devtools-v1", "n": 900, "correct": 629, "accuracy": 0.6988888888888889, "interval": [ 0.6619267659670504, 0.7405337862870147 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 251.45585008431226, "p95_ms": 296.0955749149434, "added_device_peak_gib": null, "finished": "2026-10-01T14:17:37.512941+00:00", "predictions_sha256": "e909acc4ba0abf4f2ade7363260ec7456d80bf4506c14d1dead1eea616ff8da7", "timing_coverage": 1.0, "plottable": true, "config": "API" }, { "model": "jev-1.13.0", "name": "Jev 1.13.0 API", "code": "J", "suite": "industry/routing/en", "n": 40, "correct": 40, "accuracy": 1.0, "interval": [ 0.9123783988027135, 1.0 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 279.1460499865934, "p95_ms": 326.1136499349959, "added_device_peak_gib": null, "finished": "2026-10-01T14:17:50.495819+00:00", "predictions_sha256": "104d0caf34ad727188ffc1812d1cf9ca3a807a0271d37ca145047498c31da1a2", "timing_coverage": 1.0, "plottable": true, "config": "API" }, { "model": "jev-1.13.0", "name": "Jev 1.13.0 API", "code": "J", "suite": "industry/routing/nb", "n": 40, "correct": 39, "accuracy": 0.975, "interval": [ 0.8711863103652591, 0.9955731684973186 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 269.0249498700723, "p95_ms": 307.73219995899126, "added_device_peak_gib": null, "finished": "2026-10-01T14:18:02.957770+00:00", "predictions_sha256": "1e4f61652b02b4baab28f17603a4468874edfdaa52ea904cf381a038c64ecc27", "timing_coverage": 1.0, "plottable": true, "config": "API" }, { "model": "jev-1.13.0", "name": "Jev 1.13.0 API", "code": "J", "suite": "industry/documents/en", "n": 28, "correct": 28, "accuracy": 1.0, "interval": [ 0.8793566952341543, 0.9999999999999998 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 270.40994993876666, "p95_ms": 316.96095009101555, "added_device_peak_gib": null, "finished": "2026-10-01T14:18:15.679776+00:00", "predictions_sha256": "e80fec85ff6a8b75f36a1ea94f5def44596b4bf756da297024b13d6ebf194223", "timing_coverage": 1.0, "plottable": true, "config": "API" }, { "model": "jev-1.13.0", "name": "Jev 1.13.0 API", "code": "J", "suite": "industry/documents/nb", "n": 28, "correct": 28, "accuracy": 1.0, "interval": [ 0.8793566952341543, 0.9999999999999998 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 264.81319998856634, "p95_ms": 313.1429749773815, "added_device_peak_gib": null, "finished": "2026-10-01T14:18:28.354197+00:00", "predictions_sha256": "0f13231cd43d4c1dfc0b80f7cb199669a8ca29fb9657a55ae7e9bb295e226819", "timing_coverage": 1.0, "plottable": true, "config": "API" }, { "model": "clm-int8", "name": "CLM 8B INT8", "code": "CL", "suite": "fresh/news", "n": 200, "correct": 93, "accuracy": 0.465, "interval": [ 0.3971856415700402, 0.5341335312763685 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 232.38425003364682, "p95_ms": 259.61743991356343, "device_peak_gib": 10.519062042236328, "device_min_gib": 10.377765655517578, "samples": 452, "added_device_peak_gib": 8.65750503540039, "torch_peak_allocated_gib": 7.8818359375, "torch_peak_reserved_gib": 8.337890625, "warmup_errors": [], "finished": "2026-10-01T14:30:58.390406+00:00", "predictions_sha256": "8d46b57e057fa3ec64b501f6b60894b9df8d093498ec2f218b68f5941a6d4f68", "timing_coverage": 1.0, "plottable": true, "config": "int8 encoder, native projection heads" }, { "model": "clm-int8", "name": "CLM 8B INT8", "code": "CL", "suite": "fresh/emotion", "n": 200, "correct": 51, "accuracy": 0.255, "interval": [ 0.1996049517203135, 0.3196292582045472 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 229.3765499489382, "p95_ms": 243.63984519150108, "device_peak_gib": 10.214374542236328, "device_min_gib": 10.143085479736328, "samples": 440, "added_device_peak_gib": 8.35281753540039, "torch_peak_allocated_gib": 7.81480073928833, "torch_peak_reserved_gib": 8.033203125, "warmup_errors": [], "finished": "2026-10-01T14:31:13.990410+00:00", "predictions_sha256": "50ba45baac51bd5cc18c50124f9489b12d2237c893498774159c12035b4421e7", "timing_coverage": 1.0, "plottable": true, "config": "int8 encoder, native projection heads" }, { "model": "clm-int8", "name": "CLM 8B INT8", "code": "CL", "suite": "fresh/spam", "n": 200, "correct": 132, "accuracy": 0.66, "interval": [ 0.5918849552683626, 0.7220845402909121 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 227.4988000281155, "p95_ms": 239.57579990383235, "device_peak_gib": 10.702655792236328, "device_min_gib": 10.506366729736328, "samples": 442, "added_device_peak_gib": 8.84109878540039, "torch_peak_allocated_gib": 7.980143070220947, "torch_peak_reserved_gib": 8.51953125, "warmup_errors": [], "finished": "2026-10-01T14:31:29.624458+00:00", "predictions_sha256": "39b78fe8c93fc957c4c0feb5d027c77405459ca104ed5510195b4815c96a9168", "timing_coverage": 1.0, "plottable": true, "config": "int8 encoder, native projection heads" }, { "model": "clm-int8", "name": "CLM 8B INT8", "code": "CL", "suite": "fresh/phishing", "n": 200, "correct": 80, "accuracy": 0.4, "interval": [ 0.33460578566807336, 0.46916327960737997 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 229.57135003525764, "p95_ms": 244.78858982911333, "device_peak_gib": 10.565937042236328, "device_min_gib": 10.558612823486328, "samples": 444, "added_device_peak_gib": 8.70438003540039, "torch_peak_allocated_gib": 7.945065975189209, "torch_peak_reserved_gib": 8.3828125, "warmup_errors": [], "finished": "2026-10-01T14:31:45.306312+00:00", "predictions_sha256": "5a149487f24e2aea011e7f9e6fbd5adbaeab3be0f3714bbd787bb69d3fb34c03", "timing_coverage": 1.0, "plottable": true, "config": "int8 encoder, native projection heads" }, { "model": "clm-int8", "name": "CLM 8B INT8", "code": "CL", "suite": "massive_en", "n": 2948, "correct": 1229, "accuracy": 0.4168928086838535, "interval": [ 0.3992142368266082, 0.4347876881291574 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 225.09385005105287, "p95_ms": 248.23972990270704, "device_peak_gib": 10.118671417236328, "device_min_gib": 10.013690948486328, "samples": 436, "added_device_peak_gib": 8.25711441040039, "torch_peak_allocated_gib": 7.794018745422363, "torch_peak_reserved_gib": 7.935546875, "warmup_errors": [], "finished": "2026-10-01T14:32:00.748782+00:00", "predictions_sha256": "b93a0768abbe6e382e8e2a4900323ab4253315986b6004626159c29f56c970d3", "timing_coverage": 1.0, "plottable": true, "config": "int8 encoder, native projection heads" }, { "model": "clm-int8", "name": "CLM 8B INT8", "code": "CL", "suite": "massive_nb", "n": 2948, "correct": 894, "accuracy": 0.30325644504748983, "interval": [ 0.2869282706203023, 0.32009669458110196 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 227.06860001198947, "p95_ms": 242.4827449140139, "device_peak_gib": 10.333515167236328, "device_min_gib": 10.326190948486328, "samples": 437, "added_device_peak_gib": 8.47195816040039, "torch_peak_allocated_gib": 7.881561279296875, "torch_peak_reserved_gib": 8.150390625, "warmup_errors": [], "finished": "2026-10-01T14:32:16.196048+00:00", "predictions_sha256": "76e8d1eb37bdd6b08c845820d700c9db7112f40b97edbac953a1861435637dff", "timing_coverage": 1.0, "plottable": true, "config": "int8 encoder, native projection heads" }, { "model": "clm-int8", "name": "CLM 8B INT8", "code": "CL", "suite": "public_jevbench", "n": 231, "correct": 96, "accuracy": 0.4155844155844156, "interval": [ 0.3539195652555485, 0.4800109502682149 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 15, "median_ms": 235.34204997122288, "p95_ms": 829.005985124968, "device_peak_gib": 11.687030792236328, "device_min_gib": 10.355976104736328, "samples": 729, "added_device_peak_gib": 9.82547378540039, "torch_peak_allocated_gib": 9.362330436706543, "torch_peak_reserved_gib": 9.50390625, "warmup_errors": [], "finished": "2026-10-01T14:32:40.839363+00:00", "predictions_sha256": "4a2c09aadf17157df085ad6f03a4982dec942ae9b821a496cec65f57617e39b5", "timing_coverage": 1.0, "plottable": true, "config": "int8 encoder, native projection heads" }, { "model": "clm-int8", "name": "CLM 8B INT8", "code": "CL", "suite": "kev_claim/devtools-v1", "n": 900, "correct": 391, "accuracy": 0.43444444444444447, "interval": [ 0.36731533709374165, 0.49904832321098497 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 229.24190002959222, "p95_ms": 322.80099991476163, "device_peak_gib": 11.415454864501953, "device_min_gib": 11.408130645751953, "samples": 275, "added_device_peak_gib": 9.553897857666016, "torch_peak_allocated_gib": 8.196001052856445, "torch_peak_reserved_gib": 9.232421875, "warmup_errors": [], "finished": "2026-10-01T14:32:51.314157+00:00", "predictions_sha256": "c0a2dbdd535c57422b118fe4e7335747f2a521493e5387a96d287c052b45fadb", "timing_coverage": 1.0, "plottable": true, "config": "int8 encoder, native projection heads" }, { "model": "clm-int8", "name": "CLM 8B INT8", "code": "CL", "suite": "industry/routing/en", "n": 40, "correct": 8, "accuracy": 0.2, "interval": [ 0.10499989725437704, 0.347573063463995 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 227.42290003225207, "p95_ms": 244.8159250197932, "device_peak_gib": 10.022876739501953, "device_min_gib": 10.015552520751953, "samples": 265, "added_device_peak_gib": 8.161319732666016, "torch_peak_allocated_gib": 7.77547550201416, "torch_peak_reserved_gib": 7.83984375, "warmup_errors": [], "finished": "2026-10-01T14:33:01.386129+00:00", "predictions_sha256": "9500175899047b8c12c2a040610af80fc1d98d1c6f2ffe98f28b3cb4fd288e13", "timing_coverage": 1.0, "plottable": true, "config": "int8 encoder, native projection heads" }, { "model": "clm-int8", "name": "CLM 8B INT8", "code": "CL", "suite": "industry/routing/nb", "n": 40, "correct": 13, "accuracy": 0.325, "interval": [ 0.200845022317827, 0.47982253810122333 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 228.68205001577735, "p95_ms": 244.14609995437786, "device_peak_gib": 10.784595489501953, "device_min_gib": 10.775318145751953, "samples": 267, "added_device_peak_gib": 8.923038482666016, "torch_peak_allocated_gib": 7.914557456970215, "torch_peak_reserved_gib": 8.603515625, "warmup_errors": [], "finished": "2026-10-01T14:33:11.579379+00:00", "predictions_sha256": "a084f7db00a685d7c7adc7e35ae37ae9620c420ef30e6bff6650bfa3f66101bc", "timing_coverage": 1.0, "plottable": true, "config": "int8 encoder, native projection heads" }, { "model": "clm-int8", "name": "CLM 8B INT8", "code": "CL", "suite": "industry/documents/en", "n": 28, "correct": 15, "accuracy": 0.5357142857142857, "interval": [ 0.358126892344679, 0.7046843001720462 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 228.1686500646174, "p95_ms": 243.79580008098856, "device_peak_gib": 10.212329864501953, "device_min_gib": 10.205005645751953, "samples": 265, "added_device_peak_gib": 8.350772857666016, "torch_peak_allocated_gib": 7.823240756988525, "torch_peak_reserved_gib": 8.029296875, "warmup_errors": [], "finished": "2026-10-01T14:33:21.668143+00:00", "predictions_sha256": "b93cd801a5612c0e514b6102d5bbf75c74e9c4cf2e610ecdf6bbccbacd4b909d", "timing_coverage": 1.0, "plottable": true, "config": "int8 encoder, native projection heads" }, { "model": "clm-int8", "name": "CLM 8B INT8", "code": "CL", "suite": "industry/documents/nb", "n": 28, "correct": 8, "accuracy": 0.2857142857142857, "interval": [ 0.1525399605666909, 0.47059288433295715 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 233.60229993704706, "p95_ms": 244.50597510440275, "device_peak_gib": 10.536548614501953, "device_min_gib": 10.529224395751953, "samples": 270, "added_device_peak_gib": 8.674991607666016, "torch_peak_allocated_gib": 7.941054344177246, "torch_peak_reserved_gib": 8.353515625, "warmup_errors": [], "finished": "2026-10-01T14:33:31.902372+00:00", "predictions_sha256": "34b1f89d1d071ac542e31c06086a4a19585ce6cbbf85d476f1519c8386031293", "timing_coverage": 1.0, "plottable": true, "config": "int8 encoder, native projection heads" }, { "model": "cygnet-12b-nf4", "name": "Cygnet 12B NF4", "code": "CY", "suite": "fresh/news", "n": 200, "correct": 169, "accuracy": 0.845, "interval": [ 0.7883930940765792, 0.8886036307231068 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 139.7043999750167, "p95_ms": 149.7789251501672, "device_peak_gib": 9.535480499267578, "device_min_gib": 9.528156280517578, "samples": 266, "added_device_peak_gib": 7.671482086181641, "torch_peak_allocated_gib": 7.315627574920654, "torch_peak_reserved_gib": 7.3984375, "warmup_errors": [], "finished": "2026-10-01T14:27:15.312800+00:00", "predictions_sha256": "2c1bcfce33094abd321ea5a505be1a23b2dca0404ec2dfa874f0876cd3c03bd5", "timing_coverage": 1.0, "plottable": true, "config": "NF4 backbone, BF16 compute" }, { "model": "cygnet-12b-nf4", "name": "Cygnet 12B NF4", "code": "CY", "suite": "fresh/emotion", "n": 200, "correct": 49, "accuracy": 0.245, "interval": [ 0.1905686808903718, 0.30904243556203415 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 139.72454995382577, "p95_ms": 149.96339505305514, "device_peak_gib": 9.533527374267578, "device_min_gib": 9.526203155517578, "samples": 273, "added_device_peak_gib": 7.669528961181641, "torch_peak_allocated_gib": 7.312671661376953, "torch_peak_reserved_gib": 7.396484375, "warmup_errors": [], "finished": "2026-10-01T14:27:24.937529+00:00", "predictions_sha256": "ba8ecbea6093327c0172fd3e81635f3cefe5c4d2448850c2ac5bf3344df5d36d", "timing_coverage": 1.0, "plottable": true, "config": "NF4 backbone, BF16 compute" }, { "model": "cygnet-12b-nf4", "name": "Cygnet 12B NF4", "code": "CY", "suite": "fresh/spam", "n": 200, "correct": 187, "accuracy": 0.935, "interval": [ 0.8919809207009312, 0.961623645350847 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 141.19664998725057, "p95_ms": 202.3355649318546, "device_peak_gib": 9.535480499267578, "device_min_gib": 9.528156280517578, "samples": 296, "added_device_peak_gib": 7.671482086181641, "torch_peak_allocated_gib": 7.3296709060668945, "torch_peak_reserved_gib": 7.3984375, "warmup_errors": [], "finished": "2026-10-01T14:27:35.318263+00:00", "predictions_sha256": "1ad5b61722d312282dd8096b8148df0718e6b0f82194879eaa5d515ae3e5d57b", "timing_coverage": 1.0, "plottable": true, "config": "NF4 backbone, BF16 compute" }, { "model": "cygnet-12b-nf4", "name": "Cygnet 12B NF4", "code": "CY", "suite": "fresh/phishing", "n": 200, "correct": 178, "accuracy": 0.89, "interval": [ 0.839073090026985, 0.926227555398747 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 141.28520002122968, "p95_ms": 193.40345490491018, "device_peak_gib": 9.535480499267578, "device_min_gib": 9.528156280517578, "samples": 285, "added_device_peak_gib": 7.671482086181641, "torch_peak_allocated_gib": 7.323605060577393, "torch_peak_reserved_gib": 7.3984375, "warmup_errors": [], "finished": "2026-10-01T14:27:45.386939+00:00", "predictions_sha256": "ce0e5c31437ef2c9c70ce4385f00f6f14e6bff7ca7ba5d370d6ca85e8ba787cb", "timing_coverage": 1.0, "plottable": true, "config": "NF4 backbone, BF16 compute" }, { "model": "cygnet-12b-nf4", "name": "Cygnet 12B NF4", "code": "CY", "suite": "massive_en", "n": 2948, "correct": 2272, "accuracy": 0.7706919945725916, "interval": [ 0.7551703102658747, 0.7855091341639175 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 139.4162999931723, "p95_ms": 150.86846500635147, "device_peak_gib": 9.811805725097656, "device_min_gib": 9.804481506347656, "samples": 275, "added_device_peak_gib": 7.673923492431641, "torch_peak_allocated_gib": 7.3168253898620605, "torch_peak_reserved_gib": 7.3984375, "warmup_errors": [], "finished": "2026-10-04T19:34:32.648652+00:00", "predictions_sha256": "5573d3d1501868f2faa36cb3af4a0b3d4b134b97fc228012eb341d32a257f791", "timing_coverage": 1.0, "plottable": true, "config": "NF4 backbone, BF16 compute" }, { "model": "cygnet-12b-nf4", "name": "Cygnet 12B NF4", "code": "CY", "suite": "massive_nb", "n": 2948, "correct": 2013, "accuracy": 0.6828358208955224, "interval": [ 0.6658080870797453, 0.6993876780175706 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 138.65335006266832, "p95_ms": 150.03967508673668, "device_peak_gib": 9.811805725097656, "device_min_gib": 9.804481506347656, "samples": 272, "added_device_peak_gib": 7.673923492431641, "torch_peak_allocated_gib": 7.3171067237854, "torch_peak_reserved_gib": 7.3984375, "warmup_errors": [], "finished": "2026-10-04T19:34:42.265898+00:00", "predictions_sha256": "66d86963af6dc03d1f5f1e2db508b40891b7417ed913bb40e01a3584fc847aa0", "timing_coverage": 1.0, "plottable": true, "config": "NF4 backbone, BF16 compute" }, { "model": "cygnet-12b-nf4", "name": "Cygnet 12B NF4", "code": "CY", "suite": "public_jevbench", "n": 231, "correct": 195, "accuracy": 0.8441558441558441, "interval": [ 0.7917966720914348, 0.8852558415424524 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 171.30565003026277, "p95_ms": 1307.6673449249936, "device_peak_gib": 9.969074249267578, "device_min_gib": 9.528644561767578, "samples": 819, "added_device_peak_gib": 8.10507583618164, "torch_peak_allocated_gib": 7.637777805328369, "torch_peak_reserved_gib": 7.83203125, "warmup_errors": [], "finished": "2026-10-01T14:28:12.128459+00:00", "predictions_sha256": "e2c7034e4a24077b0a66c0feef0f7ad59c29c0c89dc9d9658712ada7b34bcc62", "timing_coverage": 1.0, "plottable": true, "config": "NF4 backbone, BF16 compute" }, { "model": "cygnet-12b-nf4", "name": "Cygnet 12B NF4", "code": "CY", "suite": "kev_claim/devtools-v1", "n": 900, "correct": 602, "accuracy": 0.6688888888888889, "interval": [ 0.6275310160092734, 0.7188314094380855 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 199.68530000187457, "p95_ms": 557.9657000489533, "device_peak_gib": 9.925086975097656, "device_min_gib": 9.917762756347656, "samples": 281, "added_device_peak_gib": 7.787204742431641, "torch_peak_allocated_gib": 7.405233383178711, "torch_peak_reserved_gib": 7.51171875, "warmup_errors": [], "finished": "2026-10-04T19:34:53.092255+00:00", "predictions_sha256": "c7cf9a5e7aff0d40b85ca6198a6de219aa7cb307b3dc2b5deec7a51a8a947935", "timing_coverage": 1.0, "plottable": true, "config": "NF4 backbone, BF16 compute" }, { "model": "cygnet-12b-nf4", "name": "Cygnet 12B NF4", "code": "CY", "suite": "industry/routing/en", "n": 40, "correct": 40, "accuracy": 1.0, "interval": [ 0.9123783988027135, 1.0 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 137.55380001384765, "p95_ms": 149.1317749605514, "device_peak_gib": 9.809852600097656, "device_min_gib": 9.802528381347656, "samples": 163, "added_device_peak_gib": 7.671970367431641, "torch_peak_allocated_gib": 7.314642429351807, "torch_peak_reserved_gib": 7.396484375, "warmup_errors": [], "finished": "2026-10-04T19:34:59.270378+00:00", "predictions_sha256": "3f3972eb8caab62eb8931a211bb085bacdc746a6f3264328d279a6509eb4e8a9", "timing_coverage": 1.0, "plottable": true, "config": "NF4 backbone, BF16 compute" }, { "model": "cygnet-12b-nf4", "name": "Cygnet 12B NF4", "code": "CY", "suite": "industry/routing/nb", "n": 40, "correct": 38, "accuracy": 0.95, "interval": [ 0.8349612263085903, 0.9861793326138516 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 136.93754991982132, "p95_ms": 149.15612497134134, "device_peak_gib": 9.809852600097656, "device_min_gib": 9.802528381347656, "samples": 162, "added_device_peak_gib": 7.671970367431641, "torch_peak_allocated_gib": 7.3153462409973145, "torch_peak_reserved_gib": 7.396484375, "warmup_errors": [], "finished": "2026-10-04T19:35:05.427463+00:00", "predictions_sha256": "73e31601abd7ebeb9a60150e75dfbc6b3cb7626ba56a093cafe57a6585be17ce", "timing_coverage": 1.0, "plottable": true, "config": "NF4 backbone, BF16 compute" }, { "model": "cygnet-12b-nf4", "name": "Cygnet 12B NF4", "code": "CY", "suite": "industry/documents/en", "n": 28, "correct": 27, "accuracy": 0.9642857142857143, "interval": [ 0.8228780225664668, 0.9936674801509623 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 137.0943000074476, "p95_ms": 147.95902487821877, "device_peak_gib": 9.809852600097656, "device_min_gib": 9.802528381347656, "samples": 163, "added_device_peak_gib": 7.671970367431641, "torch_peak_allocated_gib": 7.315486431121826, "torch_peak_reserved_gib": 7.396484375, "warmup_errors": [], "finished": "2026-10-04T19:35:11.589777+00:00", "predictions_sha256": "235c1c1ef94914c8803294d08e92c8c755bc7f8584f014e99814fde14cae9fa7", "timing_coverage": 1.0, "plottable": true, "config": "NF4 backbone, BF16 compute" }, { "model": "cygnet-12b-nf4", "name": "Cygnet 12B NF4", "code": "CY", "suite": "industry/documents/nb", "n": 28, "correct": 28, "accuracy": 1.0, "interval": [ 0.8793566952341543, 0.9999999999999998 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 137.17745011672378, "p95_ms": 149.53407493885607, "device_peak_gib": 9.809852600097656, "device_min_gib": 9.802528381347656, "samples": 162, "added_device_peak_gib": 7.671970367431641, "torch_peak_allocated_gib": 7.31555700302124, "torch_peak_reserved_gib": 7.396484375, "warmup_errors": [], "finished": "2026-10-04T19:35:17.764137+00:00", "predictions_sha256": "1ae2c8a72a04f0e34d3045347f85d9e025811377e85478590ab059a9889d32ed", "timing_coverage": 1.0, "plottable": true, "config": "NF4 backbone, BF16 compute" }, { "model": "decider-08b", "name": "Decider 0.8B", "code": "D.8", "suite": "fresh/news", "n": 200, "correct": 174, "accuracy": 0.87, "interval": [ 0.8163364812775159, 0.9097179772033067 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 48.363949987106025, "p95_ms": 58.67171514546499, "device_peak_gib": 3.701587677001953, "device_min_gib": 3.701587677001953, "samples": 97, "added_device_peak_gib": 1.8272438049316406, "torch_peak_allocated_gib": 1.4898395538330078, "torch_peak_reserved_gib": 1.556640625, "warmup_errors": [], "finished": "2026-10-01T14:33:46.739534+00:00", "predictions_sha256": "621f8395061bdd7af28d06305998167a21bd91a80c8e81f02d74dc9fb3a9297a", "timing_coverage": 1.0, "plottable": true, "config": "bf16" }, { "model": "decider-08b", "name": "Decider 0.8B", "code": "D.8", "suite": "fresh/emotion", "n": 200, "correct": 150, "accuracy": 0.75, "interval": [ 0.6856590168795417, 0.8049183199318249 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 48.10919996816665, "p95_ms": 63.58950495487078, "device_peak_gib": 3.700611114501953, "device_min_gib": 3.693286895751953, "samples": 99, "added_device_peak_gib": 1.8262672424316406, "torch_peak_allocated_gib": 1.4848999977111816, "torch_peak_reserved_gib": 1.55078125, "warmup_errors": [], "finished": "2026-10-01T14:33:50.173571+00:00", "predictions_sha256": "50e365c88492e71215792c347fc81dcd5bdcab242e438fea77ba951585a288a1", "timing_coverage": 1.0, "plottable": true, "config": "bf16" }, { "model": "decider-08b", "name": "Decider 0.8B", "code": "D.8", "suite": "fresh/spam", "n": 200, "correct": 196, "accuracy": 0.98, "interval": [ 0.9497129130941735, 0.9921955735836505 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 47.72579995915294, "p95_ms": 59.92426992161199, "device_peak_gib": 3.706470489501953, "device_min_gib": 3.699146270751953, "samples": 97, "added_device_peak_gib": 1.8321266174316406, "torch_peak_allocated_gib": 1.506577491760254, "torch_peak_reserved_gib": 1.556640625, "warmup_errors": [], "finished": "2026-10-01T14:33:53.633712+00:00", "predictions_sha256": "23f4b6c7fe89372d1752247dcdf2d04d3ea7d87891a0aa44b7c180bc5422b10f", "timing_coverage": 1.0, "plottable": true, "config": "bf16" }, { "model": "decider-08b", "name": "Decider 0.8B", "code": "D.8", "suite": "fresh/phishing", "n": 200, "correct": 184, "accuracy": 0.92, "interval": [ 0.874010511732856, 0.9501594141102401 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 49.109900020994246, "p95_ms": 59.44396002450958, "device_peak_gib": 3.703540802001953, "device_min_gib": 3.703540802001953, "samples": 98, "added_device_peak_gib": 1.8291969299316406, "torch_peak_allocated_gib": 1.4942984580993652, "torch_peak_reserved_gib": 1.55859375, "warmup_errors": [], "finished": "2026-10-01T14:33:57.049421+00:00", "predictions_sha256": "40ab465664f81a67e206ca0232a52e1cda87c463d50703cc74d5ad8d95f2b40b", "timing_coverage": 1.0, "plottable": true, "config": "bf16" }, { "model": "decider-08b", "name": "Decider 0.8B", "code": "D.8", "suite": "massive_en", "n": 2948, "correct": 2423, "accuracy": 0.8219131614654003, "interval": [ 0.8076862420608573, 0.835302220049259 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 47.588299959897995, "p95_ms": 55.638169951271266, "device_peak_gib": 3.708423614501953, "device_min_gib": 3.701099395751953, "samples": 95, "added_device_peak_gib": 1.8340797424316406, "torch_peak_allocated_gib": 1.4942984580993652, "torch_peak_reserved_gib": 1.55859375, "warmup_errors": [], "finished": "2026-10-01T14:34:00.350490+00:00", "predictions_sha256": "516e866dbce6c26fb88b50248ca5651eb2717717e539e316b08a413ca24aad8e", "timing_coverage": 1.0, "plottable": true, "config": "bf16" }, { "model": "decider-08b", "name": "Decider 0.8B", "code": "D.8", "suite": "massive_nb", "n": 2948, "correct": 2021, "accuracy": 0.6855495251017639, "interval": [ 0.6685569969442564, 0.7020591134606262 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 47.06115007866174, "p95_ms": 56.57372489804402, "device_peak_gib": 3.703540802001953, "device_min_gib": 3.703540802001953, "samples": 94, "added_device_peak_gib": 1.8291969299316406, "torch_peak_allocated_gib": 1.4942984580993652, "torch_peak_reserved_gib": 1.55859375, "warmup_errors": [], "finished": "2026-10-01T14:34:03.626859+00:00", "predictions_sha256": "ed7faccd2c94ce98b6d7567b6aaa53aa7f42bfd6d69e4ff926a09fc76aa3c24f", "timing_coverage": 1.0, "plottable": true, "config": "bf16" }, { "model": "decider-08b", "name": "Decider 0.8B", "code": "D.8", "suite": "public_jevbench", "n": 231, "correct": 152, "accuracy": 0.658008658008658, "interval": [ 0.5946975685296276, 0.71615044087461 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 49.27255003713071, "p95_ms": 115.76565506402397, "device_peak_gib": 4.517017364501953, "device_min_gib": 3.703540802001953, "samples": 122, "added_device_peak_gib": 2.6426734924316406, "torch_peak_allocated_gib": 2.2682533264160156, "torch_peak_reserved_gib": 2.3671875, "warmup_errors": [], "finished": "2026-10-01T14:34:07.832033+00:00", "predictions_sha256": "e50e67ed9c5a767fac9a68b433933583fa3926fbc878198466cf85f9e6575ffe", "timing_coverage": 1.0, "plottable": true, "config": "bf16" }, { "model": "decider-08b", "name": "Decider 0.8B", "code": "D.8", "suite": "kev_claim/devtools-v1", "n": 900, "correct": 431, "accuracy": 0.47888888888888886, "interval": [ 0.45102548056820624, 0.5059847660500544 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 46.680649975314736, "p95_ms": 54.95462496764958, "device_peak_gib": 3.748462677001953, "device_min_gib": 3.740650177001953, "samples": 57, "added_device_peak_gib": 1.8741188049316406, "torch_peak_allocated_gib": 1.5731463432312012, "torch_peak_reserved_gib": 1.603515625, "warmup_errors": [], "finished": "2026-10-01T14:34:09.970077+00:00", "predictions_sha256": "71c47d816110c984ed33b43e4cafc5bbe215ff30d7a4a8f866521b14329a4f24", "timing_coverage": 1.0, "plottable": true, "config": "bf16" }, { "model": "decider-08b", "name": "Decider 0.8B", "code": "D.8", "suite": "industry/routing/en", "n": 40, "correct": 35, "accuracy": 0.875, "interval": [ 0.738878801611489, 0.945404997490546 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 47.812100034207106, "p95_ms": 57.4757750146091, "device_peak_gib": 3.700611114501953, "device_min_gib": 3.693286895751953, "samples": 58, "added_device_peak_gib": 1.8262672424316406, "torch_peak_allocated_gib": 1.4848999977111816, "torch_peak_reserved_gib": 1.55078125, "warmup_errors": [], "finished": "2026-10-01T14:34:12.203332+00:00", "predictions_sha256": "de8a8e6137ff94e0f3fc6914fbfc0ff5840fbb2418b30617c4ab24c7938338ef", "timing_coverage": 1.0, "plottable": true, "config": "bf16" }, { "model": "decider-08b", "name": "Decider 0.8B", "code": "D.8", "suite": "industry/routing/nb", "n": 40, "correct": 30, "accuracy": 0.75, "interval": [ 0.5980603857923197, 0.858128813609037 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 47.282450017519295, "p95_ms": 56.92514998372644, "device_peak_gib": 3.701587677001953, "device_min_gib": 3.701587677001953, "samples": 58, "added_device_peak_gib": 1.8272438049316406, "torch_peak_allocated_gib": 1.4898395538330078, "torch_peak_reserved_gib": 1.556640625, "warmup_errors": [], "finished": "2026-10-01T14:34:14.380789+00:00", "predictions_sha256": "be20aba8a652fde2058649ac8104e430bb943b630a7daa7bf8f3ab958d775d6c", "timing_coverage": 1.0, "plottable": true, "config": "bf16" }, { "model": "decider-08b", "name": "Decider 0.8B", "code": "D.8", "suite": "industry/documents/en", "n": 28, "correct": 26, "accuracy": 0.9285714285714286, "interval": [ 0.7735463743160095, 0.9801879358846942 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 47.19454993028194, "p95_ms": 51.81052495026961, "device_peak_gib": 3.701587677001953, "device_min_gib": 3.701587677001953, "samples": 57, "added_device_peak_gib": 1.8272438049316406, "torch_peak_allocated_gib": 1.4898395538330078, "torch_peak_reserved_gib": 1.556640625, "warmup_errors": [], "finished": "2026-10-01T14:34:16.496638+00:00", "predictions_sha256": "2e84f149e4cc40aa22cbbf1db52849c6682d5fa629ebe22e3b1e12cdbb57a8eb", "timing_coverage": 1.0, "plottable": true, "config": "bf16" }, { "model": "decider-08b", "name": "Decider 0.8B", "code": "D.8", "suite": "industry/documents/nb", "n": 28, "correct": 27, "accuracy": 0.9642857142857143, "interval": [ 0.8228780225664668, 0.9936674801509623 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 46.45344987511635, "p95_ms": 56.64317507762462, "device_peak_gib": 3.701587677001953, "device_min_gib": 3.699146270751953, "samples": 57, "added_device_peak_gib": 1.8272438049316406, "torch_peak_allocated_gib": 1.4898395538330078, "torch_peak_reserved_gib": 1.556640625, "warmup_errors": [], "finished": "2026-10-01T14:34:18.627096+00:00", "predictions_sha256": "2f33a899250270e9b2da90f1812a8c9c85f687216c76b3d48d66a400db56a80c", "timing_coverage": 1.0, "plottable": true, "config": "bf16" }, { "model": "decider-2b", "name": "Decider 2B", "code": "D2", "suite": "fresh/news", "n": 200, "correct": 176, "accuracy": 0.88, "interval": [ 0.8276574790170326, 0.9180200729362449 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 47.682750155217946, "p95_ms": 59.2713599675335, "device_peak_gib": 5.778736114501953, "device_min_gib": 5.771411895751953, "samples": 97, "added_device_peak_gib": 3.9043922424316406, "torch_peak_allocated_gib": 3.5713539123535156, "torch_peak_reserved_gib": 3.62890625, "warmup_errors": [], "finished": "2026-10-01T14:34:33.795718+00:00", "predictions_sha256": "658995a136c8eef47e509dc50d6e4179fa45b09bf113282747030e2528c4bc5a", "timing_coverage": 1.0, "plottable": true, "config": "bf16" }, { "model": "decider-2b", "name": "Decider 2B", "code": "D2", "suite": "fresh/emotion", "n": 200, "correct": 166, "accuracy": 0.83, "interval": [ 0.7718411637168737, 0.8757209208741302 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 47.19085001852363, "p95_ms": 54.51614005723968, "device_peak_gib": 5.767993927001953, "device_min_gib": 5.765552520751953, "samples": 94, "added_device_peak_gib": 3.8936500549316406, "torch_peak_allocated_gib": 3.566624164581299, "torch_peak_reserved_gib": 3.623046875, "warmup_errors": [], "finished": "2026-10-01T14:34:37.123466+00:00", "predictions_sha256": "3aec21eba88f55825571220210edfd6b06d6f2483e73f543e48f2f40bc772055", "timing_coverage": 1.0, "plottable": true, "config": "bf16" }, { "model": "decider-2b", "name": "Decider 2B", "code": "D2", "suite": "fresh/spam", "n": 200, "correct": 196, "accuracy": 0.98, "interval": [ 0.9497129130941735, 0.9921955735836505 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 46.64935008622706, "p95_ms": 55.59103999985382, "device_peak_gib": 5.778736114501953, "device_min_gib": 5.771411895751953, "samples": 94, "added_device_peak_gib": 3.9043922424316406, "torch_peak_allocated_gib": 3.5834531784057617, "torch_peak_reserved_gib": 3.62890625, "warmup_errors": [], "finished": "2026-10-01T14:34:40.389173+00:00", "predictions_sha256": "1fdf4dedff3775a978d0c98bcc5b4901d3e5bea251a0a5d56f6233590d3c1be7", "timing_coverage": 1.0, "plottable": true, "config": "bf16" }, { "model": "decider-2b", "name": "Decider 2B", "code": "D2", "suite": "fresh/phishing", "n": 200, "correct": 189, "accuracy": 0.945, "interval": [ 0.9042130012276914, 0.9690146582965411 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 46.107799862511456, "p95_ms": 58.0459950841032, "device_peak_gib": 5.775806427001953, "device_min_gib": 5.775806427001953, "samples": 93, "added_device_peak_gib": 3.9014625549316406, "torch_peak_allocated_gib": 3.577521800994873, "torch_peak_reserved_gib": 3.630859375, "warmup_errors": [], "finished": "2026-10-01T14:34:43.636679+00:00", "predictions_sha256": "b0dbf2c25fc6d526577fc7d5a2b7b2ecf221c93500c40b2fcb8a7f52b6c6bd20", "timing_coverage": 1.0, "plottable": true, "config": "bf16" }, { "model": "decider-2b", "name": "Decider 2B", "code": "D2", "suite": "massive_en", "n": 2948, "correct": 2433, "accuracy": 0.8253052917232022, "interval": [ 0.8111776712314901, 0.8385862225130848 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 47.515050042420626, "p95_ms": 55.4277200717479, "device_peak_gib": 5.780689239501953, "device_min_gib": 5.773365020751953, "samples": 96, "added_device_peak_gib": 3.9063453674316406, "torch_peak_allocated_gib": 3.577521800994873, "torch_peak_reserved_gib": 3.630859375, "warmup_errors": [], "finished": "2026-10-01T14:34:47.005801+00:00", "predictions_sha256": "43931a722d83fd06fb47337ce6d87a29299633e0aa0db23fb9e95c3cd6dc03cd", "timing_coverage": 1.0, "plottable": true, "config": "bf16" }, { "model": "decider-2b", "name": "Decider 2B", "code": "D2", "suite": "massive_nb", "n": 2948, "correct": 2099, "accuracy": 0.7120081411126187, "interval": [ 0.6953943437102668, 0.7280701334433923 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 47.23490006290376, "p95_ms": 55.07228994974866, "device_peak_gib": 5.775806427001953, "device_min_gib": 5.775806427001953, "samples": 96, "added_device_peak_gib": 3.9014625549316406, "torch_peak_allocated_gib": 3.577521800994873, "torch_peak_reserved_gib": 3.630859375, "warmup_errors": [], "finished": "2026-10-01T14:34:50.352438+00:00", "predictions_sha256": "2d57072b9732d99f70ecef6d0a33673dbfcd96cbd7abd132176056be9fe77a91", "timing_coverage": 1.0, "plottable": true, "config": "bf16" }, { "model": "decider-2b", "name": "Decider 2B", "code": "D2", "suite": "public_jevbench", "n": 231, "correct": 175, "accuracy": 0.7575757575757576, "interval": [ 0.6983904669524358, 0.808334370295568 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 50.68959994241595, "p95_ms": 191.73051494872198, "device_peak_gib": 6.607837677001953, "device_min_gib": 5.773365020751953, "samples": 158, "added_device_peak_gib": 4.733493804931641, "torch_peak_allocated_gib": 4.365880966186523, "torch_peak_reserved_gib": 4.462890625, "warmup_errors": [], "finished": "2026-10-01T14:34:55.617371+00:00", "predictions_sha256": "55fc27c6d7549dadfae803e84ccfe58dd2c197c5439b4fc6746dfbd526ae0154", "timing_coverage": 1.0, "plottable": true, "config": "bf16" }, { "model": "decider-2b", "name": "Decider 2B", "code": "D2", "suite": "kev_claim/devtools-v1", "n": 900, "correct": 434, "accuracy": 0.4822222222222222, "interval": [ 0.4503348643178603, 0.5115815960341172 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 46.93645006045699, "p95_ms": 56.30452511832118, "device_peak_gib": 5.843189239501953, "device_min_gib": 5.830005645751953, "samples": 57, "added_device_peak_gib": 3.9688453674316406, "torch_peak_allocated_gib": 3.660275936126709, "torch_peak_reserved_gib": 3.6953125, "warmup_errors": [], "finished": "2026-10-01T14:34:57.768223+00:00", "predictions_sha256": "978a5830cd7bcc013a2576ce78c3be00380f1473ebd6785f9fd170791a128aae", "timing_coverage": 1.0, "plottable": true, "config": "bf16" }, { "model": "decider-2b", "name": "Decider 2B", "code": "D2", "suite": "industry/routing/en", "n": 40, "correct": 35, "accuracy": 0.875, "interval": [ 0.738878801611489, 0.945404997490546 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 46.75854998640716, "p95_ms": 54.32212504092604, "device_peak_gib": 5.767993927001953, "device_min_gib": 5.767993927001953, "samples": 57, "added_device_peak_gib": 3.8936500549316406, "torch_peak_allocated_gib": 3.566624164581299, "torch_peak_reserved_gib": 3.623046875, "warmup_errors": [], "finished": "2026-10-01T14:34:59.906060+00:00", "predictions_sha256": "a161d54ef0e58fbf38715d1471615be66d888c97bf0b5f1289c83db27df46208", "timing_coverage": 1.0, "plottable": true, "config": "bf16" }, { "model": "decider-2b", "name": "Decider 2B", "code": "D2", "suite": "industry/routing/nb", "n": 40, "correct": 30, "accuracy": 0.75, "interval": [ 0.5980603857923197, 0.858128813609037 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 46.82265012525022, "p95_ms": 56.37380009284243, "device_peak_gib": 5.778736114501953, "device_min_gib": 5.771411895751953, "samples": 57, "added_device_peak_gib": 3.9043922424316406, "torch_peak_allocated_gib": 3.5713539123535156, "torch_peak_reserved_gib": 3.62890625, "warmup_errors": [], "finished": "2026-10-01T14:35:02.021069+00:00", "predictions_sha256": "f8ed410427d36632b530304ad3cf6218b7ebefc9d192297255b5f520754bcf4e", "timing_coverage": 1.0, "plottable": true, "config": "bf16" }, { "model": "decider-2b", "name": "Decider 2B", "code": "D2", "suite": "industry/documents/en", "n": 28, "correct": 28, "accuracy": 1.0, "interval": [ 0.8793566952341543, 0.9999999999999998 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 47.54649999085814, "p95_ms": 58.76267480198294, "device_peak_gib": 5.773853302001953, "device_min_gib": 5.773853302001953, "samples": 57, "added_device_peak_gib": 3.8995094299316406, "torch_peak_allocated_gib": 3.5713539123535156, "torch_peak_reserved_gib": 3.62890625, "warmup_errors": [], "finished": "2026-10-01T14:35:04.196402+00:00", "predictions_sha256": "8c17fbde511cbb1b3bd2ef7617d5e4661b374597eaaa920c6122f29211e6e15c", "timing_coverage": 1.0, "plottable": true, "config": "bf16" }, { "model": "decider-2b", "name": "Decider 2B", "code": "D2", "suite": "industry/documents/nb", "n": 28, "correct": 27, "accuracy": 0.9642857142857143, "interval": [ 0.8228780225664668, 0.9936674801509623 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 48.947100061923265, "p95_ms": 54.62679994525388, "device_peak_gib": 5.773853302001953, "device_min_gib": 5.773853302001953, "samples": 58, "added_device_peak_gib": 3.8995094299316406, "torch_peak_allocated_gib": 3.5713539123535156, "torch_peak_reserved_gib": 3.62890625, "warmup_errors": [], "finished": "2026-10-01T14:35:06.383566+00:00", "predictions_sha256": "c68fa5897270952b16eb97303cae7087ceaea6414e26986d9ee35a86108f02fe", "timing_coverage": 1.0, "plottable": true, "config": "bf16" }, { "model": "decider-4b", "name": "Decider 4B v2.1", "code": "D4", "suite": "fresh/news", "n": 200, "correct": 180, "accuracy": 0.9, "interval": [ 0.8505941875672826, 0.9343295513309041 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 61.69775000307709, "p95_ms": 83.13743000617251, "device_peak_gib": 10.193317413330078, "device_min_gib": 10.185993194580078, "samples": 125, "added_device_peak_gib": 8.33212661743164, "torch_peak_allocated_gib": 8.049638748168945, "torch_peak_reserved_gib": 8.056640625, "warmup_errors": [], "finished": "2026-10-01T14:24:10.251124+00:00", "predictions_sha256": "8e13fa8df8c9f1f96caeabb357c7e2ca1a7cca485890d5ef423116a161c08c0a", "timing_coverage": 1.0, "plottable": true, "config": "bf16" }, { "model": "decider-4b", "name": "Decider 4B v2.1", "code": "D4", "suite": "fresh/emotion", "n": 200, "correct": 153, "accuracy": 0.765, "interval": [ 0.7015863111723746, 0.8184256658476741 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 67.60890001896769, "p95_ms": 93.13666995149106, "device_peak_gib": 10.173786163330078, "device_min_gib": 10.166461944580078, "samples": 140, "added_device_peak_gib": 8.31259536743164, "torch_peak_allocated_gib": 8.027344226837158, "torch_peak_reserved_gib": 8.037109375, "warmup_errors": [], "finished": "2026-10-01T14:24:15.072354+00:00", "predictions_sha256": "adeb9bec8b79e58bd5e75a03494ac13114a06dcd7b4e263a52267343b9c29ff0", "timing_coverage": 1.0, "plottable": true, "config": "bf16" }, { "model": "decider-4b", "name": "Decider 4B v2.1", "code": "D4", "suite": "fresh/spam", "n": 200, "correct": 195, "accuracy": 0.975, "interval": [ 0.942821659593358, 0.9892752803482386 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 62.12014995981008, "p95_ms": 70.41701487032697, "device_peak_gib": 10.227497100830078, "device_min_gib": 10.188434600830078, "samples": 127, "added_device_peak_gib": 8.36630630493164, "torch_peak_allocated_gib": 8.068127632141113, "torch_peak_reserved_gib": 8.095703125, "warmup_errors": [], "finished": "2026-10-01T14:24:19.331896+00:00", "predictions_sha256": "d8f74616be540c9600c4f6689bdd81ed9decef7da7db223d70ac30a98f952db8", "timing_coverage": 1.0, "plottable": true, "config": "bf16" }, { "model": "decider-4b", "name": "Decider 4B v2.1", "code": "D4", "suite": "fresh/phishing", "n": 200, "correct": 194, "accuracy": 0.97, "interval": [ 0.9361057075454807, 0.9861796856598887 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 59.53335005324334, "p95_ms": 70.30287502566352, "device_peak_gib": 10.212848663330078, "device_min_gib": 10.205524444580078, "samples": 120, "added_device_peak_gib": 8.35165786743164, "torch_peak_allocated_gib": 8.049968242645264, "torch_peak_reserved_gib": 8.076171875, "warmup_errors": [], "finished": "2026-10-01T14:24:23.504249+00:00", "predictions_sha256": "ae7c5e670081e79b3cdea10c8a61dd3914726bcfe8aa35d6ff5813cc87273b8e", "timing_coverage": 1.0, "plottable": true, "config": "bf16" }, { "model": "decider-4b", "name": "Decider 4B v2.1", "code": "D4", "suite": "massive_en", "n": 2948, "correct": 2603, "accuracy": 0.8829715061058344, "interval": [ 0.8708660763602786, 0.8940801551700913 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 58.11190011445433, "p95_ms": 65.6022650655359, "device_peak_gib": 10.208942413330078, "device_min_gib": 10.201618194580078, "samples": 119, "added_device_peak_gib": 8.34775161743164, "torch_peak_allocated_gib": 8.049968242645264, "torch_peak_reserved_gib": 8.072265625, "warmup_errors": [], "finished": "2026-10-01T14:24:27.558554+00:00", "predictions_sha256": "36e090caec72f35d911895050308604d3de7de7f799df4ed28f6047fdfb61b6d", "timing_coverage": 1.0, "plottable": true, "config": "bf16" }, { "model": "decider-4b", "name": "Decider 4B v2.1", "code": "D4", "suite": "massive_nb", "n": 2948, "correct": 2418, "accuracy": 0.8202170963364993, "interval": [ 0.8059412231536095, 0.8336595231392775 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 58.453200035728514, "p95_ms": 72.06724494462833, "device_peak_gib": 10.208942413330078, "device_min_gib": 10.201618194580078, "samples": 118, "added_device_peak_gib": 8.34775161743164, "torch_peak_allocated_gib": 8.054559707641602, "torch_peak_reserved_gib": 8.072265625, "warmup_errors": [], "finished": "2026-10-01T14:24:31.674224+00:00", "predictions_sha256": "bcfae5a0b0ae94830682572a7467af2ff616502d61da614c29037517d91706bf", "timing_coverage": 1.0, "plottable": true, "config": "bf16" }, { "model": "decider-4b", "name": "Decider 4B v2.1", "code": "D4", "suite": "public_jevbench", "n": 231, "correct": 191, "accuracy": 0.8268398268398268, "interval": [ 0.772804635538447, 0.8701823428182636 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 63.73119994532317, "p95_ms": 461.87673496315244, "device_peak_gib": 11.867145538330078, "device_min_gib": 10.207965850830078, "samples": 304, "added_device_peak_gib": 10.00595474243164, "torch_peak_allocated_gib": 9.589852333068848, "torch_peak_reserved_gib": 9.73046875, "warmup_errors": [], "finished": "2026-10-01T14:24:41.628369+00:00", "predictions_sha256": "6f6dc869758742c4065e5aa256a1bc38946ceefd585bad743fa0d375a36a5071", "timing_coverage": 1.0, "plottable": true, "config": "bf16" }, { "model": "decider-4b", "name": "Decider 4B v2.1", "code": "D4", "suite": "kev_claim/devtools-v1", "n": 900, "correct": 531, "accuracy": 0.59, "interval": [ 0.5581390555800533, 0.6296783157389794 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 62.63319996651262, "p95_ms": 133.70697502978146, "device_peak_gib": 10.422809600830078, "device_min_gib": 10.402790069580078, "samples": 89, "added_device_peak_gib": 8.56161880493164, "torch_peak_allocated_gib": 8.221017360687256, "torch_peak_reserved_gib": 8.291015625, "warmup_errors": [], "finished": "2026-10-01T14:24:44.961948+00:00", "predictions_sha256": "57f9e60cfe48e3cbd76bbd1b4f47e97e736f0595a42a4e73021750960100ef71", "timing_coverage": 1.0, "plottable": true, "config": "bf16" }, { "model": "decider-4b", "name": "Decider 4B v2.1", "code": "D4", "suite": "industry/routing/en", "n": 40, "correct": 38, "accuracy": 0.95, "interval": [ 0.8349612263085903, 0.9861793326138516 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 61.66925001889467, "p95_ms": 66.73937500454485, "device_peak_gib": 10.168903350830078, "device_min_gib": 10.166461944580078, "samples": 73, "added_device_peak_gib": 8.30771255493164, "torch_peak_allocated_gib": 8.027344226837158, "torch_peak_reserved_gib": 8.037109375, "warmup_errors": [], "finished": "2026-10-01T14:24:47.694902+00:00", "predictions_sha256": "731f30352e61c700642448755e4fce03dd23553c3b3237dc0460c678f371a08c", "timing_coverage": 1.0, "plottable": true, "config": "bf16" }, { "model": "decider-4b", "name": "Decider 4B v2.1", "code": "D4", "suite": "industry/routing/nb", "n": 40, "correct": 33, "accuracy": 0.825, "interval": [ 0.6805000966821229, 0.9125458625396408 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 59.45970001630485, "p95_ms": 77.00112491147593, "device_peak_gib": 10.188434600830078, "device_min_gib": 10.188434600830078, "samples": 73, "added_device_peak_gib": 8.32724380493164, "torch_peak_allocated_gib": 8.049638748168945, "torch_peak_reserved_gib": 8.056640625, "warmup_errors": [], "finished": "2026-10-01T14:24:50.454383+00:00", "predictions_sha256": "cb0ebaeb71a4f68391fbdee46387f1d817e3b1a5ade64abd61277a17c8d5364c", "timing_coverage": 1.0, "plottable": true, "config": "bf16" }, { "model": "decider-4b", "name": "Decider 4B v2.1", "code": "D4", "suite": "industry/documents/en", "n": 28, "correct": 28, "accuracy": 1.0, "interval": [ 0.8793566952341543, 0.9999999999999998 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 62.74769990704954, "p95_ms": 73.71582492487505, "device_peak_gib": 10.188434600830078, "device_min_gib": 10.185993194580078, "samples": 76, "added_device_peak_gib": 8.32724380493164, "torch_peak_allocated_gib": 8.049638748168945, "torch_peak_reserved_gib": 8.056640625, "warmup_errors": [], "finished": "2026-10-01T14:24:53.253383+00:00", "predictions_sha256": "e5d046793bdb046bc0662e091ef8e3c60f86efe6bba0af108261bbbd3525828b", "timing_coverage": 1.0, "plottable": true, "config": "bf16" }, { "model": "decider-4b", "name": "Decider 4B v2.1", "code": "D4", "suite": "industry/documents/nb", "n": 28, "correct": 28, "accuracy": 1.0, "interval": [ 0.8793566952341543, 0.9999999999999998 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 59.48884994722903, "p95_ms": 73.12195014674217, "device_peak_gib": 10.184528350830078, "device_min_gib": 10.182086944580078, "samples": 73, "added_device_peak_gib": 8.32333755493164, "torch_peak_allocated_gib": 8.049638748168945, "torch_peak_reserved_gib": 8.052734375, "warmup_errors": [], "finished": "2026-10-01T14:24:55.984068+00:00", "predictions_sha256": "c3d9283051590f4f83a8050711b4d89ea4243f8c1f27198220c857ca4535ab31", "timing_coverage": 1.0, "plottable": true, "config": "bf16" }, { "model": "decision-4b-v12", "name": "Decision 4B v1.2", "code": "DN", "suite": "fresh/news", "n": 200, "correct": 174, "accuracy": 0.87, "interval": [ 0.8163364812775159, 0.9097179772033067 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 53.56060015037656, "p95_ms": 63.365440152119845, "device_peak_gib": 10.144855499267578, "device_min_gib": 10.137531280517578, "samples": 108, "added_device_peak_gib": 8.28366470336914, "torch_peak_allocated_gib": 7.992424964904785, "torch_peak_reserved_gib": 8.0078125, "warmup_errors": [], "finished": "2026-10-01T14:25:18.493151+00:00", "predictions_sha256": "afc74d5901266f3c243a9ff1dd48b5587a863648be494d8be09eda7716854aa2", "timing_coverage": 1.0, "plottable": true, "config": "bf16 merged LoRA; fp32 letter projection" }, { "model": "decision-4b-v12", "name": "Decision 4B v1.2", "code": "DN", "suite": "fresh/emotion", "n": 200, "correct": 116, "accuracy": 0.58, "interval": [ 0.5107207393958046, 0.6462640083838327 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 55.30404997989535, "p95_ms": 66.2652850849554, "device_peak_gib": 10.144855499267578, "device_min_gib": 10.120441436767578, "samples": 111, "added_device_peak_gib": 8.28366470336914, "torch_peak_allocated_gib": 7.9881110191345215, "torch_peak_reserved_gib": 8.0078125, "warmup_errors": [], "finished": "2026-10-01T14:25:22.369255+00:00", "predictions_sha256": "c37d5d4c81b675a3107cac6bef900266eafcf23934d361ac9a30975093672399", "timing_coverage": 1.0, "plottable": true, "config": "bf16 merged LoRA; fp32 letter projection" }, { "model": "decision-4b-v12", "name": "Decision 4B v1.2", "code": "DN", "suite": "fresh/spam", "n": 200, "correct": 189, "accuracy": 0.945, "interval": [ 0.9042130012276914, 0.9690146582965411 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 52.614850108511746, "p95_ms": 63.004340103361756, "device_peak_gib": 10.184894561767578, "device_min_gib": 10.165363311767578, "samples": 106, "added_device_peak_gib": 8.32370376586914, "torch_peak_allocated_gib": 8.014613151550293, "torch_peak_reserved_gib": 8.052734375, "warmup_errors": [], "finished": "2026-10-01T14:25:26.093974+00:00", "predictions_sha256": "f411d4db9289f67e4a429246acc81bc50d797aa042716bd6541adbd2af079851", "timing_coverage": 1.0, "plottable": true, "config": "bf16 merged LoRA; fp32 letter projection" }, { "model": "decision-4b-v12", "name": "Decision 4B v1.2", "code": "DN", "suite": "fresh/phishing", "n": 200, "correct": 185, "accuracy": 0.925, "interval": [ 0.8799563897637055, 0.954025082815618 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 54.6448000241071, "p95_ms": 64.49170495616272, "device_peak_gib": 10.164386749267578, "device_min_gib": 10.157062530517578, "samples": 108, "added_device_peak_gib": 8.30319595336914, "torch_peak_allocated_gib": 7.999449729919434, "torch_peak_reserved_gib": 8.02734375, "warmup_errors": [], "finished": "2026-10-01T14:25:29.860291+00:00", "predictions_sha256": "174a3c33181be2b1f5908fa387b3bdd43f16ab6b593170bea75ef18efebc87e8", "timing_coverage": 1.0, "plottable": true, "config": "bf16 merged LoRA; fp32 letter projection" }, { "model": "decision-4b-v12", "name": "Decision 4B v1.2", "code": "DN", "suite": "massive_en", "n": 2948, "correct": 2425, "accuracy": 0.8225915875169606, "interval": [ 0.808384378706726, 0.8359591697302817 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 63.23070009239018, "p95_ms": 73.83125498890877, "device_peak_gib": 10.432876586914062, "device_min_gib": 10.432876586914062, "samples": 125, "added_device_peak_gib": 8.31552505493164, "torch_peak_allocated_gib": 8.020895004272461, "torch_peak_reserved_gib": 8.044921875, "warmup_errors": [], "finished": "2026-10-04T20:53:34.610190+00:00", "predictions_sha256": "9eec27c7dd7008158554a92541b0a2e0a639bffd862d9dccefd1bad74e45c024", "timing_coverage": 1.0, "plottable": true, "config": "bf16 merged LoRA; fp32 letter projection" }, { "model": "decision-4b-v12", "name": "Decision 4B v1.2", "code": "DN", "suite": "massive_nb", "n": 2948, "correct": 2150, "accuracy": 0.7293080054274084, "interval": [ 0.712978241741253, 0.7450409367481446 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 63.267650082707405, "p95_ms": 73.95813988987356, "device_peak_gib": 10.432876586914062, "device_min_gib": 10.432876586914062, "samples": 125, "added_device_peak_gib": 8.31552505493164, "torch_peak_allocated_gib": 8.022064208984375, "torch_peak_reserved_gib": 8.044921875, "warmup_errors": [], "finished": "2026-10-04T20:53:38.992670+00:00", "predictions_sha256": "a095dc5fdcd9b6c5f4165877ac5d2c3dfaac3edb945f59457dcd2afd1ab7062d", "timing_coverage": 1.0, "plottable": true, "config": "bf16 merged LoRA; fp32 letter projection" }, { "model": "decision-4b-v12", "name": "Decision 4B v1.2", "code": "DN", "suite": "public_jevbench", "n": 231, "correct": 203, "accuracy": 0.8787878787878788, "interval": [ 0.8303921928936235, 0.9147913912946171 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 60.72359986137599, "p95_ms": 535.751825047191, "device_peak_gib": 13.277667999267578, "device_min_gib": 10.159503936767578, "samples": 321, "added_device_peak_gib": 11.41647720336914, "torch_peak_allocated_gib": 9.886847972869873, "torch_peak_reserved_gib": 11.140625, "warmup_errors": [], "finished": "2026-10-01T14:25:40.246273+00:00", "predictions_sha256": "77ee6d1d7da3fc56baba4c3695d0766ea9697a0d959e2f09467f9e29b69e5f95", "timing_coverage": 1.0, "plottable": true, "config": "bf16 merged LoRA; fp32 letter projection" }, { "model": "decision-4b-v12", "name": "Decision 4B v1.2", "code": "DN", "suite": "kev_claim/devtools-v1", "n": 900, "correct": 578, "accuracy": 0.6422222222222222, "interval": [ 0.6038746159324917, 0.6862759558100526 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 62.17489985283464, "p95_ms": 187.55772506119683, "device_peak_gib": 10.660415649414062, "device_min_gib": 10.504653930664062, "samples": 101, "added_device_peak_gib": 8.54306411743164, "torch_peak_allocated_gib": 8.214274406433105, "torch_peak_reserved_gib": 8.267578125, "warmup_errors": [], "finished": "2026-10-04T20:53:42.848526+00:00", "predictions_sha256": "996e3fa95d9efd31a57983a9cd9fefd487164268e5bb6b2491f469e90e289111", "timing_coverage": 1.0, "plottable": true, "config": "bf16 merged LoRA; fp32 letter projection" }, { "model": "decision-4b-v12", "name": "Decision 4B v1.2", "code": "DN", "suite": "industry/routing/en", "n": 40, "correct": 40, "accuracy": 1.0, "interval": [ 0.9123783988027135, 1.0 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 52.452900097705424, "p95_ms": 61.43589993007481, "device_peak_gib": 10.391860961914062, "device_min_gib": 10.391860961914062, "samples": 64, "added_device_peak_gib": 8.27450942993164, "torch_peak_allocated_gib": 7.99250602722168, "torch_peak_reserved_gib": 8.00390625, "warmup_errors": [], "finished": "2026-10-04T20:53:45.230360+00:00", "predictions_sha256": "0273a41098bea1dabfeb579f7a08e80206b128eccfb7e2a9abde2a18ec303297", "timing_coverage": 1.0, "plottable": true, "config": "bf16 merged LoRA; fp32 letter projection" }, { "model": "decision-4b-v12", "name": "Decision 4B v1.2", "code": "DN", "suite": "industry/routing/nb", "n": 40, "correct": 38, "accuracy": 0.95, "interval": [ 0.8349612263085903, 0.9861793326138516 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 52.49569995794445, "p95_ms": 63.16542509011924, "device_peak_gib": 10.398696899414062, "device_min_gib": 10.391372680664062, "samples": 64, "added_device_peak_gib": 8.28134536743164, "torch_peak_allocated_gib": 7.993659973144531, "torch_peak_reserved_gib": 8.005859375, "warmup_errors": [], "finished": "2026-10-04T20:53:47.613454+00:00", "predictions_sha256": "1c09017c5e054da4eb66e90013c08cc9a0eca6505f79e45d144ed83ae08d757b", "timing_coverage": 1.0, "plottable": true, "config": "bf16 merged LoRA; fp32 letter projection" }, { "model": "decision-4b-v12", "name": "Decision 4B v1.2", "code": "DN", "suite": "industry/documents/en", "n": 28, "correct": 27, "accuracy": 0.9642857142857143, "interval": [ 0.8228780225664668, 0.9936674801509623 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 53.125200094655156, "p95_ms": 60.88827509665862, "device_peak_gib": 10.395767211914062, "device_min_gib": 10.395767211914062, "samples": 64, "added_device_peak_gib": 8.27841567993164, "torch_peak_allocated_gib": 7.992801189422607, "torch_peak_reserved_gib": 8.0078125, "warmup_errors": [], "finished": "2026-10-04T20:53:50.004024+00:00", "predictions_sha256": "ee4b9d4b7d729c903d93286033f5101cefc1fe49302f6d7127074f624b2b8706", "timing_coverage": 1.0, "plottable": true, "config": "bf16 merged LoRA; fp32 letter projection" }, { "model": "decision-4b-v12", "name": "Decision 4B v1.2", "code": "DN", "suite": "industry/documents/nb", "n": 28, "correct": 24, "accuracy": 0.8571428571428571, "interval": [ 0.685101978364747, 0.9430099468025059 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 53.08694997802377, "p95_ms": 61.26685003982857, "device_peak_gib": 10.396743774414062, "device_min_gib": 10.389419555664062, "samples": 64, "added_device_peak_gib": 8.27939224243164, "torch_peak_allocated_gib": 7.99274206161499, "torch_peak_reserved_gib": 8.00390625, "warmup_errors": [], "finished": "2026-10-04T20:53:52.396724+00:00", "predictions_sha256": "6a5252c3709827ba46685400939988925f4968cc2955236ce3c9d134083a5fcb", "timing_coverage": 1.0, "plottable": true, "config": "bf16 merged LoRA; fp32 letter projection" }, { "model": "gliner-decide-1b", "name": "GLiNER Decide 1B", "code": "G1", "suite": "fresh/news", "n": 200, "correct": 142, "accuracy": 0.71, "interval": [ 0.6436252189922268, 0.7684597439293211 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 19.983200007118285, "p95_ms": 27.961589989718046, "device_peak_gib": 6.814868927001953, "device_min_gib": 6.814868927001953, "samples": 43, "added_device_peak_gib": 4.938083648681641, "torch_peak_allocated_gib": 4.48210334777832, "torch_peak_reserved_gib": 4.671875, "warmup_errors": [], "finished": "2026-10-01T14:35:29.415952+00:00", "predictions_sha256": "3f4bed86429ee531132ebad0b0f4d010a3a61f23829fb43947a4414bf4ba3f11", "timing_coverage": 1.0, "plottable": true, "config": "fp32" }, { "model": "gliner-decide-1b", "name": "GLiNER Decide 1B", "code": "G1", "suite": "fresh/emotion", "n": 200, "correct": 91, "accuracy": 0.455, "interval": [ 0.387482193098835, 0.524213886275119 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 19.2374000325799, "p95_ms": 30.909325031097975, "device_peak_gib": 6.821704864501953, "device_min_gib": 6.814380645751953, "samples": 41, "added_device_peak_gib": 4.944919586181641, "torch_peak_allocated_gib": 4.476273536682129, "torch_peak_reserved_gib": 4.671875, "warmup_errors": [], "finished": "2026-10-01T14:35:30.826559+00:00", "predictions_sha256": "563495c8c6f1cf9afc98e6a89f4f00fd13d366b4db8f7bda75479158c378de55", "timing_coverage": 1.0, "plottable": true, "config": "fp32" }, { "model": "gliner-decide-1b", "name": "GLiNER Decide 1B", "code": "G1", "suite": "fresh/spam", "n": 200, "correct": 173, "accuracy": 0.865, "interval": [ 0.8107086162924745, 0.9055342954521209 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 26.587999891489744, "p95_ms": 30.43828987283632, "device_peak_gib": 6.816822052001953, "device_min_gib": 6.816822052001953, "samples": 49, "added_device_peak_gib": 4.940036773681641, "torch_peak_allocated_gib": 4.4875807762146, "torch_peak_reserved_gib": 4.671875, "warmup_errors": [], "finished": "2026-10-01T14:35:32.532597+00:00", "predictions_sha256": "778753c79efbf2d509472466fb1522191b99f6d6c0041ed5b9ca75aa0f22f1e1", "timing_coverage": 1.0, "plottable": true, "config": "fp32" }, { "model": "gliner-decide-1b", "name": "GLiNER Decide 1B", "code": "G1", "suite": "fresh/phishing", "n": 200, "correct": 122, "accuracy": 0.61, "interval": [ 0.5409374595716976, 0.6749165686253038 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 27.35104993917048, "p95_ms": 30.287845188286155, "device_peak_gib": 6.816822052001953, "device_min_gib": 6.816822052001953, "samples": 50, "added_device_peak_gib": 4.940036773681641, "torch_peak_allocated_gib": 4.487522602081299, "torch_peak_reserved_gib": 4.671875, "warmup_errors": [], "finished": "2026-10-01T14:35:34.289656+00:00", "predictions_sha256": "837dcc5ca1a32f3293322d6876b7305ea783487723f8b8077ed5eeb499293992", "timing_coverage": 1.0, "plottable": true, "config": "fp32" }, { "model": "gliner-decide-1b", "name": "GLiNER Decide 1B", "code": "G1", "suite": "massive_en", "n": 2948, "correct": 1729, "accuracy": 0.5864993215739485, "interval": [ 0.5686210598294915, 0.6041524468492016 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 29.77485000155866, "p95_ms": 31.522204983048116, "device_peak_gib": 6.815845489501953, "device_min_gib": 6.808521270751953, "samples": 60, "added_device_peak_gib": 4.939060211181641, "torch_peak_allocated_gib": 4.485198020935059, "torch_peak_reserved_gib": 4.666015625, "warmup_errors": [], "finished": "2026-10-01T14:35:36.374724+00:00", "predictions_sha256": "7f6e0b4ce45177ac084f8bef426d702de5f0a1ac222c83fb8730d24c7fc807f0", "timing_coverage": 1.0, "plottable": true, "config": "fp32" }, { "model": "gliner-decide-1b", "name": "GLiNER Decide 1B", "code": "G1", "suite": "massive_nb", "n": 2948, "correct": 729, "accuracy": 0.2472862957937585, "interval": [ 0.232047866064693, 0.26318247716814525 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 29.805899946950376, "p95_ms": 39.532079990021884, "device_peak_gib": 6.810962677001953, "device_min_gib": 6.810962677001953, "samples": 60, "added_device_peak_gib": 4.934177398681641, "torch_peak_allocated_gib": 4.485491752624512, "torch_peak_reserved_gib": 4.666015625, "warmup_errors": [], "finished": "2026-10-01T14:35:38.449519+00:00", "predictions_sha256": "61585cebfbeb59da78d169330e54fab76f97453a38753ac1d844604e605f7f24", "timing_coverage": 1.0, "plottable": true, "config": "fp32" }, { "model": "gliner-decide-1b", "name": "GLiNER Decide 1B", "code": "G1", "suite": "public_jevbench", "n": 231, "correct": 121, "accuracy": 0.5238095238095238, "interval": [ 0.45954286201018196, 0.587297248995936 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 29.900499968789518, "p95_ms": 417.4396501039154, "device_peak_gib": 7.372486114501953, "device_min_gib": 6.816822052001953, "samples": 234, "added_device_peak_gib": 5.495700836181641, "torch_peak_allocated_gib": 4.769181728363037, "torch_peak_reserved_gib": 5.22265625, "warmup_errors": [], "finished": "2026-10-01T14:35:45.916974+00:00", "predictions_sha256": "c5fff21ab27b0ad913f9ef0f181e527cb1420668809d29eaab3cbf25a3e519d2", "timing_coverage": 1.0, "plottable": true, "config": "fp32" }, { "model": "gliner-decide-1b", "name": "GLiNER Decide 1B", "code": "G1", "suite": "kev_claim/devtools-v1", "n": 900, "correct": 472, "accuracy": 0.5244444444444445, "interval": [ 0.49227594728358604, 0.5550240369338042 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 31.445500091649592, "p95_ms": 174.05134986620396, "device_peak_gib": 6.985767364501953, "device_min_gib": 6.887134552001953, "samples": 53, "added_device_peak_gib": 5.108982086181641, "torch_peak_allocated_gib": 4.6019392013549805, "torch_peak_reserved_gib": 4.8359375, "warmup_errors": [], "finished": "2026-10-01T14:35:48.127722+00:00", "predictions_sha256": "ff326445654ac18c671e189e99e1db2767068abc5c2cf64554cbf086bb60c8b2", "timing_coverage": 1.0, "plottable": true, "config": "fp32" }, { "model": "gliner-decide-1b", "name": "GLiNER Decide 1B", "code": "G1", "suite": "industry/routing/en", "n": 40, "correct": 32, "accuracy": 0.8, "interval": [ 0.652426936536005, 0.895000102745623 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 26.91414998844266, "p95_ms": 31.422600033693016, "device_peak_gib": 6.816822052001953, "device_min_gib": 6.816822052001953, "samples": 31, "added_device_peak_gib": 4.940036773681641, "torch_peak_allocated_gib": 4.478109359741211, "torch_peak_reserved_gib": 4.671875, "warmup_errors": [], "finished": "2026-10-01T14:35:49.269757+00:00", "predictions_sha256": "2b359e1d32d460cad1fd270eb1e5b82047c3f84b67840a03974ab31c8d13ca00", "timing_coverage": 1.0, "plottable": true, "config": "fp32" }, { "model": "gliner-decide-1b", "name": "GLiNER Decide 1B", "code": "G1", "suite": "industry/routing/nb", "n": 40, "correct": 23, "accuracy": 0.575, "interval": [ 0.42195068541991065, 0.7149060744004962 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 27.354500023648143, "p95_ms": 30.749274999834597, "device_peak_gib": 6.816822052001953, "device_min_gib": 6.816822052001953, "samples": 34, "added_device_peak_gib": 4.940036773681641, "torch_peak_allocated_gib": 4.480465412139893, "torch_peak_reserved_gib": 4.671875, "warmup_errors": [], "finished": "2026-10-01T14:35:50.504171+00:00", "predictions_sha256": "ac4b9fb38ab2530dfd9efa82614017a5f68d14f2bbd678ab3d16a46df7115a84", "timing_coverage": 1.0, "plottable": true, "config": "fp32" }, { "model": "gliner-decide-1b", "name": "GLiNER Decide 1B", "code": "G1", "suite": "industry/documents/en", "n": 28, "correct": 26, "accuracy": 0.9285714285714286, "interval": [ 0.7735463743160095, 0.9801879358846942 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 24.274899973534048, "p95_ms": 30.959349998738617, "device_peak_gib": 6.816822052001953, "device_min_gib": 6.816822052001953, "samples": 30, "added_device_peak_gib": 4.940036773681641, "torch_peak_allocated_gib": 4.478384017944336, "torch_peak_reserved_gib": 4.671875, "warmup_errors": [], "finished": "2026-10-01T14:35:51.596772+00:00", "predictions_sha256": "ee70a12cb8c23cc94448aabf97330b6aacb676894b37a64197164b048adc2036", "timing_coverage": 1.0, "plottable": true, "config": "fp32" }, { "model": "gliner-decide-1b", "name": "GLiNER Decide 1B", "code": "G1", "suite": "industry/documents/nb", "n": 28, "correct": 21, "accuracy": 0.75, "interval": [ 0.5664432968100104, 0.8732350508070668 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 27.83575002104044, "p95_ms": 37.448725081048906, "device_peak_gib": 6.821704864501953, "device_min_gib": 6.814380645751953, "samples": 34, "added_device_peak_gib": 4.944919586181641, "torch_peak_allocated_gib": 4.480465412139893, "torch_peak_reserved_gib": 4.671875, "warmup_errors": [], "finished": "2026-10-01T14:35:52.859372+00:00", "predictions_sha256": "79dc8aa7bfb655d3b7bc42ca8f76c81fbf7320773704b3a7fbe8572722403845", "timing_coverage": 1.0, "plottable": true, "config": "fp32" }, { "model": "gliner-decide-multi", "name": "GLiNER multilingual", "code": "GM", "suite": "fresh/news", "n": 200, "correct": 149, "accuracy": 0.745, "interval": [ 0.6803707417954529, 0.8003950482796865 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 16.21149992570281, "p95_ms": 27.52794994739815, "device_peak_gib": 3.325611114501953, "device_min_gib": 3.318286895751953, "samples": 37, "added_device_peak_gib": 1.4512672424316406, "torch_peak_allocated_gib": 1.1300530433654785, "torch_peak_reserved_gib": 1.17578125, "warmup_errors": [], "finished": "2026-10-01T14:36:16.081996+00:00", "predictions_sha256": "6d05c0ac2682932e8df5bb67baac5ee774182e068a0d20145f911f94371b7bf1", "timing_coverage": 1.0, "plottable": true, "config": "fp32" }, { "model": "gliner-decide-multi", "name": "GLiNER multilingual", "code": "GM", "suite": "fresh/emotion", "n": 200, "correct": 78, "accuracy": 0.39, "interval": [ 0.3250834313746962, 0.4590625404283025 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 13.735149870626628, "p95_ms": 23.033094964921474, "device_peak_gib": 3.299243927001953, "device_min_gib": 3.295337677001953, "samples": 31, "added_device_peak_gib": 1.4249000549316406, "torch_peak_allocated_gib": 1.1198334693908691, "torch_peak_reserved_gib": 1.154296875, "warmup_errors": [], "finished": "2026-10-01T14:36:17.132985+00:00", "predictions_sha256": "73b03fecfb112d1f8e47c1af10ac3554eb4943fc2758679a50434a654467615a", "timing_coverage": 1.0, "plottable": true, "config": "fp32" }, { "model": "gliner-decide-multi", "name": "GLiNER multilingual", "code": "GM", "suite": "fresh/spam", "n": 200, "correct": 125, "accuracy": 0.625, "interval": [ 0.5561430369897816, 0.6891456314159018 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 14.191749971359968, "p95_ms": 21.898774977307767, "device_peak_gib": 3.346118927001953, "device_min_gib": 3.324634552001953, "samples": 31, "added_device_peak_gib": 1.4717750549316406, "torch_peak_allocated_gib": 1.14923095703125, "torch_peak_reserved_gib": 1.201171875, "warmup_errors": [], "finished": "2026-10-01T14:36:18.190740+00:00", "predictions_sha256": "906ca7d0c9fcdd5413441155cbc438a72887f57daaddee6a9fb9e2bc2665bcbe", "timing_coverage": 1.0, "plottable": true, "config": "fp32" }, { "model": "gliner-decide-multi", "name": "GLiNER multilingual", "code": "GM", "suite": "fresh/phishing", "n": 200, "correct": 129, "accuracy": 0.645, "interval": [ 0.5765211307549043, 0.7080137245956883 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 14.140550047159195, "p95_ms": 22.935159888584163, "device_peak_gib": 3.326587677001953, "device_min_gib": 3.326587677001953, "samples": 31, "added_device_peak_gib": 1.4522438049316406, "torch_peak_allocated_gib": 1.1459288597106934, "torch_peak_reserved_gib": 1.181640625, "warmup_errors": [], "finished": "2026-10-01T14:36:19.240807+00:00", "predictions_sha256": "13688b7d2072f4af1e68cc46fcb802214ee775ea87fc8bc96ad6c3b833a34ced", "timing_coverage": 1.0, "plottable": true, "config": "fp32" }, { "model": "gliner-decide-multi", "name": "GLiNER multilingual", "code": "GM", "suite": "massive_en", "n": 2948, "correct": 1625, "accuracy": 0.5512211668928086, "interval": [ 0.5332119780749904, 0.5690970396053336 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 13.943100115284324, "p95_ms": 22.941044985782355, "device_peak_gib": 3.320728302001953, "device_min_gib": 3.320728302001953, "samples": 31, "added_device_peak_gib": 1.4463844299316406, "torch_peak_allocated_gib": 1.1341447830200195, "torch_peak_reserved_gib": 1.17578125, "warmup_errors": [], "finished": "2026-10-01T14:36:20.279010+00:00", "predictions_sha256": "9fbd4220359d8ae801fe6b9edece35dba76bd2a8d18258c35a920ce6eb812b78", "timing_coverage": 1.0, "plottable": true, "config": "fp32" }, { "model": "gliner-decide-multi", "name": "GLiNER multilingual", "code": "GM", "suite": "massive_nb", "n": 2948, "correct": 1455, "accuracy": 0.4935549525101764, "interval": [ 0.47552753436167267, 0.5115991455328561 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 14.069249969907105, "p95_ms": 22.629609878640622, "device_peak_gib": 3.325611114501953, "device_min_gib": 3.318286895751953, "samples": 31, "added_device_peak_gib": 1.4512672424316406, "torch_peak_allocated_gib": 1.1342668533325195, "torch_peak_reserved_gib": 1.17578125, "warmup_errors": [], "finished": "2026-10-01T14:36:21.325714+00:00", "predictions_sha256": "08cac979f91e5689fd62f1a9dcbf290d940ee23988b50bb5c7f8134f4584069f", "timing_coverage": 1.0, "plottable": true, "config": "fp32" }, { "model": "gliner-decide-multi", "name": "GLiNER multilingual", "code": "GM", "suite": "public_jevbench", "n": 231, "correct": 131, "accuracy": 0.5670995670995671, "interval": [ 0.5026223726319217, 0.6293815765671382 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 15.30585007276386, "p95_ms": 384.8555350676179, "device_peak_gib": 10.714282989501953, "device_min_gib": 3.346118927001953, "samples": 191, "added_device_peak_gib": 8.83993911743164, "torch_peak_allocated_gib": 4.846553325653076, "torch_peak_reserved_gib": 8.564453125, "warmup_errors": [], "finished": "2026-10-01T14:36:27.401927+00:00", "predictions_sha256": "a2615cdbda3b4df248c9c7897e24aa31f55bd2ba62478b126d4413d24251d122", "timing_coverage": 1.0, "plottable": true, "config": "fp32" }, { "model": "gliner-decide-multi", "name": "GLiNER multilingual", "code": "GM", "suite": "kev_claim/devtools-v1", "n": 900, "correct": 393, "accuracy": 0.43666666666666665, "interval": [ 0.3969729721539701, 0.4731615119170784 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 15.180250047706068, "p95_ms": 121.75962486071512, "device_peak_gib": 4.572681427001953, "device_min_gib": 3.512134552001953, "samples": 31, "added_device_peak_gib": 2.6983375549316406, "torch_peak_allocated_gib": 2.20139217376709, "torch_peak_reserved_gib": 2.427734375, "warmup_errors": [], "finished": "2026-10-01T14:36:28.507414+00:00", "predictions_sha256": "edc46e2036955a49fffe21d55d4b0f15f3f3fb985f3551f95ea3348e3de3bac7", "timing_coverage": 1.0, "plottable": true, "config": "fp32" }, { "model": "gliner-decide-multi", "name": "GLiNER multilingual", "code": "GM", "suite": "industry/routing/en", "n": 40, "correct": 27, "accuracy": 0.675, "interval": [ 0.5201774618987768, 0.799154977682173 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 13.917400036007166, "p95_ms": 24.31127493036911, "device_peak_gib": 3.297290802001953, "device_min_gib": 3.297290802001953, "samples": 19, "added_device_peak_gib": 1.4229469299316406, "torch_peak_allocated_gib": 1.1284914016723633, "torch_peak_reserved_gib": 1.15234375, "warmup_errors": [], "finished": "2026-10-01T14:36:29.199876+00:00", "predictions_sha256": "dc5da2cd019cda3d52c714a6387ff67c3f64176e8203379c6da11f003aab2634", "timing_coverage": 1.0, "plottable": true, "config": "fp32" }, { "model": "gliner-decide-multi", "name": "GLiNER multilingual", "code": "GM", "suite": "industry/routing/nb", "n": 40, "correct": 27, "accuracy": 0.675, "interval": [ 0.5201774618987768, 0.799154977682173 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 14.186199987307191, "p95_ms": 21.783950098324567, "device_peak_gib": 3.297290802001953, "device_min_gib": 3.297290802001953, "samples": 19, "added_device_peak_gib": 1.4229469299316406, "torch_peak_allocated_gib": 1.1284914016723633, "torch_peak_reserved_gib": 1.15234375, "warmup_errors": [], "finished": "2026-10-01T14:36:29.870341+00:00", "predictions_sha256": "57ac0628c1500aadd99453f124f7d277a3529a68a9817ac68d39f044ee8a05f3", "timing_coverage": 1.0, "plottable": true, "config": "fp32" }, { "model": "gliner-decide-multi", "name": "GLiNER multilingual", "code": "GM", "suite": "industry/documents/en", "n": 28, "correct": 22, "accuracy": 0.7857142857142857, "interval": [ 0.6046141434627812, 0.8978753966710211 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 14.038449851796031, "p95_ms": 22.144025017041713, "device_peak_gib": 3.301197052001953, "device_min_gib": 3.301197052001953, "samples": 20, "added_device_peak_gib": 1.4268531799316406, "torch_peak_allocated_gib": 1.127549171447754, "torch_peak_reserved_gib": 1.15625, "warmup_errors": [], "finished": "2026-10-01T14:36:30.558877+00:00", "predictions_sha256": "d5487e44350bebb89d51f7c8bbd46a765d69ecdf3443958c361c250c39dfd205", "timing_coverage": 1.0, "plottable": true, "config": "fp32" }, { "model": "gliner-decide-multi", "name": "GLiNER multilingual", "code": "GM", "suite": "industry/documents/nb", "n": 28, "correct": 21, "accuracy": 0.75, "interval": [ 0.5664432968100104, 0.8732350508070668 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 13.843300053849816, "p95_ms": 21.77087514428422, "device_peak_gib": 3.301197052001953, "device_min_gib": 3.301197052001953, "samples": 19, "added_device_peak_gib": 1.4268531799316406, "torch_peak_allocated_gib": 1.1273918151855469, "torch_peak_reserved_gib": 1.15625, "warmup_errors": [], "finished": "2026-10-01T14:36:31.230845+00:00", "predictions_sha256": "243479f1e15e1f0dfc171478ea3058123e9c3ae4f06209caa776652d2659c023", "timing_coverage": 1.0, "plottable": true, "config": "fp32" }, { "model": "gliner-decide", "name": "GLiNER Decide", "code": "G", "suite": "fresh/news", "n": 200, "correct": 140, "accuracy": 0.7, "interval": [ 0.6332093162728074, 0.7592525531762859 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 25.923000066541135, "p95_ms": 35.08913518162444, "device_peak_gib": 4.042407989501953, "device_min_gib": 4.035083770751953, "samples": 54, "added_device_peak_gib": 2.1680641174316406, "torch_peak_allocated_gib": 1.8749256134033203, "torch_peak_reserved_gib": 1.892578125, "warmup_errors": [], "finished": "2026-10-01T14:36:45.786267+00:00", "predictions_sha256": "76ab5ef4b74f38514cd19ee6bc5b8f6e9257c35b23e991df432050a26fee3ebd", "timing_coverage": 1.0, "plottable": true, "config": "fp32" }, { "model": "gliner-decide", "name": "GLiNER Decide", "code": "G", "suite": "fresh/emotion", "n": 200, "correct": 106, "accuracy": 0.53, "interval": [ 0.4609168291500183, 0.5979524512673458 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 25.98880010191351, "p95_ms": 35.752765147481114, "device_peak_gib": 4.031665802001953, "device_min_gib": 4.027759552001953, "samples": 55, "added_device_peak_gib": 2.1573219299316406, "torch_peak_allocated_gib": 1.8645539283752441, "torch_peak_reserved_gib": 1.88671875, "warmup_errors": [], "finished": "2026-10-01T14:36:47.672987+00:00", "predictions_sha256": "925ffc3c7f0cea382be8277714e83644a2888e4fd3f66f9a21b00dfb9553630f", "timing_coverage": 1.0, "plottable": true, "config": "fp32" }, { "model": "gliner-decide", "name": "GLiNER Decide", "code": "G", "suite": "fresh/spam", "n": 200, "correct": 143, "accuracy": 0.715, "interval": [ 0.6488465397039482, 0.7730499699538271 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 26.672700070776045, "p95_ms": 38.66325003327802, "device_peak_gib": 4.069751739501953, "device_min_gib": 4.045337677001953, "samples": 56, "added_device_peak_gib": 2.1954078674316406, "torch_peak_allocated_gib": 1.896249771118164, "torch_peak_reserved_gib": 1.919921875, "warmup_errors": [], "finished": "2026-10-01T14:36:49.636229+00:00", "predictions_sha256": "855cf85551b4d4afbf616eeba02b2d1009596ae99cb4cb62366a1c15a133d55c", "timing_coverage": 1.0, "plottable": true, "config": "fp32" }, { "model": "gliner-decide", "name": "GLiNER Decide", "code": "G", "suite": "fresh/phishing", "n": 200, "correct": 127, "accuracy": 0.635, "interval": [ 0.566317026527406, 0.698594735350732 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 26.856250129640102, "p95_ms": 40.542915102560066, "device_peak_gib": 4.076587677001953, "device_min_gib": 4.076587677001953, "samples": 57, "added_device_peak_gib": 2.2022438049316406, "torch_peak_allocated_gib": 1.9050250053405762, "torch_peak_reserved_gib": 1.931640625, "warmup_errors": [], "finished": "2026-10-01T14:36:51.599632+00:00", "predictions_sha256": "16ace7c224b9550bd9c01bd8f1a5fed64db3cca2fd3113345532cd8a314e8aad", "timing_coverage": 1.0, "plottable": true, "config": "fp32" }, { "model": "gliner-decide", "name": "GLiNER Decide", "code": "G", "suite": "massive_en", "n": 2948, "correct": 2067, "accuracy": 0.7011533242876526, "interval": [ 0.6843762332852709, 0.717406862638121 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 24.885249906219542, "p95_ms": 34.968030045274645, "device_peak_gib": 4.045337677001953, "device_min_gib": 4.045337677001953, "samples": 53, "added_device_peak_gib": 2.1709938049316406, "torch_peak_allocated_gib": 1.882913589477539, "torch_peak_reserved_gib": 1.900390625, "warmup_errors": [], "finished": "2026-10-01T14:36:53.428814+00:00", "predictions_sha256": "801938e59115e87d5364f4c76cc192f94f400c9569bca5efbbe058950aa6f023", "timing_coverage": 1.0, "plottable": true, "config": "fp32" }, { "model": "gliner-decide", "name": "GLiNER Decide", "code": "G", "suite": "massive_nb", "n": 2948, "correct": 1557, "accuracy": 0.5281546811397557, "interval": [ 0.5101093203623563, 0.5461267622036494 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 25.397649966180325, "p95_ms": 35.03729508956894, "device_peak_gib": 4.050220489501953, "device_min_gib": 4.042896270751953, "samples": 53, "added_device_peak_gib": 2.1758766174316406, "torch_peak_allocated_gib": 1.8845558166503906, "torch_peak_reserved_gib": 1.900390625, "warmup_errors": [], "finished": "2026-10-01T14:36:55.310402+00:00", "predictions_sha256": "ddcdbbbf0ed869048c278bb9031057ed23fad08c04d1d9d0507442f80ccb4322", "timing_coverage": 1.0, "plottable": true, "config": "fp32" }, { "model": "gliner-decide", "name": "GLiNER Decide", "code": "G", "suite": "public_jevbench", "n": 231, "correct": 131, "accuracy": 0.5670995670995671, "interval": [ 0.5026223726319217, 0.6293815765671382 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 31.732449890114367, "p95_ms": 557.1949449484237, "device_peak_gib": 7.136157989501953, "device_min_gib": 4.066822052001953, "samples": 283, "added_device_peak_gib": 5.261814117431641, "torch_peak_allocated_gib": 4.580105304718018, "torch_peak_reserved_gib": 4.986328125, "warmup_errors": [], "finished": "2026-10-01T14:37:04.349099+00:00", "predictions_sha256": "ac8b0a83a64d3307fb9d2e0ab39e8b6119011aff1515ae3dd2433f174939e6eb", "timing_coverage": 1.0, "plottable": true, "config": "fp32" }, { "model": "gliner-decide", "name": "GLiNER Decide", "code": "G", "suite": "kev_claim/devtools-v1", "n": 900, "correct": 443, "accuracy": 0.4922222222222222, "interval": [ 0.4651800180892031, 0.5187178320024085 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 31.858449918217957, "p95_ms": 165.84324988070875, "device_peak_gib": 4.962329864501953, "device_min_gib": 4.215259552001953, "samples": 55, "added_device_peak_gib": 3.0879859924316406, "torch_peak_allocated_gib": 2.627049446105957, "torch_peak_reserved_gib": 2.8125, "warmup_errors": [], "finished": "2026-10-01T14:37:06.415037+00:00", "predictions_sha256": "7d4fa47494151ccbc9ed90f64069e2c2e70dca86eb55bbe7fba7a33365ade5de", "timing_coverage": 1.0, "plottable": true, "config": "fp32" }, { "model": "gliner-decide", "name": "GLiNER Decide", "code": "G", "suite": "industry/routing/en", "n": 40, "correct": 34, "accuracy": 0.85, "interval": [ 0.709276756335103, 0.9293881228267964 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 28.521950007416308, "p95_ms": 45.937499904539436, "device_peak_gib": 4.035572052001953, "device_min_gib": 4.035572052001953, "samples": 37, "added_device_peak_gib": 2.1612281799316406, "torch_peak_allocated_gib": 1.8693490028381348, "torch_peak_reserved_gib": 1.890625, "warmup_errors": [], "finished": "2026-10-01T14:37:07.754991+00:00", "predictions_sha256": "d957fc9aaae3786ab237acab4a1ed849cd6a6d7e6536ac4a6ca68b3e4b5dca5e", "timing_coverage": 1.0, "plottable": true, "config": "fp32" }, { "model": "gliner-decide", "name": "GLiNER Decide", "code": "G", "suite": "industry/routing/nb", "n": 40, "correct": 26, "accuracy": 0.65, "interval": [ 0.49505880837257693, 0.7786547112682372 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 27.789599960669875, "p95_ms": 37.35980007331818, "device_peak_gib": 4.037525177001953, "device_min_gib": 4.037525177001953, "samples": 36, "added_device_peak_gib": 2.1631813049316406, "torch_peak_allocated_gib": 1.8733868598937988, "torch_peak_reserved_gib": 1.892578125, "warmup_errors": [], "finished": "2026-10-01T14:37:09.058668+00:00", "predictions_sha256": "ff0e7f2ff48feb7076e69aba504874dfa362ef95fe7a02e2a23be356aeed05d9", "timing_coverage": 1.0, "plottable": true, "config": "fp32" }, { "model": "gliner-decide", "name": "GLiNER Decide", "code": "G", "suite": "industry/documents/en", "n": 28, "correct": 26, "accuracy": 0.9285714285714286, "interval": [ 0.7735463743160095, 0.9801879358846942 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 26.46600001025945, "p95_ms": 39.805600070394576, "device_peak_gib": 4.0359039306640625, "device_min_gib": 4.035572052001953, "samples": 35, "added_device_peak_gib": 2.16156005859375, "torch_peak_allocated_gib": 1.8705506324768066, "torch_peak_reserved_gib": 1.890625, "warmup_errors": [], "finished": "2026-10-01T14:37:10.339690+00:00", "predictions_sha256": "5ab3a3cdb423be48c1ba49cd4e915da4a63102a980b5b07f71b1f9629ecdbda9", "timing_coverage": 1.0, "plottable": true, "config": "fp32" }, { "model": "gliner-decide", "name": "GLiNER Decide", "code": "G", "suite": "industry/documents/nb", "n": 28, "correct": 22, "accuracy": 0.7857142857142857, "interval": [ 0.6046141434627812, 0.8978753966710211 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 26.8768499372527, "p95_ms": 37.86297491751611, "device_peak_gib": 4.0378570556640625, "device_min_gib": 4.0378570556640625, "samples": 35, "added_device_peak_gib": 2.16351318359375, "torch_peak_allocated_gib": 1.8737678527832031, "torch_peak_reserved_gib": 1.892578125, "warmup_errors": [], "finished": "2026-10-01T14:37:11.618941+00:00", "predictions_sha256": "b30090b0f65dd535cdad2bfee12f33d82eb539a2ea1293a60a304908855f01a2", "timing_coverage": 1.0, "plottable": true, "config": "fp32" }, { "model": "imajev-2b", "name": "Imajev 2B", "code": "I2", "suite": "fresh/news", "n": 200, "correct": 168, "accuracy": 0.84, "interval": [ 0.7828592983793289, 0.8843258796841297 ], "accuracy_failures": 4, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 66.52344996109605, "p95_ms": 77.5639700004831, "device_peak_gib": 6.5095367431640625, "device_min_gib": 6.5022125244140625, "samples": 133, "added_device_peak_gib": 4.634860992431641, "torch_peak_allocated_gib": 4.2539873123168945, "torch_peak_reserved_gib": 4.359375, "warmup_errors": [], "finished": "2026-10-01T14:37:29.660708+00:00", "predictions_sha256": "aaa34ed01da2941b6ed1d65648a0ba4a5232749853b1a962e6cd9ebce146950c", "timing_coverage": 1.0, "plottable": true, "config": "bf16 unmerged LoRA, fp32 trained decision head" }, { "model": "imajev-2b", "name": "Imajev 2B", "code": "I2", "suite": "fresh/emotion", "n": 200, "correct": 121, "accuracy": 0.605, "interval": [ 0.5358834915928536, 0.6701589898679203 ], "accuracy_failures": 9, "complete": true, "calls": 60, "unique_cases": 20, "answered": 54, "failures": 0, "abstentions": 6, "truncated_calls": 0, "median_ms": 67.95365002471954, "p95_ms": 84.18122010771185, "device_peak_gib": 6.4812164306640625, "device_min_gib": 6.4787750244140625, "samples": 136, "added_device_peak_gib": 4.606540679931641, "torch_peak_allocated_gib": 4.2493181228637695, "torch_peak_reserved_gib": 4.3359375, "warmup_errors": [], "finished": "2026-10-01T14:37:34.404160+00:00", "predictions_sha256": "232426e884f573ed930297f45a01db067e4997d30934eca3f6f3ab688cc08f39", "timing_coverage": 0.9, "plottable": true, "config": "bf16 unmerged LoRA, fp32 trained decision head" }, { "model": "imajev-2b", "name": "Imajev 2B", "code": "I2", "suite": "fresh/spam", "n": 200, "correct": 136, "accuracy": 0.68, "interval": [ 0.6124810156588646, 0.7407346668453195 ], "accuracy_failures": 15, "complete": true, "calls": 60, "unique_cases": 20, "answered": 51, "failures": 0, "abstentions": 9, "truncated_calls": 0, "median_ms": 66.55990006402135, "p95_ms": 73.38524993974715, "device_peak_gib": 6.5066070556640625, "device_min_gib": 6.5066070556640625, "samples": 131, "added_device_peak_gib": 4.631931304931641, "torch_peak_allocated_gib": 4.268219947814941, "torch_peak_reserved_gib": 4.361328125, "warmup_errors": [], "finished": "2026-10-01T14:37:39.029310+00:00", "predictions_sha256": "c275f565f47074b095cc6d8bae8afe7cb793053456dd5d00fdf797e0c982806e", "timing_coverage": 0.85, "plottable": false, "config": "bf16 unmerged LoRA, fp32 trained decision head" }, { "model": "imajev-2b", "name": "Imajev 2B", "code": "I2", "suite": "fresh/phishing", "n": 200, "correct": 144, "accuracy": 0.72, "interval": [ 0.6540769541334929, 0.7776311022605097 ], "accuracy_failures": 22, "complete": true, "calls": 60, "unique_cases": 20, "answered": 48, "failures": 0, "abstentions": 12, "truncated_calls": 0, "median_ms": 66.28854991868138, "p95_ms": 75.11169001227245, "device_peak_gib": 6.5114898681640625, "device_min_gib": 6.5041656494140625, "samples": 130, "added_device_peak_gib": 4.636814117431641, "torch_peak_allocated_gib": 4.263391494750977, "torch_peak_reserved_gib": 4.361328125, "warmup_errors": [], "finished": "2026-10-01T14:37:43.554809+00:00", "predictions_sha256": "63ec141855aed9a3356eba604910f36242d560ad905c9ea78770261c71657143", "timing_coverage": 0.8, "plottable": false, "config": "bf16 unmerged LoRA, fp32 trained decision head" }, { "model": "imajev-2b", "name": "Imajev 2B", "code": "I2", "suite": "massive_en", "n": 2948, "correct": 2439, "accuracy": 0.8273405698778833, "interval": [ 0.8132734334255909, 0.8405557192996589 ], "accuracy_failures": 62, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 69.13284992333502, "p95_ms": 82.6670249691233, "device_peak_gib": 6.7624969482421875, "device_min_gib": 6.7551727294921875, "samples": 137, "added_device_peak_gib": 4.636814117431641, "torch_peak_allocated_gib": 4.258622169494629, "torch_peak_reserved_gib": 4.361328125, "warmup_errors": [], "finished": "2026-10-04T21:18:56.395308+00:00", "predictions_sha256": "a27283aeeed872d8979ffa962eee1e63aef734f95e667ecca39b5b28adfedcef", "timing_coverage": 1.0, "plottable": true, "config": "bf16 unmerged LoRA, fp32 trained decision head" }, { "model": "imajev-2b", "name": "Imajev 2B", "code": "I2", "suite": "massive_nb", "n": 2948, "correct": 2156, "accuracy": 0.7313432835820896, "interval": [ 0.7150489207650665, 0.7470355167050065 ], "accuracy_failures": 37, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 70.33759995829314, "p95_ms": 87.0379150728695, "device_peak_gib": 6.7636566162109375, "device_min_gib": 6.7563323974609375, "samples": 143, "added_device_peak_gib": 4.637973785400391, "torch_peak_allocated_gib": 4.2627410888671875, "torch_peak_reserved_gib": 4.361328125, "warmup_errors": [], "finished": "2026-10-04T21:19:01.378430+00:00", "predictions_sha256": "7b2eeb1122e871ed388ca2cb90d1d63808368c6f476ece4a042d240b031f265b", "timing_coverage": 1.0, "plottable": true, "config": "bf16 unmerged LoRA, fp32 trained decision head" }, { "model": "imajev-2b", "name": "Imajev 2B", "code": "I2", "suite": "public_jevbench", "n": 231, "correct": 174, "accuracy": 0.7532467532467533, "interval": [ 0.6938095945211326, 0.8043988589075769 ], "accuracy_failures": 4, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 69.16804995853454, "p95_ms": 299.7804150218143, "device_peak_gib": 8.292953491210938, "device_min_gib": 6.7548675537109375, "samples": 234, "added_device_peak_gib": 6.167270660400391, "torch_peak_allocated_gib": 5.277629852294922, "torch_peak_reserved_gib": 5.890625, "warmup_errors": [], "finished": "2026-10-04T21:19:09.154940+00:00", "predictions_sha256": "caffbb5f022104a52f8de466aa284ee1d5d82f72f567bf7f4d8917bdc417ec51", "timing_coverage": 1.0, "plottable": true, "config": "bf16 unmerged LoRA, fp32 trained decision head" }, { "model": "imajev-2b", "name": "Imajev 2B", "code": "I2", "suite": "kev_claim/devtools-v1", "n": 900, "correct": 498, "accuracy": 0.5533333333333333, "interval": [ 0.5233292682926829, 0.5829113172081145 ], "accuracy_failures": 5, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 74.13485005963594, "p95_ms": 101.86019993852824, "device_peak_gib": 6.9316253662109375, "device_min_gib": 6.8564300537109375, "samples": 94, "added_device_peak_gib": 4.805942535400391, "torch_peak_allocated_gib": 4.380034923553467, "torch_peak_reserved_gib": 4.529296875, "warmup_errors": [], "finished": "2026-10-04T21:19:12.763466+00:00", "predictions_sha256": "693470e2a2077389361e12ee0015c9605b63e01186edded8e08da1ab9a4aa565", "timing_coverage": 1.0, "plottable": true, "config": "bf16 unmerged LoRA, fp32 trained decision head" }, { "model": "imajev-2b", "name": "Imajev 2B", "code": "I2", "suite": "industry/routing/en", "n": 40, "correct": 36, "accuracy": 0.9, "interval": [ 0.7694822477247771, 0.9604204713173936 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 67.474199924618, "p95_ms": 77.15064991498366, "device_peak_gib": 6.5095367431640625, "device_min_gib": 6.5022125244140625, "samples": 81, "added_device_peak_gib": 4.634860992431641, "torch_peak_allocated_gib": 4.252573013305664, "torch_peak_reserved_gib": 4.359375, "warmup_errors": [], "finished": "2026-10-01T14:37:46.612156+00:00", "predictions_sha256": "b36a813a3feabc1e1b1102caea54c959ba5b730aa06fa52d9c7f4711fada2535", "timing_coverage": 1.0, "plottable": true, "config": "bf16 unmerged LoRA, fp32 trained decision head" }, { "model": "imajev-2b", "name": "Imajev 2B", "code": "I2", "suite": "industry/routing/nb", "n": 40, "correct": 30, "accuracy": 0.75, "interval": [ 0.5980603857923197, 0.858128813609037 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 68.2902499102056, "p95_ms": 72.90177507093176, "device_peak_gib": 6.5046539306640625, "device_min_gib": 6.5046539306640625, "samples": 80, "added_device_peak_gib": 4.629978179931641, "torch_peak_allocated_gib": 4.253330230712891, "torch_peak_reserved_gib": 4.359375, "warmup_errors": [], "finished": "2026-10-01T14:37:49.618727+00:00", "predictions_sha256": "7a92755a1dac9b6daab236dcb91710074eafe7782ce63c0d7f070ad92f830812", "timing_coverage": 1.0, "plottable": true, "config": "bf16 unmerged LoRA, fp32 trained decision head" }, { "model": "imajev-2b", "name": "Imajev 2B", "code": "I2", "suite": "industry/documents/en", "n": 28, "correct": 28, "accuracy": 1.0, "interval": [ 0.8793566952341543, 0.9999999999999998 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 66.55955000314862, "p95_ms": 73.98604997433722, "device_peak_gib": 6.5095367431640625, "device_min_gib": 6.5022125244140625, "samples": 80, "added_device_peak_gib": 4.634860992431641, "torch_peak_allocated_gib": 4.254178047180176, "torch_peak_reserved_gib": 4.359375, "warmup_errors": [], "finished": "2026-10-01T14:37:52.644419+00:00", "predictions_sha256": "ce9c6512ecd3726fcb619f796186f3b78bb994f317cb636e62cdbbe0a4c7f1e4", "timing_coverage": 1.0, "plottable": true, "config": "bf16 unmerged LoRA, fp32 trained decision head" }, { "model": "imajev-2b", "name": "Imajev 2B", "code": "I2", "suite": "industry/documents/nb", "n": 28, "correct": 27, "accuracy": 0.9642857142857143, "interval": [ 0.8228780225664668, 0.9936674801509623 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 66.54490006621927, "p95_ms": 77.0815999712795, "device_peak_gib": 6.5046539306640625, "device_min_gib": 6.5046539306640625, "samples": 80, "added_device_peak_gib": 4.629978179931641, "torch_peak_allocated_gib": 4.254082679748535, "torch_peak_reserved_gib": 4.359375, "warmup_errors": [], "finished": "2026-10-01T14:37:55.661345+00:00", "predictions_sha256": "0a071f71b0d11aae4b729750dddbe793bac6614186a0219524eb4e5f29e7e854", "timing_coverage": 1.0, "plottable": true, "config": "bf16 unmerged LoRA, fp32 trained decision head" }, { "model": "imajev-4b", "name": "Imajev 4B", "code": "I4", "suite": "fresh/news", "n": 200, "correct": 168, "accuracy": 0.84, "interval": [ 0.7828592983793289, 0.8843258796841297 ], "accuracy_failures": 13, "complete": true, "calls": 60, "unique_cases": 20, "answered": 57, "failures": 0, "abstentions": 3, "truncated_calls": 0, "median_ms": 84.57010006532073, "p95_ms": 93.74341992661357, "device_peak_gib": 11.337661743164062, "device_min_gib": 11.330337524414062, "samples": 166, "added_device_peak_gib": 9.46298599243164, "torch_peak_allocated_gib": 9.13988733291626, "torch_peak_reserved_gib": 9.1875, "warmup_errors": [], "finished": "2026-10-01T14:38:20.718951+00:00", "predictions_sha256": "d62992d1f23421c4510bd1d8eb748e93ecaafe0096ec4786e5e0484e174c0f90", "timing_coverage": 0.95, "plottable": true, "config": "bf16 unmerged LoRA, fp32 trained decision head" }, { "model": "imajev-4b", "name": "Imajev 4B", "code": "I4", "suite": "fresh/emotion", "n": 200, "correct": 80, "accuracy": 0.4, "interval": [ 0.33460578566807336, 0.46916327960737997 ], "accuracy_failures": 60, "complete": true, "calls": 60, "unique_cases": 20, "answered": 36, "failures": 0, "abstentions": 24, "truncated_calls": 0, "median_ms": 85.0802999921143, "p95_ms": 98.40939991408959, "device_peak_gib": 11.331802368164062, "device_min_gib": 11.305435180664062, "samples": 169, "added_device_peak_gib": 9.45712661743164, "torch_peak_allocated_gib": 9.128344058990479, "torch_peak_reserved_gib": 9.181640625, "warmup_errors": [], "finished": "2026-10-01T14:38:26.651168+00:00", "predictions_sha256": "2974d50a72d8e4597a2e67a6551f31a2d480c1fba1f2a040adc86918544ddba1", "timing_coverage": 0.6, "plottable": false, "config": "bf16 unmerged LoRA, fp32 trained decision head" }, { "model": "imajev-4b", "name": "Imajev 4B", "code": "I4", "suite": "fresh/spam", "n": 200, "correct": 169, "accuracy": 0.845, "interval": [ 0.7883930940765792, 0.8886036307231068 ], "accuracy_failures": 6, "complete": true, "calls": 60, "unique_cases": 20, "answered": 57, "failures": 0, "abstentions": 3, "truncated_calls": 0, "median_ms": 82.66020007431507, "p95_ms": 97.14577989652751, "device_peak_gib": 11.359146118164062, "device_min_gib": 11.351821899414062, "samples": 165, "added_device_peak_gib": 9.48447036743164, "torch_peak_allocated_gib": 9.160549640655518, "torch_peak_reserved_gib": 9.208984375, "warmup_errors": [], "finished": "2026-10-01T14:38:32.458979+00:00", "predictions_sha256": "62f1602f50f1c95b11d6a72e8d67ffb4c3ab280c49ef67eac93f12363bdd3348", "timing_coverage": 0.95, "plottable": true, "config": "bf16 unmerged LoRA, fp32 trained decision head" }, { "model": "imajev-4b", "name": "Imajev 4B", "code": "I4", "suite": "fresh/phishing", "n": 200, "correct": 164, "accuracy": 0.82, "interval": [ 0.7608852497127511, 0.8670537414057983 ], "accuracy_failures": 5, "complete": true, "calls": 60, "unique_cases": 20, "answered": 54, "failures": 0, "abstentions": 6, "truncated_calls": 0, "median_ms": 83.36345001589507, "p95_ms": 97.10675512906163, "device_peak_gib": 11.359146118164062, "device_min_gib": 11.351821899414062, "samples": 165, "added_device_peak_gib": 9.48447036743164, "torch_peak_allocated_gib": 9.14341115951538, "torch_peak_reserved_gib": 9.208984375, "warmup_errors": [], "finished": "2026-10-01T14:38:38.229410+00:00", "predictions_sha256": "e4dd3d0bef1b56b064bf75fed51fd3c48e8dc63d3201f3b22efdc04575015d47", "timing_coverage": 0.9, "plottable": true, "config": "bf16 unmerged LoRA, fp32 trained decision head" }, { "model": "imajev-4b", "name": "Imajev 4B", "code": "I4", "suite": "massive_en", "n": 2948, "correct": 2524, "accuracy": 0.8561736770691994, "interval": [ 0.8430426106601469, 0.8683777109580008 ], "accuracy_failures": 154, "complete": true, "calls": 60, "unique_cases": 20, "answered": 57, "failures": 0, "abstentions": 3, "truncated_calls": 0, "median_ms": 87.71130000241101, "p95_ms": 97.19643983989954, "device_peak_gib": 11.620315551757812, "device_min_gib": 11.612991333007812, "samples": 173, "added_device_peak_gib": 9.48172378540039, "torch_peak_allocated_gib": 9.13877534866333, "torch_peak_reserved_gib": 9.205078125, "warmup_errors": [], "finished": "2026-10-04T21:56:18.045855+00:00", "predictions_sha256": "72274055ba70b4265df3b89b40e274e06695ecbe77026ffc98ae5bf47daa2d75", "timing_coverage": 0.95, "plottable": true, "config": "bf16 unmerged LoRA, fp32 trained decision head" }, { "model": "imajev-4b", "name": "Imajev 4B", "code": "I4", "suite": "massive_nb", "n": 2948, "correct": 2424, "accuracy": 0.8222523744911805, "interval": [ 0.8080353011224061, 0.835630704151156 ], "accuracy_failures": 135, "complete": true, "calls": 60, "unique_cases": 20, "answered": 57, "failures": 0, "abstentions": 3, "truncated_calls": 0, "median_ms": 87.50419993884861, "p95_ms": 100.37651993334293, "device_peak_gib": 11.624221801757812, "device_min_gib": 11.615432739257812, "samples": 172, "added_device_peak_gib": 9.48563003540039, "torch_peak_allocated_gib": 9.14278793334961, "torch_peak_reserved_gib": 9.208984375, "warmup_errors": [], "finished": "2026-10-04T21:56:24.106884+00:00", "predictions_sha256": "ed8faa88c43fa92ef61f0378fa3a3e82a93ecc926ed44a6219829e038e4af731", "timing_coverage": 0.95, "plottable": true, "config": "bf16 unmerged LoRA, fp32 trained decision head" }, { "model": "imajev-4b", "name": "Imajev 4B", "code": "I4", "suite": "public_jevbench", "n": 231, "correct": 198, "accuracy": 0.8571428571428571, "interval": [ 0.8061664873837949, 0.8964351777079748 ], "accuracy_failures": 1, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 85.4597999714315, "p95_ms": 707.2891849675214, "device_peak_gib": 14.796096801757812, "device_min_gib": 11.623245239257812, "samples": 438, "added_device_peak_gib": 12.65750503540039, "torch_peak_allocated_gib": 11.117136478424072, "torch_peak_reserved_gib": 12.380859375, "warmup_errors": [], "finished": "2026-10-04T21:56:38.503920+00:00", "predictions_sha256": "86072a36567f1995b2624013e0242f6fadfc235e49c87c6be663c41a16ea1d7d", "timing_coverage": 1.0, "plottable": true, "config": "bf16 unmerged LoRA, fp32 trained decision head" }, { "model": "imajev-4b", "name": "Imajev 4B", "code": "I4", "suite": "kev_claim/devtools-v1", "n": 900, "correct": 553, "accuracy": 0.6144444444444445, "interval": [ 0.575903113493475, 0.6601144049307298 ], "accuracy_failures": 37, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 86.6856500506401, "p95_ms": 218.02327508339658, "device_peak_gib": 12.059768676757812, "device_min_gib": 11.923538208007812, "samples": 130, "added_device_peak_gib": 9.92117691040039, "torch_peak_allocated_gib": 9.370636940002441, "torch_peak_reserved_gib": 9.64453125, "warmup_errors": [], "finished": "2026-10-04T21:56:43.521195+00:00", "predictions_sha256": "d2d3d91e139ce47406205d198f3e056f5e2f7050cddcb77ea9a3e8063a8caf7e", "timing_coverage": 1.0, "plottable": true, "config": "bf16 unmerged LoRA, fp32 trained decision head" }, { "model": "imajev-4b", "name": "Imajev 4B", "code": "I4", "suite": "industry/routing/en", "n": 40, "correct": 38, "accuracy": 0.95, "interval": [ 0.8349612263085903, 0.9861793326138516 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 83.08640005998313, "p95_ms": 93.61492493189871, "device_peak_gib": 11.332778930664062, "device_min_gib": 11.332778930664062, "samples": 100, "added_device_peak_gib": 9.45810317993164, "torch_peak_allocated_gib": 9.143192291259766, "torch_peak_reserved_gib": 9.1875, "warmup_errors": [], "finished": "2026-10-01T14:38:42.038249+00:00", "predictions_sha256": "494c06082337da6794f45096dbe76d0827e29356fd804c3c7c9560c6f288644b", "timing_coverage": 1.0, "plottable": true, "config": "bf16 unmerged LoRA, fp32 trained decision head" }, { "model": "imajev-4b", "name": "Imajev 4B", "code": "I4", "suite": "industry/routing/nb", "n": 40, "correct": 37, "accuracy": 0.925, "interval": [ 0.8013576647568946, 0.9741639742254119 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 84.98374989721924, "p95_ms": 94.23887502634898, "device_peak_gib": 11.337661743164062, "device_min_gib": 11.330337524414062, "samples": 100, "added_device_peak_gib": 9.46298599243164, "torch_peak_allocated_gib": 9.128826141357422, "torch_peak_reserved_gib": 9.1875, "warmup_errors": [], "finished": "2026-10-01T14:38:45.798040+00:00", "predictions_sha256": "727498eb8bd64b167da7aa2ea6539a578eef560aa919b9269df9b7f878ae52f6", "timing_coverage": 1.0, "plottable": true, "config": "bf16 unmerged LoRA, fp32 trained decision head" }, { "model": "imajev-4b", "name": "Imajev 4B", "code": "I4", "suite": "industry/documents/en", "n": 28, "correct": 28, "accuracy": 1.0, "interval": [ 0.8793566952341543, 0.9999999999999998 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 83.1610499881208, "p95_ms": 91.84762503718957, "device_peak_gib": 11.337661743164062, "device_min_gib": 11.330337524414062, "samples": 99, "added_device_peak_gib": 9.46298599243164, "torch_peak_allocated_gib": 9.143192291259766, "torch_peak_reserved_gib": 9.1875, "warmup_errors": [], "finished": "2026-10-01T14:38:49.572857+00:00", "predictions_sha256": "935d977925ce0876afd90d74e2e67026645051f60d643126343af2b5a11ffe13", "timing_coverage": 1.0, "plottable": true, "config": "bf16 unmerged LoRA, fp32 trained decision head" }, { "model": "imajev-4b", "name": "Imajev 4B", "code": "I4", "suite": "industry/documents/nb", "n": 28, "correct": 28, "accuracy": 1.0, "interval": [ 0.8793566952341543, 0.9999999999999998 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 83.84030009619892, "p95_ms": 92.10767492186278, "device_peak_gib": 11.332778930664062, "device_min_gib": 11.332778930664062, "samples": 99, "added_device_peak_gib": 9.45810317993164, "torch_peak_allocated_gib": 9.131089687347412, "torch_peak_reserved_gib": 9.1875, "warmup_errors": [], "finished": "2026-10-01T14:38:53.321922+00:00", "predictions_sha256": "d3f89a3435398e3a0e0b3d2eb75a32a8c1361972977b16f5ff1758a7f6a50439", "timing_coverage": 1.0, "plottable": true, "config": "bf16 unmerged LoRA, fp32 trained decision head" }, { "model": "intern-decision-4b", "name": "Intern-Decision 4B", "code": "IN", "suite": "fresh/news", "n": 200, "correct": 176, "accuracy": 0.88, "interval": [ 0.8276574790170326, 0.9180200729362449 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 57.02385003678501, "p95_ms": 70.2761449269019, "device_peak_gib": 10.812271118164062, "device_min_gib": 10.804946899414062, "samples": 114, "added_device_peak_gib": 8.93759536743164, "torch_peak_allocated_gib": 8.623971939086914, "torch_peak_reserved_gib": 8.662109375, "warmup_errors": [], "finished": "2026-10-01T14:39:14.579471+00:00", "predictions_sha256": "d61c077a69514c8c9524c0d6fe50e1c66d907117b6e60fa329fb6cb8a376e66e", "timing_coverage": 1.0, "plottable": true, "config": "bf16" }, { "model": "intern-decision-4b", "name": "Intern-Decision 4B", "code": "IN", "suite": "fresh/emotion", "n": 200, "correct": 96, "accuracy": 0.48, "interval": [ 0.411791663783233, 0.5489621492718576 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 56.09824997372925, "p95_ms": 65.5589401256293, "device_peak_gib": 10.792739868164062, "device_min_gib": 10.768325805664062, "samples": 112, "added_device_peak_gib": 8.91806411743164, "torch_peak_allocated_gib": 8.619272708892822, "torch_peak_reserved_gib": 8.642578125, "warmup_errors": [], "finished": "2026-10-01T14:39:18.558380+00:00", "predictions_sha256": "84cbd104031d1e154896ac92ac902523fde603625a4ed6d6438297ffbdc6fbb8", "timing_coverage": 1.0, "plottable": true, "config": "bf16" }, { "model": "intern-decision-4b", "name": "Intern-Decision 4B", "code": "IN", "suite": "fresh/spam", "n": 200, "correct": 188, "accuracy": 0.94, "interval": [ 0.8980683070423372, 0.9653478057456681 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 62.741550034843385, "p95_ms": 71.3426849921234, "device_peak_gib": 10.843521118164062, "device_min_gib": 10.826919555664062, "samples": 122, "added_device_peak_gib": 8.96884536743164, "torch_peak_allocated_gib": 8.657200813293457, "torch_peak_reserved_gib": 8.693359375, "warmup_errors": [], "finished": "2026-10-01T14:39:22.818144+00:00", "predictions_sha256": "e071009d85ba6c1a639f4935a886bcb08ab7f8f89183198d0560e7d55c92454d", "timing_coverage": 1.0, "plottable": true, "config": "bf16" }, { "model": "intern-decision-4b", "name": "Intern-Decision 4B", "code": "IN", "suite": "fresh/phishing", "n": 200, "correct": 183, "accuracy": 0.915, "interval": [ 0.8681041292843443, 0.9462542498225244 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 60.37414993625134, "p95_ms": 67.24108484340832, "device_peak_gib": 10.826919555664062, "device_min_gib": 10.826919555664062, "samples": 117, "added_device_peak_gib": 8.95224380493164, "torch_peak_allocated_gib": 8.634562015533447, "torch_peak_reserved_gib": 8.681640625, "warmup_errors": [], "finished": "2026-10-01T14:39:26.976340+00:00", "predictions_sha256": "0c0af6f52ce5cc1990c011abfe23840a2c1655ce389a0c5888a7bc34af65ac74", "timing_coverage": 1.0, "plottable": true, "config": "bf16" }, { "model": "intern-decision-4b", "name": "Intern-Decision 4B", "code": "IN", "suite": "massive_en", "n": 2948, "correct": 2448, "accuracy": 0.830393487109905, "interval": [ 0.8164183736355343, 0.8435086675607283 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 62.97520000953227, "p95_ms": 70.52199518075213, "device_peak_gib": 11.100173950195312, "device_min_gib": 11.092849731445312, "samples": 122, "added_device_peak_gib": 8.95517349243164, "torch_peak_allocated_gib": 8.634302139282227, "torch_peak_reserved_gib": 8.6796875, "warmup_errors": [], "finished": "2026-10-04T22:29:18.405478+00:00", "predictions_sha256": "0cf89584bc604d096f0c8ed55baa271a080dea4f20a2a65263552462ce66a5e7", "timing_coverage": 1.0, "plottable": true, "config": "bf16" }, { "model": "intern-decision-4b", "name": "Intern-Decision 4B", "code": "IN", "suite": "massive_nb", "n": 2948, "correct": 2126, "accuracy": 0.7211668928086838, "interval": [ 0.7046997474720419, 0.7370583950946553 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 63.210499938577414, "p95_ms": 73.11247497564182, "device_peak_gib": 11.100173950195312, "device_min_gib": 11.092849731445312, "samples": 124, "added_device_peak_gib": 8.95517349243164, "torch_peak_allocated_gib": 8.634562015533447, "torch_peak_reserved_gib": 8.6796875, "warmup_errors": [], "finished": "2026-10-04T22:29:22.753832+00:00", "predictions_sha256": "cbf1ff4e113bb7ba4c006853b5a00d2529236c2922455bf07a98ce0ebd1263ef", "timing_coverage": 1.0, "plottable": true, "config": "bf16" }, { "model": "intern-decision-4b", "name": "Intern-Decision 4B", "code": "IN", "suite": "public_jevbench", "n": 231, "correct": 201, "accuracy": 0.8701298701298701, "interval": [ 0.8206572491001091, 0.9074935674495431 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 64.5133500220254, "p95_ms": 542.322139837779, "device_peak_gib": 14.330642700195312, "device_min_gib": 11.097244262695312, "samples": 334, "added_device_peak_gib": 12.18564224243164, "torch_peak_allocated_gib": 10.57463788986206, "torch_peak_reserved_gib": 11.91015625, "warmup_errors": [], "finished": "2026-10-04T22:29:33.598140+00:00", "predictions_sha256": "b98217c505711df8f7b9ec8f755deb3a99e4e501bed2646e519283a3af599d75", "timing_coverage": 1.0, "plottable": true, "config": "bf16" }, { "model": "intern-decision-4b", "name": "Intern-Decision 4B", "code": "IN", "suite": "kev_claim/devtools-v1", "n": 900, "correct": 557, "accuracy": 0.6188888888888889, "interval": [ 0.5837645734672641, 0.6561690492891651 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 63.69594985153526, "p95_ms": 158.83695002412423, "device_peak_gib": 11.396072387695312, "device_min_gib": 11.210525512695312, "samples": 87, "added_device_peak_gib": 9.25107192993164, "torch_peak_allocated_gib": 8.863449096679688, "torch_peak_reserved_gib": 8.98046875, "warmup_errors": [], "finished": "2026-10-04T22:29:36.868686+00:00", "predictions_sha256": "5370f5a3c588cd8a1ff68cacf1a4ac7b850eb01e8ec020f737596896e909099e", "timing_coverage": 1.0, "plottable": true, "config": "bf16" }, { "model": "intern-decision-4b", "name": "Intern-Decision 4B", "code": "IN", "suite": "industry/routing/en", "n": 40, "correct": 39, "accuracy": 0.975, "interval": [ 0.8711863103652591, 0.9955731684973186 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 56.76544993184507, "p95_ms": 65.50987501395866, "device_peak_gib": 10.792739868164062, "device_min_gib": 10.785415649414062, "samples": 68, "added_device_peak_gib": 8.91806411743164, "torch_peak_allocated_gib": 8.621359825134277, "torch_peak_reserved_gib": 8.642578125, "warmup_errors": [], "finished": "2026-10-01T14:39:29.502532+00:00", "predictions_sha256": "8c780c043caf5011f5a94bb2d875121d3c923af76925a2b5a35e3efb6a5cfb26", "timing_coverage": 1.0, "plottable": true, "config": "bf16" }, { "model": "intern-decision-4b", "name": "Intern-Decision 4B", "code": "IN", "suite": "industry/routing/nb", "n": 40, "correct": 39, "accuracy": 0.975, "interval": [ 0.8711863103652591, 0.9955731684973186 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 61.044399975799024, "p95_ms": 71.95949991000816, "device_peak_gib": 10.807388305664062, "device_min_gib": 10.807388305664062, "samples": 73, "added_device_peak_gib": 8.93271255493164, "torch_peak_allocated_gib": 8.623281955718994, "torch_peak_reserved_gib": 8.662109375, "warmup_errors": [], "finished": "2026-10-01T14:39:32.235471+00:00", "predictions_sha256": "1a912ae6cc0c288935e6b5310d48af5b0734ebe291567ddf3d15d85ad09adb00", "timing_coverage": 1.0, "plottable": true, "config": "bf16" }, { "model": "intern-decision-4b", "name": "Intern-Decision 4B", "code": "IN", "suite": "industry/documents/en", "n": 28, "correct": 28, "accuracy": 1.0, "interval": [ 0.8793566952341543, 0.9999999999999998 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 57.381800026632845, "p95_ms": 63.758724951185286, "device_peak_gib": 10.812271118164062, "device_min_gib": 10.804946899414062, "samples": 68, "added_device_peak_gib": 8.93759536743164, "torch_peak_allocated_gib": 8.623740673065186, "torch_peak_reserved_gib": 8.662109375, "warmup_errors": [], "finished": "2026-10-01T14:39:34.799454+00:00", "predictions_sha256": "c31b6f6e1595b62c23fc430981837f4721936ef8f95b0dfc4fae66591c5d0267", "timing_coverage": 1.0, "plottable": true, "config": "bf16" }, { "model": "intern-decision-4b", "name": "Intern-Decision 4B", "code": "IN", "suite": "industry/documents/nb", "n": 28, "correct": 28, "accuracy": 1.0, "interval": [ 0.8793566952341543, 0.9999999999999998 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 61.15190009586513, "p95_ms": 71.25247514341027, "device_peak_gib": 10.805435180664062, "device_min_gib": 10.805435180664062, "samples": 74, "added_device_peak_gib": 8.93075942993164, "torch_peak_allocated_gib": 8.623626232147217, "torch_peak_reserved_gib": 8.66015625, "warmup_errors": [], "finished": "2026-10-01T14:39:37.555774+00:00", "predictions_sha256": "aa57d9ceaa4183def786b002135429135f5d5f3c99bcfa62125edf2d1b4815b1", "timing_coverage": 1.0, "plottable": true, "config": "bf16" }, { "model": "jev-omni-12b-nf4", "name": "Jev-Omni 12B NF4", "code": "OM", "suite": "fresh/news", "n": 200, "correct": 170, "accuracy": 0.85, "interval": [ 0.7939442071583334, 0.89286406437758 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 139.37650003936142, "p95_ms": 150.245904980693, "device_peak_gib": 9.566730499267578, "device_min_gib": 9.559406280517578, "samples": 267, "added_device_peak_gib": 7.705173492431641, "torch_peak_allocated_gib": 7.314614772796631, "torch_peak_reserved_gib": 7.4296875, "warmup_errors": [], "finished": "2026-10-01T14:28:56.606848+00:00", "predictions_sha256": "00ece8fe1a8d5761672d844bcde3c1940f3226194e0f951ce7170c73b8a70023", "timing_coverage": 1.0, "plottable": true, "config": "NF4 backbone; publisher fp32 classifier head" }, { "model": "jev-omni-12b-nf4", "name": "Jev-Omni 12B NF4", "code": "OM", "suite": "fresh/emotion", "n": 200, "correct": 106, "accuracy": 0.53, "interval": [ 0.4609168291500183, 0.5979524512673458 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 132.34234997071326, "p95_ms": 143.99043500889093, "device_peak_gib": 9.566730499267578, "device_min_gib": 9.559406280517578, "samples": 260, "added_device_peak_gib": 7.705173492431641, "torch_peak_allocated_gib": 7.30839204788208, "torch_peak_reserved_gib": 7.4296875, "warmup_errors": [], "finished": "2026-10-01T14:29:05.754621+00:00", "predictions_sha256": "248aeb61bf96724bd5f517b6bc168bea803d8245027d64b9a9af3ed10e55fcc6", "timing_coverage": 1.0, "plottable": true, "config": "NF4 backbone; publisher fp32 classifier head" }, { "model": "jev-omni-12b-nf4", "name": "Jev-Omni 12B NF4", "code": "OM", "suite": "fresh/spam", "n": 200, "correct": 193, "accuracy": 0.965, "interval": [ 0.9295293884777092, 0.9829444579914328 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 140.62524994369596, "p95_ms": 155.03469501854838, "device_peak_gib": 9.566730499267578, "device_min_gib": 9.559406280517578, "samples": 275, "added_device_peak_gib": 7.705173492431641, "torch_peak_allocated_gib": 7.3265862464904785, "torch_peak_reserved_gib": 7.4296875, "warmup_errors": [], "finished": "2026-10-01T14:29:15.441211+00:00", "predictions_sha256": "49f05e56d2765c62260ebcf3b30148e3271f27b240d77402e2a0ced81207ce33", "timing_coverage": 1.0, "plottable": true, "config": "NF4 backbone; publisher fp32 classifier head" }, { "model": "jev-omni-12b-nf4", "name": "Jev-Omni 12B NF4", "code": "OM", "suite": "fresh/phishing", "n": 200, "correct": 176, "accuracy": 0.88, "interval": [ 0.8276574790170326, 0.9180200729362449 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 140.12174983508885, "p95_ms": 151.0261499788612, "device_peak_gib": 9.566730499267578, "device_min_gib": 9.559406280517578, "samples": 271, "added_device_peak_gib": 7.705173492431641, "torch_peak_allocated_gib": 7.321145534515381, "torch_peak_reserved_gib": 7.4296875, "warmup_errors": [], "finished": "2026-10-01T14:29:25.014450+00:00", "predictions_sha256": "9cfd6d8a5bd334fe7237d63bd3fc47f2bb1d37029af11076ef6b08a84d4b8c31", "timing_coverage": 1.0, "plottable": true, "config": "NF4 backbone; publisher fp32 classifier head" }, { "model": "jev-omni-12b-nf4", "name": "Jev-Omni 12B NF4", "code": "OM", "suite": "massive_en", "n": 2948, "correct": 2365, "accuracy": 0.8022388059701493, "interval": [ 0.7874711828979016, 0.8162197757323552 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 140.15960006508976, "p95_ms": 153.37792002828792, "device_peak_gib": 9.895248413085938, "device_min_gib": 9.887924194335938, "samples": 279, "added_device_peak_gib": 7.701267242431641, "torch_peak_allocated_gib": 7.317409992218018, "torch_peak_reserved_gib": 7.42578125, "warmup_errors": [], "finished": "2026-10-04T23:48:18.499601+00:00", "predictions_sha256": "67c5cd76385f89d98da9a662cac5bd45bddb2f0be0fbfb798b36752959e5b4b9", "timing_coverage": 1.0, "plottable": true, "config": "NF4 backbone; publisher fp32 classifier head" }, { "model": "jev-omni-12b-nf4", "name": "Jev-Omni 12B NF4", "code": "OM", "suite": "massive_nb", "n": 2948, "correct": 2055, "accuracy": 0.6970827679782904, "interval": [ 0.6802473125057262, 0.7134052654563154 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 139.8786500794813, "p95_ms": 153.03791002370417, "device_peak_gib": 9.895248413085938, "device_min_gib": 9.887924194335938, "samples": 277, "added_device_peak_gib": 7.701267242431641, "torch_peak_allocated_gib": 7.317694187164307, "torch_peak_reserved_gib": 7.42578125, "warmup_errors": [], "finished": "2026-10-04T23:48:28.210494+00:00", "predictions_sha256": "0cf974231f84be9d478652fdcc7aa9f897751d962a0388da12bc1feb4d36801e", "timing_coverage": 1.0, "plottable": true, "config": "NF4 backbone; publisher fp32 classifier head" }, { "model": "jev-omni-12b-nf4", "name": "Jev-Omni 12B NF4", "code": "OM", "suite": "public_jevbench", "n": 231, "correct": 198, "accuracy": 0.8571428571428571, "interval": [ 0.8061664873837949, 0.8964351777079748 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 171.95274983532727, "p95_ms": 1232.7670450555156, "device_peak_gib": 10.033527374267578, "device_min_gib": 9.559406280517578, "samples": 790, "added_device_peak_gib": 8.17197036743164, "torch_peak_allocated_gib": 7.636138916015625, "torch_peak_reserved_gib": 7.896484375, "warmup_errors": [], "finished": "2026-10-01T14:29:50.861377+00:00", "predictions_sha256": "de25afdff8ae670194df9cd0ea2399659a2c66ff4a07e08f682fac4aefb8758e", "timing_coverage": 1.0, "plottable": true, "config": "NF4 backbone; publisher fp32 classifier head" }, { "model": "jev-omni-12b-nf4", "name": "Jev-Omni 12B NF4", "code": "OM", "suite": "kev_claim/devtools-v1", "n": 900, "correct": 573, "accuracy": 0.6366666666666667, "interval": [ 0.5948990182670704, 0.6743953013420767 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 169.27900002337992, "p95_ms": 545.4270000918768, "device_peak_gib": 10.012435913085938, "device_min_gib": 10.001205444335938, "samples": 258, "added_device_peak_gib": 7.818454742431641, "torch_peak_allocated_gib": 7.397754192352295, "torch_peak_reserved_gib": 7.54296875, "warmup_errors": [], "finished": "2026-10-04T23:48:38.178029+00:00", "predictions_sha256": "4522fd6e3175545cebb98aa40d0163db04fd8afeb537eb18bb226f3327ae4b88", "timing_coverage": 1.0, "plottable": true, "config": "NF4 backbone; publisher fp32 classifier head" }, { "model": "jev-omni-12b-nf4", "name": "Jev-Omni 12B NF4", "code": "OM", "suite": "industry/routing/en", "n": 40, "correct": 40, "accuracy": 1.0, "interval": [ 0.9123783988027135, 1.0 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 137.119049904868, "p95_ms": 148.39210006175563, "device_peak_gib": 9.895248413085938, "device_min_gib": 9.887924194335938, "samples": 162, "added_device_peak_gib": 7.701267242431641, "torch_peak_allocated_gib": 7.314178943634033, "torch_peak_reserved_gib": 7.42578125, "warmup_errors": [], "finished": "2026-10-04T23:48:44.333624+00:00", "predictions_sha256": "40065273b1108369be9f65ebe16202e2cad75d152be8b1178329cf5c9b41ef94", "timing_coverage": 1.0, "plottable": true, "config": "NF4 backbone; publisher fp32 classifier head" }, { "model": "jev-omni-12b-nf4", "name": "Jev-Omni 12B NF4", "code": "OM", "suite": "industry/routing/nb", "n": 40, "correct": 39, "accuracy": 0.975, "interval": [ 0.8711863103652591, 0.9955731684973186 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 137.48240005224943, "p95_ms": 148.3001500600949, "device_peak_gib": 9.895248413085938, "device_min_gib": 9.887924194335938, "samples": 164, "added_device_peak_gib": 7.701267242431641, "torch_peak_allocated_gib": 7.314559459686279, "torch_peak_reserved_gib": 7.42578125, "warmup_errors": [], "finished": "2026-10-04T23:48:50.505798+00:00", "predictions_sha256": "dbba0bcf8346a2029454d5d2ee58fb5845ac21e4375fd4da9ed619f36082b4b3", "timing_coverage": 1.0, "plottable": true, "config": "NF4 backbone; publisher fp32 classifier head" }, { "model": "jev-omni-12b-nf4", "name": "Jev-Omni 12B NF4", "code": "OM", "suite": "industry/documents/en", "n": 28, "correct": 28, "accuracy": 1.0, "interval": [ 0.8793566952341543, 0.9999999999999998 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 136.59115007612854, "p95_ms": 148.090174945537, "device_peak_gib": 9.895248413085938, "device_min_gib": 9.887924194335938, "samples": 161, "added_device_peak_gib": 7.701267242431641, "torch_peak_allocated_gib": 7.315371990203857, "torch_peak_reserved_gib": 7.42578125, "warmup_errors": [], "finished": "2026-10-04T23:48:56.626161+00:00", "predictions_sha256": "533fef18b174360ebb2b03abb83432bb90bcd84cfc59c73e5075480bc233aeb5", "timing_coverage": 1.0, "plottable": true, "config": "NF4 backbone; publisher fp32 classifier head" }, { "model": "jev-omni-12b-nf4", "name": "Jev-Omni 12B NF4", "code": "OM", "suite": "industry/documents/nb", "n": 28, "correct": 26, "accuracy": 0.9285714285714286, "interval": [ 0.7735463743160095, 0.9801879358846942 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 137.11184996645898, "p95_ms": 149.24319996498525, "device_peak_gib": 9.896408081054688, "device_min_gib": 9.889083862304688, "samples": 162, "added_device_peak_gib": 7.702426910400391, "torch_peak_allocated_gib": 7.3154425621032715, "torch_peak_reserved_gib": 7.42578125, "warmup_errors": [], "finished": "2026-10-04T23:49:02.749056+00:00", "predictions_sha256": "5351cfba0b6d6f96637cb9b370f36ef260e9654f0bdba0d2cf292cef3594d88a", "timing_coverage": 1.0, "plottable": true, "config": "NF4 backbone; publisher fp32 classifier head" }, { "model": "jevk5-2b", "name": "JevK5 2B", "code": "J52", "suite": "fresh/news", "n": 200, "correct": 170, "accuracy": 0.85, "interval": [ 0.7939442071583334, 0.89286406437758 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 42.68015013076365, "p95_ms": 51.56751989852637, "device_peak_gib": 5.7243804931640625, "device_min_gib": 5.7170562744140625, "samples": 85, "added_device_peak_gib": 3.8497047424316406, "torch_peak_allocated_gib": 3.554107666015625, "torch_peak_reserved_gib": 3.57421875, "warmup_errors": [], "finished": "2026-10-01T14:39:51.251222+00:00", "predictions_sha256": "a08fa0a486b6569cc67225784cb00d638f4fae0f0e4c73e5b58c7c0c6b5a7f96", "timing_coverage": 1.0, "plottable": true, "config": "bf16" }, { "model": "jevk5-2b", "name": "JevK5 2B", "code": "J52", "suite": "fresh/emotion", "n": 200, "correct": 124, "accuracy": 0.62, "interval": [ 0.5510671801561923, 0.6844099415132637 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 42.00350004248321, "p95_ms": 51.79823515936732, "device_peak_gib": 5.7194366455078125, "device_min_gib": 5.7194366455078125, "samples": 85, "added_device_peak_gib": 3.8447608947753906, "torch_peak_allocated_gib": 3.5525684356689453, "torch_peak_reserved_gib": 3.57421875, "warmup_errors": [], "finished": "2026-10-01T14:39:54.256581+00:00", "predictions_sha256": "fbfff59c7b2e4d1d148d51174548e6a9edac06f9dd54c21e53785f837ea52e34", "timing_coverage": 1.0, "plottable": true, "config": "bf16" }, { "model": "jevk5-2b", "name": "JevK5 2B", "code": "J52", "suite": "fresh/spam", "n": 200, "correct": 141, "accuracy": 0.705, "interval": [ 0.6384128547538122, 0.7638605614315084 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 42.48359997291118, "p95_ms": 52.51148983370512, "device_peak_gib": 5.7438507080078125, "device_min_gib": 5.7194366455078125, "samples": 85, "added_device_peak_gib": 3.8691749572753906, "torch_peak_allocated_gib": 3.5656485557556152, "torch_peak_reserved_gib": 3.59375, "warmup_errors": [], "finished": "2026-10-01T14:39:57.249209+00:00", "predictions_sha256": "6f3a070a46fd0f8695b3eeed6daa730e96e7ecf7939ce5f07e9191a1a2f7dd44", "timing_coverage": 1.0, "plottable": true, "config": "bf16" }, { "model": "jevk5-2b", "name": "JevK5 2B", "code": "J52", "suite": "fresh/phishing", "n": 200, "correct": 147, "accuracy": 0.735, "interval": [ 0.6698242380737758, 0.7913184585289088 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 41.630350169725716, "p95_ms": 51.364615024067454, "device_peak_gib": 5.7243194580078125, "device_min_gib": 5.7169952392578125, "samples": 84, "added_device_peak_gib": 3.8496437072753906, "torch_peak_allocated_gib": 3.558732032775879, "torch_peak_reserved_gib": 3.57421875, "warmup_errors": [], "finished": "2026-10-01T14:40:00.142497+00:00", "predictions_sha256": "6106e6c092a42a263aa77d92683e166be77baf46181e5e0bb91f3838d9b14f70", "timing_coverage": 1.0, "plottable": true, "config": "bf16" }, { "model": "jevk5-2b", "name": "JevK5 2B", "code": "J52", "suite": "massive_en", "n": 2948, "correct": 2089, "accuracy": 0.7086160108548168, "interval": [ 0.6919499745020325, 0.7247390710171682 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 129.44994994904846, "p95_ms": 142.53068496473134, "device_peak_gib": 5.7438507080078125, "device_min_gib": 5.7365264892578125, "samples": 251, "added_device_peak_gib": 3.8691749572753906, "torch_peak_allocated_gib": 3.5655460357666016, "torch_peak_reserved_gib": 3.59375, "warmup_errors": [], "finished": "2026-10-01T14:40:08.988427+00:00", "predictions_sha256": "681ea83b826f04a26ae6d16e8acc36fed9edd53ef37d37e87df1ce2f1cf8d31b", "timing_coverage": 1.0, "plottable": true, "config": "bf16" }, { "model": "jevk5-2b", "name": "JevK5 2B", "code": "J52", "suite": "massive_nb", "n": 2948, "correct": 1682, "accuracy": 0.5705563093622795, "interval": [ 0.5526074401050074, 0.5883215378917304 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 127.99960002303123, "p95_ms": 136.38320001773536, "device_peak_gib": 5.7438507080078125, "device_min_gib": 5.7365264892578125, "samples": 248, "added_device_peak_gib": 3.8691749572753906, "torch_peak_allocated_gib": 3.5656485557556152, "torch_peak_reserved_gib": 3.59375, "warmup_errors": [], "finished": "2026-10-01T14:40:17.753704+00:00", "predictions_sha256": "f5fa9fff191ef752dcd2c61a30a4175e97ea8f70c4622d037d02622c91a59dba", "timing_coverage": 1.0, "plottable": true, "config": "bf16" }, { "model": "jevk5-2b", "name": "JevK5 2B", "code": "J52", "suite": "public_jevbench", "n": 231, "correct": 173, "accuracy": 0.7489177489177489, "interval": [ 0.6892360162608874, 0.8004560533485278 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 43.41090016532689, "p95_ms": 214.95023980969552, "device_peak_gib": 7.2907257080078125, "device_min_gib": 5.7389678955078125, "samples": 158, "added_device_peak_gib": 5.416049957275391, "torch_peak_allocated_gib": 4.537691116333008, "torch_peak_reserved_gib": 5.140625, "warmup_errors": [], "finished": "2026-10-01T14:40:23.016028+00:00", "predictions_sha256": "7bd0d1a375c6e316d3d132ab33f267ea434f912862044cccb50a2bb57fc76788", "timing_coverage": 1.0, "plottable": true, "config": "bf16" }, { "model": "jevk5-2b", "name": "JevK5 2B", "code": "J52", "suite": "kev_claim/devtools-v1", "n": 900, "correct": 412, "accuracy": 0.4577777777777778, "interval": [ 0.4263959390862944, 0.48585707552727697 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 42.90474997833371, "p95_ms": 72.98369996715337, "device_peak_gib": 5.9108428955078125, "device_min_gib": 5.8444366455078125, "samples": 57, "added_device_peak_gib": 4.036167144775391, "torch_peak_allocated_gib": 3.6713929176330566, "torch_peak_reserved_gib": 3.765625, "warmup_errors": [], "finished": "2026-10-01T14:40:25.258614+00:00", "predictions_sha256": "6ea68f602bd3a30daa5da76963d1ef41af2f972bff38dec84202e5359fa2ec7f", "timing_coverage": 1.0, "plottable": true, "config": "bf16" }, { "model": "jevk5-2b", "name": "JevK5 2B", "code": "J52", "suite": "industry/routing/en", "n": 40, "correct": 37, "accuracy": 0.925, "interval": [ 0.8013576647568946, 0.9741639742254119 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 41.192600154317915, "p95_ms": 48.39229985373095, "device_peak_gib": 5.7243194580078125, "device_min_gib": 5.7169952392578125, "samples": 50, "added_device_peak_gib": 3.8496437072753906, "torch_peak_allocated_gib": 3.5539064407348633, "torch_peak_reserved_gib": 3.57421875, "warmup_errors": [], "finished": "2026-10-01T14:40:27.172123+00:00", "predictions_sha256": "edae14777089d0c54f3919b193eb17c2a0e19ece0a5bc5e7d3d6a33a76978aae", "timing_coverage": 1.0, "plottable": true, "config": "bf16" }, { "model": "jevk5-2b", "name": "JevK5 2B", "code": "J52", "suite": "industry/routing/nb", "n": 40, "correct": 32, "accuracy": 0.8, "interval": [ 0.652426936536005, 0.895000102745623 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 41.4112500147894, "p95_ms": 47.54434997448698, "device_peak_gib": 5.7194366455078125, "device_min_gib": 5.7194366455078125, "samples": 50, "added_device_peak_gib": 3.8447608947753906, "torch_peak_allocated_gib": 3.554513931274414, "torch_peak_reserved_gib": 3.57421875, "warmup_errors": [], "finished": "2026-10-01T14:40:29.057525+00:00", "predictions_sha256": "a353d29935dd86fca213bae23eebeffe7f4fb4ae009dc58e0db139fbc4a9adcf", "timing_coverage": 1.0, "plottable": true, "config": "bf16" }, { "model": "jevk5-2b", "name": "JevK5 2B", "code": "J52", "suite": "industry/documents/en", "n": 28, "correct": 27, "accuracy": 0.9642857142857143, "interval": [ 0.8228780225664668, 0.9936674801509623 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 42.3990999115631, "p95_ms": 50.84857507608831, "device_peak_gib": 5.7194366455078125, "device_min_gib": 5.7194366455078125, "samples": 52, "added_device_peak_gib": 3.8447608947753906, "torch_peak_allocated_gib": 3.554243564605713, "torch_peak_reserved_gib": 3.57421875, "warmup_errors": [], "finished": "2026-10-01T14:40:30.978101+00:00", "predictions_sha256": "fc095ea51728436476f3c3605b6352e35d2595485ac5fe672a4ab54bb53497b6", "timing_coverage": 1.0, "plottable": true, "config": "bf16" }, { "model": "jevk5-2b", "name": "JevK5 2B", "code": "J52", "suite": "industry/documents/nb", "n": 28, "correct": 27, "accuracy": 0.9642857142857143, "interval": [ 0.8228780225664668, 0.9936674801509623 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 42.415449977852404, "p95_ms": 51.35222495300695, "device_peak_gib": 5.7243194580078125, "device_min_gib": 5.7169952392578125, "samples": 52, "added_device_peak_gib": 3.8496437072753906, "torch_peak_allocated_gib": 3.554175853729248, "torch_peak_reserved_gib": 3.57421875, "warmup_errors": [], "finished": "2026-10-01T14:40:32.909406+00:00", "predictions_sha256": "037da8a17f54741e52e8d9685f7a1409606c0d20eac94615cce395ac92475a8c", "timing_coverage": 1.0, "plottable": true, "config": "bf16" }, { "model": "jevk5", "name": "JevK5 4B", "code": "J54", "suite": "fresh/news", "n": 200, "correct": 173, "accuracy": 0.865, "interval": [ 0.8107086162924745, 0.9055342954521209 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 54.21634984668344, "p95_ms": 64.41536013735458, "device_peak_gib": 10.065139770507812, "device_min_gib": 10.062698364257812, "samples": 109, "added_device_peak_gib": 8.19052505493164, "torch_peak_allocated_gib": 7.898333549499512, "torch_peak_reserved_gib": 7.919921875, "warmup_errors": [], "finished": "2026-10-01T14:40:51.827741+00:00", "predictions_sha256": "feb29ed95e4bff7d711783465350182e71fa32f967543d0e26494738a0b709cb", "timing_coverage": 1.0, "plottable": true, "config": "bf16" }, { "model": "jevk5", "name": "JevK5 4B", "code": "J54", "suite": "fresh/emotion", "n": 200, "correct": 121, "accuracy": 0.605, "interval": [ 0.5358834915928536, 0.6701589898679203 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 55.31224992591888, "p95_ms": 68.13065997557715, "device_peak_gib": 10.050491333007812, "device_min_gib": 10.043167114257812, "samples": 111, "added_device_peak_gib": 8.17587661743164, "torch_peak_allocated_gib": 7.894337177276611, "torch_peak_reserved_gib": 7.900390625, "warmup_errors": [], "finished": "2026-10-01T14:40:55.718989+00:00", "predictions_sha256": "6c3782b9aa4b6a67ae72a927da2c663a3093337caa8b6cf43a9ac6eb9e2c00f8", "timing_coverage": 1.0, "plottable": true, "config": "bf16" }, { "model": "jevk5", "name": "JevK5 4B", "code": "J54", "suite": "fresh/spam", "n": 200, "correct": 166, "accuracy": 0.83, "interval": [ 0.7718411637168737, 0.8757209208741302 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 55.01429992727935, "p95_ms": 68.53702501393855, "device_peak_gib": 10.097366333007812, "device_min_gib": 10.072952270507812, "samples": 112, "added_device_peak_gib": 8.22275161743164, "torch_peak_allocated_gib": 7.920492172241211, "torch_peak_reserved_gib": 7.947265625, "warmup_errors": [], "finished": "2026-10-01T14:40:59.638854+00:00", "predictions_sha256": "846f6f034b93b18977aeb77df4fe6be5f6f6f1198968bf7eae1fc0e042270714", "timing_coverage": 1.0, "plottable": true, "config": "bf16" }, { "model": "jevk5", "name": "JevK5 4B", "code": "J54", "suite": "fresh/phishing", "n": 200, "correct": 180, "accuracy": 0.9, "interval": [ 0.8505941875672826, 0.9343295513309041 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 54.33695006649941, "p95_ms": 63.570580095984035, "device_peak_gib": 10.067092895507812, "device_min_gib": 10.064651489257812, "samples": 108, "added_device_peak_gib": 8.19247817993164, "torch_peak_allocated_gib": 7.905336380004883, "torch_peak_reserved_gib": 7.921875, "warmup_errors": [], "finished": "2026-10-01T14:41:03.453971+00:00", "predictions_sha256": "e0dfbba86b0a869365b0091750127552273c6adcf1ac82d39a009ab478dc4d6a", "timing_coverage": 1.0, "plottable": true, "config": "bf16" }, { "model": "jevk5", "name": "JevK5 4B", "code": "J54", "suite": "massive_en", "n": 2948, "correct": 2375, "accuracy": 0.8056309362279511, "interval": [ 0.7909524218774165, 0.8195139683872986 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 174.6439499547705, "p95_ms": 184.60299018770456, "device_peak_gib": 10.089553833007812, "device_min_gib": 10.082229614257812, "samples": 338, "added_device_peak_gib": 8.21493911743164, "torch_peak_allocated_gib": 7.920109748840332, "torch_peak_reserved_gib": 7.939453125, "warmup_errors": [], "finished": "2026-10-01T14:41:15.357983+00:00", "predictions_sha256": "ba3dbeb0b9002010a84ed71e819a170783d04edfef98d017d6ad6c01b105634c", "timing_coverage": 1.0, "plottable": true, "config": "bf16" }, { "model": "jevk5", "name": "JevK5 4B", "code": "J54", "suite": "massive_nb", "n": 2948, "correct": 2133, "accuracy": 0.7235413839891451, "interval": [ 0.7071136166964754, 0.7393873280143427 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 180.14505004975945, "p95_ms": 192.1468600514345, "device_peak_gib": 10.301071166992188, "device_min_gib": 10.281356811523438, "samples": 349, "added_device_peak_gib": 8.22580337524414, "torch_peak_allocated_gib": 7.921629428863525, "torch_peak_reserved_gib": 7.939453125, "warmup_errors": [], "finished": "2026-10-04T20:12:45.019729+00:00", "predictions_sha256": "f23993e8594ab561173f9206d2ed86804dffefa3f7b2e7f145a5f237d2a9f8df", "timing_coverage": 1.0, "plottable": true, "config": "bf16" }, { "model": "jevk5", "name": "JevK5 4B", "code": "J54", "suite": "public_jevbench", "n": 231, "correct": 202, "accuracy": 0.8744588744588745, "interval": [ 0.8255168640609313, 0.9111503363080152 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 61.933099874295294, "p95_ms": 561.1418500659054, "device_peak_gib": 13.395614624023438, "device_min_gib": 10.261825561523438, "samples": 330, "added_device_peak_gib": 11.32034683227539, "torch_peak_allocated_gib": 9.793753147125244, "torch_peak_reserved_gib": 11.044921875, "warmup_errors": [], "finished": "2026-10-04T20:12:55.731717+00:00", "predictions_sha256": "c186fb378b586fea111048a33599fa955d08d9d5519bf32994c49f3b62d40ff7", "timing_coverage": 1.0, "plottable": true, "config": "bf16" }, { "model": "jevk5", "name": "JevK5 4B", "code": "J54", "suite": "industry/routing/en", "n": 40, "correct": 37, "accuracy": 0.925, "interval": [ 0.8013576647568946, 0.9741639742254119 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 54.83839998487383, "p95_ms": 64.85197501024231, "device_peak_gib": 10.068069458007812, "device_min_gib": 10.060745239257812, "samples": 67, "added_device_peak_gib": 8.19345474243164, "torch_peak_allocated_gib": 7.898271560668945, "torch_peak_reserved_gib": 7.91796875, "warmup_errors": [], "finished": "2026-10-01T14:41:17.831461+00:00", "predictions_sha256": "9a2b1a582cd1a17c0348f71a1d80a30a152b085e087062a7066872c61170b95f", "timing_coverage": 1.0, "plottable": true, "config": "bf16" }, { "model": "jevk5", "name": "JevK5 4B", "code": "J54", "suite": "industry/routing/nb", "n": 40, "correct": 36, "accuracy": 0.9, "interval": [ 0.7694822477247771, 0.9604204713173936 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 54.27605006843805, "p95_ms": 62.64572497457266, "device_peak_gib": 10.065139770507812, "device_min_gib": 10.065139770507812, "samples": 66, "added_device_peak_gib": 8.19052505493164, "torch_peak_allocated_gib": 7.899028778076172, "torch_peak_reserved_gib": 7.919921875, "warmup_errors": [], "finished": "2026-10-01T14:41:20.295633+00:00", "predictions_sha256": "65937ec863f3dcbcf9ae90966c5a43847cdc534b08d6ee7a44a4667e8272a45a", "timing_coverage": 1.0, "plottable": true, "config": "bf16" }, { "model": "jevk5", "name": "JevK5 4B", "code": "J54", "suite": "industry/documents/en", "n": 28, "correct": 27, "accuracy": 0.9642857142857143, "interval": [ 0.8228780225664668, 0.9936674801509623 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 54.54070004634559, "p95_ms": 60.2661250741221, "device_peak_gib": 10.071975708007812, "device_min_gib": 10.064651489257812, "samples": 66, "added_device_peak_gib": 8.19736099243164, "torch_peak_allocated_gib": 7.898556709289551, "torch_peak_reserved_gib": 7.921875, "warmup_errors": [], "finished": "2026-10-01T14:41:22.731610+00:00", "predictions_sha256": "ea4cdd90fae40b68c1434aec8731569b006727492fa9942bfcc2a9c9587a94fa", "timing_coverage": 1.0, "plottable": true, "config": "bf16" }, { "model": "jevk5", "name": "JevK5 4B", "code": "J54", "suite": "industry/documents/nb", "n": 28, "correct": 27, "accuracy": 0.9642857142857143, "interval": [ 0.8228780225664668, 0.9936674801509623 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 54.27299998700619, "p95_ms": 60.20564993377775, "device_peak_gib": 10.063186645507812, "device_min_gib": 10.063186645507812, "samples": 65, "added_device_peak_gib": 8.18857192993164, "torch_peak_allocated_gib": 7.898438930511475, "torch_peak_reserved_gib": 7.91796875, "warmup_errors": [], "finished": "2026-10-01T14:41:25.157500+00:00", "predictions_sha256": "58b1f3d88947726e17241cd4b018a2b9e87c17171358456ba4559e270f171d90", "timing_coverage": 1.0, "plottable": true, "config": "bf16" }, { "model": "julia", "name": "Julia", "code": "JU", "suite": "fresh/news", "n": 200, "correct": 179, "accuracy": 0.895, "interval": [ 0.8448196800887696, 0.9302925120731899 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 13.797599938698113, "p95_ms": 22.994819958694276, "device_peak_gib": 2.8209991455078125, "device_min_gib": 2.8209991455078125, "samples": 31, "added_device_peak_gib": 0.9463844299316406, "torch_peak_allocated_gib": 0.6589202880859375, "torch_peak_reserved_gib": 0.677734375, "warmup_errors": [], "finished": "2026-10-01T14:41:36.128896+00:00", "predictions_sha256": "d029b8e0e2fb5db0c2c44e8ffa93b22d66ddaf43bfcb694af6fbbdb9c474948a", "timing_coverage": 1.0, "plottable": true, "config": "native fp32 weights with bf16 autocast" }, { "model": "julia", "name": "Julia", "code": "JU", "suite": "fresh/emotion", "n": 200, "correct": 182, "accuracy": 0.91, "interval": [ 0.8622342812328346, 0.9423125511378069 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 13.671849970705807, "p95_ms": 22.88561494788155, "device_peak_gib": 2.8209991455078125, "device_min_gib": 2.8190460205078125, "samples": 31, "added_device_peak_gib": 0.9463844299316406, "torch_peak_allocated_gib": 0.6585421562194824, "torch_peak_reserved_gib": 0.677734375, "warmup_errors": [], "finished": "2026-10-01T14:41:37.191272+00:00", "predictions_sha256": "317afd11e2fb727dea377a29b3d77a834951a3544fa5ea70da9795e53d8b50f3", "timing_coverage": 1.0, "plottable": true, "config": "native fp32 weights with bf16 autocast" }, { "model": "julia", "name": "Julia", "code": "JU", "suite": "fresh/spam", "n": 200, "correct": 93, "accuracy": 0.465, "interval": [ 0.3971856415700402, 0.5341335312763685 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 13.854449964128435, "p95_ms": 21.02955484297126, "device_peak_gib": 2.8229522705078125, "device_min_gib": 2.8209991455078125, "samples": 31, "added_device_peak_gib": 0.9483375549316406, "torch_peak_allocated_gib": 0.6600565910339355, "torch_peak_reserved_gib": 0.6796875, "warmup_errors": [], "finished": "2026-10-01T14:41:38.260294+00:00", "predictions_sha256": "b512267177a9c52cb4ce29470de9b1fedfe7cb422823136a7f992582a13c48ce", "timing_coverage": 1.0, "plottable": true, "config": "native fp32 weights with bf16 autocast" }, { "model": "julia", "name": "Julia", "code": "JU", "suite": "fresh/phishing", "n": 200, "correct": 76, "accuracy": 0.38, "interval": [ 0.31559005848673627, 0.44893281984380773 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 14.062299975194037, "p95_ms": 22.128595039248466, "device_peak_gib": 2.8229522705078125, "device_min_gib": 2.8229522705078125, "samples": 31, "added_device_peak_gib": 0.9483375549316406, "torch_peak_allocated_gib": 0.6596145629882812, "torch_peak_reserved_gib": 0.677734375, "warmup_errors": [], "finished": "2026-10-01T14:41:39.319038+00:00", "predictions_sha256": "c342e9dbf268bb0e47ad56b84cb4f2f1c463943769234f8d2549719a6991dab4", "timing_coverage": 1.0, "plottable": true, "config": "native fp32 weights with bf16 autocast" }, { "model": "julia", "name": "Julia", "code": "JU", "suite": "massive_en", "n": 2948, "correct": 1256, "accuracy": 0.4260515603799186, "interval": [ 0.40830859347268206, 0.4439869968961216 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 14.076499966904521, "p95_ms": 22.9078549426049, "device_peak_gib": 2.8278350830078125, "device_min_gib": 2.8205108642578125, "samples": 32, "added_device_peak_gib": 0.9532203674316406, "torch_peak_allocated_gib": 0.6588573455810547, "torch_peak_reserved_gib": 0.677734375, "warmup_errors": [], "finished": "2026-10-01T14:41:40.410705+00:00", "predictions_sha256": "2aace9d1e322384537d84f9d7926d6616d75149a62885b38500f67f300cb17d5", "timing_coverage": 1.0, "plottable": true, "config": "native fp32 weights with bf16 autocast" }, { "model": "julia", "name": "Julia", "code": "JU", "suite": "massive_nb", "n": 2948, "correct": 1174, "accuracy": 0.39823609226594303, "interval": [ 0.38070826254437656, 0.416028788421867 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 13.553850003518164, "p95_ms": 19.68907006084919, "device_peak_gib": 2.8229522705078125, "device_min_gib": 2.8229522705078125, "samples": 30, "added_device_peak_gib": 0.9483375549316406, "torch_peak_allocated_gib": 0.6588573455810547, "torch_peak_reserved_gib": 0.677734375, "warmup_errors": [], "finished": "2026-10-01T14:41:41.428125+00:00", "predictions_sha256": "4f9ee5d3c5a570a11ed4bd0ccd4d130b4586ae2ec4a221d7590f25713c969160", "timing_coverage": 1.0, "plottable": true, "config": "native fp32 weights with bf16 autocast" }, { "model": "julia", "name": "Julia", "code": "JU", "suite": "public_jevbench", "n": 231, "correct": 109, "accuracy": 0.47186147186147187, "interval": [ 0.4084728884308193, 0.5361706167437684 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 16.082249931059778, "p95_ms": 51.08588978182524, "device_peak_gib": 3.0104522705078125, "device_min_gib": 2.8249053955078125, "samples": 43, "added_device_peak_gib": 1.1358375549316406, "torch_peak_allocated_gib": 0.8208847045898438, "torch_peak_reserved_gib": 0.865234375, "warmup_errors": [], "finished": "2026-10-01T14:41:42.858660+00:00", "predictions_sha256": "3a62c3c27a9433a0fb8ecbc1eccf547002294ebf37b7a19cc739e838ce55119c", "timing_coverage": 1.0, "plottable": true, "config": "native fp32 weights with bf16 autocast" }, { "model": "julia", "name": "Julia", "code": "JU", "suite": "kev_claim/devtools-v1", "n": 900, "correct": 326, "accuracy": 0.3622222222222222, "interval": [ 0.3086895993806083, 0.40641196977090266 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 14.565500081516802, "p95_ms": 27.307074866257608, "device_peak_gib": 2.8444366455078125, "device_min_gib": 2.8444366455078125, "samples": 21, "added_device_peak_gib": 0.9698219299316406, "torch_peak_allocated_gib": 0.6731290817260742, "torch_peak_reserved_gib": 0.69921875, "warmup_errors": [], "finished": "2026-10-01T14:41:43.632420+00:00", "predictions_sha256": "d2bcd70ba8227f04acadfb335693745270cd0ab11b032c94f9e7ee57a4d3cabb", "timing_coverage": 1.0, "plottable": true, "config": "native fp32 weights with bf16 autocast" }, { "model": "julia", "name": "Julia", "code": "JU", "suite": "industry/routing/en", "n": 40, "correct": 19, "accuracy": 0.475, "interval": [ 0.329354701126789, 0.6250263789330753 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 13.701200019568205, "p95_ms": 20.798400102648884, "device_peak_gib": 2.8229522705078125, "device_min_gib": 2.8229522705078125, "samples": 19, "added_device_peak_gib": 0.9483375549316406, "torch_peak_allocated_gib": 0.6587944030761719, "torch_peak_reserved_gib": 0.677734375, "warmup_errors": [], "finished": "2026-10-01T14:41:44.312917+00:00", "predictions_sha256": "9200493ade2bebe9083062ff00bb490fa99ab2cf27ea207a9f75d158af9c07ab", "timing_coverage": 1.0, "plottable": true, "config": "native fp32 weights with bf16 autocast" }, { "model": "julia", "name": "Julia", "code": "JU", "suite": "industry/routing/nb", "n": 40, "correct": 11, "accuracy": 0.275, "interval": [ 0.16108016247667337, 0.42834955806210573 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 13.873249990865588, "p95_ms": 20.30669996747747, "device_peak_gib": 2.8229522705078125, "device_min_gib": 2.8229522705078125, "samples": 19, "added_device_peak_gib": 0.9483375549316406, "torch_peak_allocated_gib": 0.6588573455810547, "torch_peak_reserved_gib": 0.677734375, "warmup_errors": [], "finished": "2026-10-01T14:41:44.983827+00:00", "predictions_sha256": "6fea8a6c64042c55bdf23799531c284b706a586c8f3cd770568238345555334a", "timing_coverage": 1.0, "plottable": true, "config": "native fp32 weights with bf16 autocast" }, { "model": "julia", "name": "Julia", "code": "JU", "suite": "industry/documents/en", "n": 28, "correct": 18, "accuracy": 0.6428571428571429, "interval": [ 0.4583030651442781, 0.7929417049226233 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 13.44570005312562, "p95_ms": 24.692699953448027, "device_peak_gib": 2.8278350830078125, "device_min_gib": 2.8205108642578125, "samples": 19, "added_device_peak_gib": 0.9532203674316406, "torch_peak_allocated_gib": 0.6589202880859375, "torch_peak_reserved_gib": 0.677734375, "warmup_errors": [], "finished": "2026-10-01T14:41:45.660803+00:00", "predictions_sha256": "6cb0d02fcf81fa4ad525efda7c4098335701c7f15bbf87f44e46f82067c880c2", "timing_coverage": 1.0, "plottable": true, "config": "native fp32 weights with bf16 autocast" }, { "model": "julia", "name": "Julia", "code": "JU", "suite": "industry/documents/nb", "n": 28, "correct": 14, "accuracy": 0.5, "interval": [ 0.32633127457293465, 0.6736687254270654 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 14.399450039491057, "p95_ms": 23.62580003682524, "device_peak_gib": 2.8229522705078125, "device_min_gib": 2.8205108642578125, "samples": 20, "added_device_peak_gib": 0.9483375549316406, "torch_peak_allocated_gib": 0.6589202880859375, "torch_peak_reserved_gib": 0.677734375, "warmup_errors": [], "finished": "2026-10-01T14:41:46.370177+00:00", "predictions_sha256": "25abba362f126a9b2ca0c896097a1b1df602c7ab57c9dab789f08738a57dbefa", "timing_coverage": 1.0, "plottable": true, "config": "native fp32 weights with bf16 autocast" }, { "model": "kev-08b", "name": "Kev 0.8B", "code": "K.8", "suite": "fresh/news", "n": 200, "correct": 177, "accuracy": 0.885, "interval": [ 0.8333528310151243, 0.9221362676743804 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 48.16410003695637, "p95_ms": 82.85064998781309, "device_peak_gib": 3.6999053955078125, "device_min_gib": 3.6999053955078125, "samples": 108, "added_device_peak_gib": 1.8252906799316406, "torch_peak_allocated_gib": 1.4862594604492188, "torch_peak_reserved_gib": 1.5546875, "warmup_errors": [], "finished": "2026-10-01T14:42:01.112157+00:00", "predictions_sha256": "6e3dc2be5ed37e14b70acc8e692c8f792379e5b434e2c282d12fe2061b47cd9b", "timing_coverage": 1.0, "plottable": true, "config": "bf16 (published serving mode, claim eval uses fp32)" }, { "model": "kev-08b", "name": "Kev 0.8B", "code": "K.8", "suite": "fresh/emotion", "n": 200, "correct": 107, "accuracy": 0.535, "interval": [ 0.4658664687236316, 0.6028143584299599 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 45.754099963232875, "p95_ms": 62.74393491912633, "device_peak_gib": 3.7008819580078125, "device_min_gib": 3.6935577392578125, "samples": 94, "added_device_peak_gib": 1.8262672424316406, "torch_peak_allocated_gib": 1.4821343421936035, "torch_peak_reserved_gib": 1.55078125, "warmup_errors": [], "finished": "2026-10-01T14:42:04.372619+00:00", "predictions_sha256": "bb9e351a9600e2e7da7a80611c2a693186f3a7adcd5658db3076148b4484326b", "timing_coverage": 1.0, "plottable": true, "config": "bf16 (published serving mode, claim eval uses fp32)" }, { "model": "kev-08b", "name": "Kev 0.8B", "code": "K.8", "suite": "fresh/spam", "n": 200, "correct": 139, "accuracy": 0.695, "interval": [ 0.6280144790202163, 0.7546358436926496 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 48.09185001067817, "p95_ms": 77.3014400852844, "device_peak_gib": 3.7106475830078125, "device_min_gib": 3.7018585205078125, "samples": 105, "added_device_peak_gib": 1.8360328674316406, "torch_peak_allocated_gib": 1.4975628852844238, "torch_peak_reserved_gib": 1.560546875, "warmup_errors": [], "finished": "2026-10-01T14:42:08.058663+00:00", "predictions_sha256": "a4d318b43766f7d73c3a103b7fd113b7ed791178f30677d038f0e506fae2331b", "timing_coverage": 1.0, "plottable": true, "config": "bf16 (published serving mode, claim eval uses fp32)" }, { "model": "kev-08b", "name": "Kev 0.8B", "code": "K.8", "suite": "fresh/phishing", "n": 200, "correct": 132, "accuracy": 0.66, "interval": [ 0.5918849552683626, 0.7220845402909121 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 46.85580008663237, "p95_ms": 62.2466450324282, "device_peak_gib": 3.7038116455078125, "device_min_gib": 3.7013702392578125, "samples": 97, "added_device_peak_gib": 1.8291969299316406, "torch_peak_allocated_gib": 1.4948320388793945, "torch_peak_reserved_gib": 1.55859375, "warmup_errors": [], "finished": "2026-10-01T14:42:11.380089+00:00", "predictions_sha256": "4b4648263be27034c6c703437c87ab2856973992bc1987a437f37e5186b21de0", "timing_coverage": 1.0, "plottable": true, "config": "bf16 (published serving mode, claim eval uses fp32)" }, { "model": "kev-08b", "name": "Kev 0.8B", "code": "K.8", "suite": "massive_en", "n": 2948, "correct": 2114, "accuracy": 0.7170963364993216, "interval": [ 0.7005629931830261, 0.7330646314223209 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 49.70155004411936, "p95_ms": 77.38314003217964, "device_peak_gib": 3.7047882080078125, "device_min_gib": 3.6974639892578125, "samples": 112, "added_device_peak_gib": 1.8301734924316406, "torch_peak_allocated_gib": 1.4926962852478027, "torch_peak_reserved_gib": 1.5546875, "warmup_errors": [], "finished": "2026-10-01T14:42:15.035179+00:00", "predictions_sha256": "5ce5a6f4386c9b2d82faa0245516948938a428dabd66b49b181370d031a5494a", "timing_coverage": 1.0, "plottable": true, "config": "bf16 (published serving mode, claim eval uses fp32)" }, { "model": "kev-08b", "name": "Kev 0.8B", "code": "K.8", "suite": "massive_nb", "n": 2948, "correct": 1719, "accuracy": 0.5831071913161465, "interval": [ 0.5652123118708426, 0.6007857631733918 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 45.3944499604404, "p95_ms": 56.00759502267465, "device_peak_gib": 3.6999053955078125, "device_min_gib": 3.6999053955078125, "samples": 91, "added_device_peak_gib": 1.8252906799316406, "torch_peak_allocated_gib": 1.4928412437438965, "torch_peak_reserved_gib": 1.5546875, "warmup_errors": [], "finished": "2026-10-01T14:42:18.286835+00:00", "predictions_sha256": "192210982c0dd192804b2d4031553a72ea94fe30e90381d230df987a843a0a11", "timing_coverage": 1.0, "plottable": true, "config": "bf16 (published serving mode, claim eval uses fp32)" }, { "model": "kev-08b", "name": "Kev 0.8B", "code": "K.8", "suite": "public_jevbench", "n": 231, "correct": 147, "accuracy": 0.6363636363636364, "interval": [ 0.5725682725446802, 0.6956978177630865 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 46.02580005303025, "p95_ms": 138.62338509643445, "device_peak_gib": 5.1471710205078125, "device_min_gib": 3.7013702392578125, "samples": 128, "added_device_peak_gib": 3.2725563049316406, "torch_peak_allocated_gib": 2.4366254806518555, "torch_peak_reserved_gib": 3.001953125, "warmup_errors": [], "finished": "2026-10-01T14:42:22.608880+00:00", "predictions_sha256": "51b554f490a717a3e0df72080950359d4e40f5f44ca78b5fb69c2e6962d5ca75", "timing_coverage": 1.0, "plottable": true, "config": "bf16 (published serving mode, claim eval uses fp32)" }, { "model": "kev-08b", "name": "Kev 0.8B", "code": "K.8", "suite": "kev_claim/devtools-v1", "n": 900, "correct": 517, "accuracy": 0.5744444444444444, "interval": [ 0.5089052873251443, 0.6337125125833389 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 44.595399987883866, "p95_ms": 55.68577500525862, "device_peak_gib": 3.7995147705078125, "device_min_gib": 3.7409210205078125, "samples": 55, "added_device_peak_gib": 1.9249000549316406, "torch_peak_allocated_gib": 1.5914030075073242, "torch_peak_reserved_gib": 1.654296875, "warmup_errors": [], "finished": "2026-10-01T14:42:24.716013+00:00", "predictions_sha256": "5253a286747d5b3cdcd40ec6ce2ff889f96b1a0f70fb45944908f300793a173c", "timing_coverage": 1.0, "plottable": true, "config": "bf16 (published serving mode, claim eval uses fp32)" }, { "model": "kev-08b", "name": "Kev 0.8B", "code": "K.8", "suite": "industry/routing/en", "n": 40, "correct": 33, "accuracy": 0.825, "interval": [ 0.6805000966821229, 0.9125458625396408 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 45.920099946670234, "p95_ms": 52.076725056394935, "device_peak_gib": 3.7047882080078125, "device_min_gib": 3.6974639892578125, "samples": 55, "added_device_peak_gib": 1.8301734924316406, "torch_peak_allocated_gib": 1.4856982231140137, "torch_peak_reserved_gib": 1.5546875, "warmup_errors": [], "finished": "2026-10-01T14:42:26.828733+00:00", "predictions_sha256": "8ec80f97070098dac0f9cde0d4e79faecdc1d854c1056e2e2aba2f82e89a84c2", "timing_coverage": 1.0, "plottable": true, "config": "bf16 (published serving mode, claim eval uses fp32)" }, { "model": "kev-08b", "name": "Kev 0.8B", "code": "K.8", "suite": "industry/routing/nb", "n": 40, "correct": 29, "accuracy": 0.725, "interval": [ 0.5716504419378943, 0.8389198375233267 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 45.74135004077107, "p95_ms": 53.53297491092235, "device_peak_gib": 3.6999053955078125, "device_min_gib": 3.6999053955078125, "samples": 56, "added_device_peak_gib": 1.8252906799316406, "torch_peak_allocated_gib": 1.4862041473388672, "torch_peak_reserved_gib": 1.5546875, "warmup_errors": [], "finished": "2026-10-01T14:42:28.910657+00:00", "predictions_sha256": "49df19c23ad029fec61a8cd9f06e6bad36e75f18d831ecfddc52858afbaaf201", "timing_coverage": 1.0, "plottable": true, "config": "bf16 (published serving mode, claim eval uses fp32)" }, { "model": "kev-08b", "name": "Kev 0.8B", "code": "K.8", "suite": "industry/documents/en", "n": 28, "correct": 27, "accuracy": 0.9642857142857143, "interval": [ 0.8228780225664668, 0.9936674801509623 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 47.33095003757626, "p95_ms": 53.40109998360276, "device_peak_gib": 3.7047882080078125, "device_min_gib": 3.6974639892578125, "samples": 56, "added_device_peak_gib": 1.8301734924316406, "torch_peak_allocated_gib": 1.4864816665649414, "torch_peak_reserved_gib": 1.5546875, "warmup_errors": [], "finished": "2026-10-01T14:42:31.011214+00:00", "predictions_sha256": "826c9b69692152b48a2c2872fdfdf2ce01a8e6abc40bd01f821f1f1d0f821d70", "timing_coverage": 1.0, "plottable": true, "config": "bf16 (published serving mode, claim eval uses fp32)" }, { "model": "kev-08b", "name": "Kev 0.8B", "code": "K.8", "suite": "industry/documents/nb", "n": 28, "correct": 28, "accuracy": 1.0, "interval": [ 0.8793566952341543, 0.9999999999999998 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 46.428399975411594, "p95_ms": 55.49800000153482, "device_peak_gib": 3.6999053955078125, "device_min_gib": 3.6999053955078125, "samples": 57, "added_device_peak_gib": 1.8252906799316406, "torch_peak_allocated_gib": 1.4864263534545898, "torch_peak_reserved_gib": 1.5546875, "warmup_errors": [], "finished": "2026-10-01T14:42:33.105074+00:00", "predictions_sha256": "667f048245d620f5bdfb3da2966aa06736a60f4eddbcffb435267fda35dc06fc", "timing_coverage": 1.0, "plottable": true, "config": "bf16 (published serving mode, claim eval uses fp32)" }, { "model": "kev-4b", "name": "Kev 4B", "code": "K4", "suite": "fresh/news", "n": 200, "correct": 179, "accuracy": 0.895, "interval": [ 0.8448196800887696, 0.9302925120731899 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 61.53820001054555, "p95_ms": 78.84655001107603, "device_peak_gib": 10.206741333007812, "device_min_gib": 10.199417114257812, "samples": 123, "added_device_peak_gib": 8.33212661743164, "torch_peak_allocated_gib": 8.032083988189697, "torch_peak_reserved_gib": 8.056640625, "warmup_errors": [], "finished": "2026-10-01T14:42:56.482528+00:00", "predictions_sha256": "0ab817693539c915f84eacf78c56320065d528705c259d59742563a683d2645d", "timing_coverage": 1.0, "plottable": true, "config": "bf16 (published serving mode, claim eval uses fp32)" }, { "model": "kev-4b", "name": "Kev 4B", "code": "K4", "suite": "fresh/emotion", "n": 200, "correct": 114, "accuracy": 0.57, "interval": [ 0.5007048776415268, 0.6366567766656557 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 60.65714999567717, "p95_ms": 73.89379508094862, "device_peak_gib": 10.183303833007812, "device_min_gib": 10.175979614257812, "samples": 122, "added_device_peak_gib": 8.30868911743164, "torch_peak_allocated_gib": 8.024236679077148, "torch_peak_reserved_gib": 8.033203125, "warmup_errors": [], "finished": "2026-10-01T14:43:00.752473+00:00", "predictions_sha256": "c1cd5f79a6486eb59363ab26577fc629310c0722462666cd62b3fd9dc914a097", "timing_coverage": 1.0, "plottable": true, "config": "bf16 (published serving mode, claim eval uses fp32)" }, { "model": "kev-4b", "name": "Kev 4B", "code": "K4", "suite": "fresh/spam", "n": 200, "correct": 186, "accuracy": 0.93, "interval": [ 0.8859451911438936, 0.9578478281716571 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 58.54904989246279, "p95_ms": 67.49881007708609, "device_peak_gib": 10.226272583007812, "device_min_gib": 10.218948364257812, "samples": 116, "added_device_peak_gib": 8.35165786743164, "torch_peak_allocated_gib": 8.060317516326904, "torch_peak_reserved_gib": 8.076171875, "warmup_errors": [], "finished": "2026-10-01T14:43:04.816520+00:00", "predictions_sha256": "fc0a6b7e5cf34d9670278497d984ae42409f7751d2ad598ffe394571b9a4017b", "timing_coverage": 1.0, "plottable": true, "config": "bf16 (published serving mode, claim eval uses fp32)" }, { "model": "kev-4b", "name": "Kev 4B", "code": "K4", "suite": "fresh/phishing", "n": 200, "correct": 182, "accuracy": 0.91, "interval": [ 0.8622342812328346, 0.9423125511378069 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 61.140150064602494, "p95_ms": 72.64180501224472, "device_peak_gib": 10.221389770507812, "device_min_gib": 10.221389770507812, "samples": 124, "added_device_peak_gib": 8.34677505493164, "torch_peak_allocated_gib": 8.059450626373291, "torch_peak_reserved_gib": 8.076171875, "warmup_errors": [], "finished": "2026-10-01T14:43:09.104586+00:00", "predictions_sha256": "9934f0651a345bf77121ced728ffa40b175fd1b832556b4496b04640c0b6a3cc", "timing_coverage": 1.0, "plottable": true, "config": "bf16 (published serving mode, claim eval uses fp32)" }, { "model": "kev-4b", "name": "Kev 4B", "code": "K4", "suite": "massive_en", "n": 2948, "correct": 2327, "accuracy": 0.7893487109905021, "interval": [ 0.7742571570337314, 0.8036871613855828 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 57.03525000717491, "p95_ms": 64.95845498284325, "device_peak_gib": 10.220413208007812, "device_min_gib": 10.213088989257812, "samples": 114, "added_device_peak_gib": 8.34579849243164, "torch_peak_allocated_gib": 8.057387828826904, "torch_peak_reserved_gib": 8.0703125, "warmup_errors": [], "finished": "2026-10-01T14:43:13.078442+00:00", "predictions_sha256": "fe4b8bb96b447d51c0fb509e2833e3ff87045ad4ca23af9e5c8f58b314dd0fc1", "timing_coverage": 1.0, "plottable": true, "config": "bf16 (published serving mode, claim eval uses fp32)" }, { "model": "kev-4b", "name": "Kev 4B", "code": "K4", "suite": "massive_nb", "n": 2948, "correct": 2123, "accuracy": 0.7201492537313433, "interval": [ 0.7036654046690269, 0.7360601084073328 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 57.8132999362424, "p95_ms": 76.93921999307348, "device_peak_gib": 10.222366333007812, "device_min_gib": 10.215042114257812, "samples": 118, "added_device_peak_gib": 8.34775161743164, "torch_peak_allocated_gib": 8.06013011932373, "torch_peak_reserved_gib": 8.072265625, "warmup_errors": [], "finished": "2026-10-01T14:43:17.220259+00:00", "predictions_sha256": "5e5b53d8e1deaf7bcf5ffc286155fa887448c9b04a290f56a33dc92767fd9458", "timing_coverage": 1.0, "plottable": true, "config": "bf16 (published serving mode, claim eval uses fp32)" }, { "model": "kev-4b", "name": "Kev 4B", "code": "K4", "suite": "public_jevbench", "n": 231, "correct": 175, "accuracy": 0.7575757575757576, "interval": [ 0.6983904669524358, 0.808334370295568 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 60.89784996584058, "p95_ms": 514.1182101331649, "device_peak_gib": 13.153961181640625, "device_min_gib": 10.219390869140625, "samples": 310, "added_device_peak_gib": 11.279346466064453, "torch_peak_allocated_gib": 9.928412914276123, "torch_peak_reserved_gib": 11.00390625, "warmup_errors": [], "finished": "2026-10-01T14:43:27.340510+00:00", "predictions_sha256": "3b75e6f1a33b5ff8f2e88d1a36ccd75e9262c1e80f1f69e8c0ed218f50580114", "timing_coverage": 1.0, "plottable": true, "config": "bf16 (published serving mode, claim eval uses fp32)" }, { "model": "kev-4b", "name": "Kev 4B", "code": "K4", "suite": "kev_claim/devtools-v1", "n": 900, "correct": 640, "accuracy": 0.7111111111111111, "interval": [ 0.6762578729046651, 0.7400251993185536 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 61.81760004255921, "p95_ms": 142.6033749594353, "device_peak_gib": 10.547515869140625, "device_min_gib": 10.445953369140625, "samples": 90, "added_device_peak_gib": 8.672901153564453, "torch_peak_allocated_gib": 8.245697021484375, "torch_peak_reserved_gib": 8.40234375, "warmup_errors": [], "finished": "2026-10-01T14:43:30.767787+00:00", "predictions_sha256": "d4a643dbe8af2eddacc9af35571822bc514415c44720a3e9a41ab0c1d7655737", "timing_coverage": 1.0, "plottable": true, "config": "bf16 (published serving mode, claim eval uses fp32)" }, { "model": "kev-4b", "name": "Kev 4B", "code": "K4", "suite": "industry/routing/en", "n": 40, "correct": 35, "accuracy": 0.875, "interval": [ 0.738878801611489, 0.945404997490546 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 58.80455009173602, "p95_ms": 68.41577496379614, "device_peak_gib": 10.206695556640625, "device_min_gib": 10.199371337890625, "samples": 72, "added_device_peak_gib": 8.332080841064453, "torch_peak_allocated_gib": 8.030498504638672, "torch_peak_reserved_gib": 8.056640625, "warmup_errors": [], "finished": "2026-10-01T14:43:33.488958+00:00", "predictions_sha256": "fa0980dabeaaaa22b6107768574cebcd0a817c826c748fefd6928fd190ada886", "timing_coverage": 1.0, "plottable": true, "config": "bf16 (published serving mode, claim eval uses fp32)" }, { "model": "kev-4b", "name": "Kev 4B", "code": "K4", "suite": "industry/routing/nb", "n": 40, "correct": 35, "accuracy": 0.875, "interval": [ 0.738878801611489, 0.945404997490546 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 60.086349956691265, "p95_ms": 70.33560000127181, "device_peak_gib": 10.201812744140625, "device_min_gib": 10.201812744140625, "samples": 73, "added_device_peak_gib": 8.327198028564453, "torch_peak_allocated_gib": 8.031925201416016, "torch_peak_reserved_gib": 8.056640625, "warmup_errors": [], "finished": "2026-10-01T14:43:36.196085+00:00", "predictions_sha256": "f441585449ea57b79a1163aa476401ee4d98cc29c189a801392908b86549e03c", "timing_coverage": 1.0, "plottable": true, "config": "bf16 (published serving mode, claim eval uses fp32)" }, { "model": "kev-4b", "name": "Kev 4B", "code": "K4", "suite": "industry/documents/en", "n": 28, "correct": 28, "accuracy": 1.0, "interval": [ 0.8793566952341543, 0.9999999999999998 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 63.09804995544255, "p95_ms": 68.79499997012317, "device_peak_gib": 10.206695556640625, "device_min_gib": 10.199371337890625, "samples": 74, "added_device_peak_gib": 8.332080841064453, "torch_peak_allocated_gib": 8.03272294998169, "torch_peak_reserved_gib": 8.056640625, "warmup_errors": [], "finished": "2026-10-01T14:43:38.969718+00:00", "predictions_sha256": "4471133863e8c4471dbf185a35d4a0559760cc4bffe928ad15929cbabe82a57b", "timing_coverage": 1.0, "plottable": true, "config": "bf16 (published serving mode, claim eval uses fp32)" }, { "model": "kev-4b", "name": "Kev 4B", "code": "K4", "suite": "industry/documents/nb", "n": 28, "correct": 28, "accuracy": 1.0, "interval": [ 0.8793566952341543, 0.9999999999999998 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 59.42305002827197, "p95_ms": 68.69230006122962, "device_peak_gib": 10.201812744140625, "device_min_gib": 10.201812744140625, "samples": 72, "added_device_peak_gib": 8.327198028564453, "torch_peak_allocated_gib": 8.032563209533691, "torch_peak_reserved_gib": 8.056640625, "warmup_errors": [], "finished": "2026-10-01T14:43:41.667975+00:00", "predictions_sha256": "3e8e00bf83d56c42db9caf7233fe6b5d8b4f82e99cedc0e2d47d5688dba31a0c", "timing_coverage": 1.0, "plottable": true, "config": "bf16 (published serving mode, claim eval uses fp32)" }, { "model": "laya-multilingual", "name": "Laya multilingual", "code": "LM", "suite": "fresh/news", "n": 200, "correct": 183, "accuracy": 0.915, "interval": [ 0.8681041292843443, 0.9462542498225244 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 22.081600036472082, "p95_ms": 29.808440047781914, "device_peak_gib": 3.643512725830078, "device_min_gib": 3.643512725830078, "samples": 46, "added_device_peak_gib": 1.7823219299316406, "torch_peak_allocated_gib": 1.4946703910827637, "torch_peak_reserved_gib": 1.51171875, "warmup_errors": [], "finished": "2026-10-01T14:23:34.052802+00:00", "predictions_sha256": "f2665c5a24647ef7e34e343fc8b1f9f1df54697728eba82bb5db3037d282cf11", "timing_coverage": 1.0, "plottable": true, "config": "native" }, { "model": "laya-multilingual", "name": "Laya multilingual", "code": "LM", "suite": "fresh/emotion", "n": 200, "correct": 103, "accuracy": 0.515, "interval": [ 0.44610849192745716, 0.5833261482812249 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 21.538599976338446, "p95_ms": 29.8965299851261, "device_peak_gib": 3.646442413330078, "device_min_gib": 3.639118194580078, "samples": 45, "added_device_peak_gib": 1.7852516174316406, "torch_peak_allocated_gib": 1.4943914413452148, "torch_peak_reserved_gib": 1.509765625, "warmup_errors": [], "finished": "2026-10-01T14:23:35.602832+00:00", "predictions_sha256": "1b7d9391ea958320072e8c169ed0dfe8e07027d909cfb7c05043eae9835a3407", "timing_coverage": 1.0, "plottable": true, "config": "native" }, { "model": "laya-multilingual", "name": "Laya multilingual", "code": "LM", "suite": "fresh/spam", "n": 200, "correct": 200, "accuracy": 1.0, "interval": [ 0.9811546736227335, 1.0 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 21.24200004618615, "p95_ms": 25.2104400540702, "device_peak_gib": 3.643512725830078, "device_min_gib": 3.643512725830078, "samples": 43, "added_device_peak_gib": 1.7823219299316406, "torch_peak_allocated_gib": 1.4968132972717285, "torch_peak_reserved_gib": 1.51171875, "warmup_errors": [], "finished": "2026-10-01T14:23:37.088791+00:00", "predictions_sha256": "56c590123c713f891e5cff96c3586f97a4472eb18515fa0317367d8704c73c20", "timing_coverage": 1.0, "plottable": true, "config": "native" }, { "model": "laya-multilingual", "name": "Laya multilingual", "code": "LM", "suite": "fresh/phishing", "n": 200, "correct": 200, "accuracy": 1.0, "interval": [ 0.9811546736227335, 1.0 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 20.663300063461065, "p95_ms": 24.057830136734992, "device_peak_gib": 3.643512725830078, "device_min_gib": 3.643512725830078, "samples": 42, "added_device_peak_gib": 1.7823219299316406, "torch_peak_allocated_gib": 1.4955897331237793, "torch_peak_reserved_gib": 1.51171875, "warmup_errors": [], "finished": "2026-10-01T14:23:38.547084+00:00", "predictions_sha256": "e9539c915725e4f192c0848afaa70b44dab7d67485b08f60ad192cdf3b51eb58", "timing_coverage": 1.0, "plottable": true, "config": "native" }, { "model": "laya-multilingual", "name": "Laya multilingual", "code": "LM", "suite": "massive_en", "n": 2948, "correct": 1815, "accuracy": 0.6156716417910447, "interval": [ 0.5979724741601596, 0.6330697445748766 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 22.923900047317147, "p95_ms": 27.046390133909878, "device_peak_gib": 3.643512725830078, "device_min_gib": 3.641071319580078, "samples": 46, "added_device_peak_gib": 1.7823219299316406, "torch_peak_allocated_gib": 1.494955062866211, "torch_peak_reserved_gib": 1.51171875, "warmup_errors": [], "finished": "2026-10-01T14:23:40.141755+00:00", "predictions_sha256": "1a81eabffb8c3329abbd94fc2eeac4114b99fba676e5a90b907f1686985a92b9", "timing_coverage": 1.0, "plottable": true, "config": "native" }, { "model": "laya-multilingual", "name": "Laya multilingual", "code": "LM", "suite": "massive_nb", "n": 2948, "correct": 1511, "accuracy": 0.512550881953867, "interval": [ 0.494502922473108, 0.5305661745743885 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 22.759899962693453, "p95_ms": 31.399234989657998, "device_peak_gib": 3.648395538330078, "device_min_gib": 3.641071319580078, "samples": 48, "added_device_peak_gib": 1.7872047424316406, "torch_peak_allocated_gib": 1.4949865341186523, "torch_peak_reserved_gib": 1.51171875, "warmup_errors": [], "finished": "2026-10-01T14:23:41.804971+00:00", "predictions_sha256": "b80fa7718b59dba5ec7d141408905830fbdd0c0d536630ef076a89fe49166fd7", "timing_coverage": 1.0, "plottable": true, "config": "native" }, { "model": "laya-multilingual", "name": "Laya multilingual", "code": "LM", "suite": "public_jevbench", "n": 231, "correct": 109, "accuracy": 0.47186147186147187, "interval": [ 0.4084728884308193, 0.5361706167437684 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 15, "median_ms": 22.931350045837462, "p95_ms": 30.938819830771536, "device_peak_gib": 3.664997100830078, "device_min_gib": 3.645465850830078, "samples": 48, "added_device_peak_gib": 1.8038063049316406, "torch_peak_allocated_gib": 1.510664939880371, "torch_peak_reserved_gib": 1.533203125, "warmup_errors": [], "finished": "2026-10-01T14:23:43.444080+00:00", "predictions_sha256": "300337059336ffd71fc2d54bb4a59126b4c409f95a90b0d1b6c14e28d434e628", "timing_coverage": 1.0, "plottable": true, "config": "native" }, { "model": "laya-multilingual", "name": "Laya multilingual", "code": "LM", "suite": "kev_claim/devtools-v1", "n": 900, "correct": 411, "accuracy": 0.45666666666666667, "interval": [ 0.4109156735371691, 0.4954216778062591 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 22.450600052252412, "p95_ms": 30.18267499282956, "device_peak_gib": 3.684528350830078, "device_min_gib": 3.684528350830078, "samples": 29, "added_device_peak_gib": 1.8233375549316406, "torch_peak_allocated_gib": 1.5309290885925293, "torch_peak_reserved_gib": 1.552734375, "warmup_errors": [], "finished": "2026-10-01T14:23:44.505879+00:00", "predictions_sha256": "a887d492fd90d282768bde7bdd880b76cfec1d75434b25285c0a1311b87da4f5", "timing_coverage": 1.0, "plottable": true, "config": "native" }, { "model": "laya-multilingual", "name": "Laya multilingual", "code": "LM", "suite": "industry/routing/en", "n": 40, "correct": 20, "accuracy": 0.5, "interval": [ 0.35199526933465386, 0.6480047306653461 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 21.157699986360967, "p95_ms": 25.841599970590323, "device_peak_gib": 3.643512725830078, "device_min_gib": 3.643512725830078, "samples": 27, "added_device_peak_gib": 1.7823219299316406, "torch_peak_allocated_gib": 1.4945144653320312, "torch_peak_reserved_gib": 1.51171875, "warmup_errors": [], "finished": "2026-10-01T14:23:45.488528+00:00", "predictions_sha256": "4f942650d8923fa3f8a184844351f375a29c3062eff1c726315c5fff20f45530", "timing_coverage": 1.0, "plottable": true, "config": "native" }, { "model": "laya-multilingual", "name": "Laya multilingual", "code": "LM", "suite": "industry/routing/nb", "n": 40, "correct": 18, "accuracy": 0.45, "interval": [ 0.3070530781090492, 0.6017090820106794 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 21.63654996547848, "p95_ms": 27.730749861802906, "device_peak_gib": 3.648395538330078, "device_min_gib": 3.641071319580078, "samples": 27, "added_device_peak_gib": 1.7872047424316406, "torch_peak_allocated_gib": 1.4945917129516602, "torch_peak_reserved_gib": 1.51171875, "warmup_errors": [], "finished": "2026-10-01T14:23:46.491876+00:00", "predictions_sha256": "19fe36fb443e4156b9ae2dea54bc373f17f46eb638cd8d486e146ba4a3a071df", "timing_coverage": 1.0, "plottable": true, "config": "native" }, { "model": "laya-multilingual", "name": "Laya multilingual", "code": "LM", "suite": "industry/documents/en", "n": 28, "correct": 23, "accuracy": 0.8214285714285714, "interval": [ 0.6440857521065705, 0.9212149805439572 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 20.99720004480332, "p95_ms": 29.757900105323642, "device_peak_gib": 3.643512725830078, "device_min_gib": 3.643512725830078, "samples": 27, "added_device_peak_gib": 1.7823219299316406, "torch_peak_allocated_gib": 1.4946389198303223, "torch_peak_reserved_gib": 1.51171875, "warmup_errors": [], "finished": "2026-10-01T14:23:47.458327+00:00", "predictions_sha256": "53dd860da23ec3e131c022b3432ff3a08f5167ad292751289cf461855ab38018", "timing_coverage": 1.0, "plottable": true, "config": "native" }, { "model": "laya-multilingual", "name": "Laya multilingual", "code": "LM", "suite": "industry/documents/nb", "n": 28, "correct": 22, "accuracy": 0.7857142857142857, "interval": [ 0.6046141434627812, 0.8978753966710211 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 21.19030011817813, "p95_ms": 25.559025001712143, "device_peak_gib": 3.643512725830078, "device_min_gib": 3.643512725830078, "samples": 26, "added_device_peak_gib": 1.7823219299316406, "torch_peak_allocated_gib": 1.4946389198303223, "torch_peak_reserved_gib": 1.51171875, "warmup_errors": [], "finished": "2026-10-01T14:23:48.432402+00:00", "predictions_sha256": "0e73a496e7dc71465cdb830e6e221396c59ee9d5f8a3d338d25a6ecde80a8344", "timing_coverage": 1.0, "plottable": true, "config": "native" }, { "model": "laya", "name": "Laya", "code": "L", "suite": "fresh/news", "n": 200, "correct": 183, "accuracy": 0.915, "interval": [ 0.8681041292843443, 0.9462542498225244 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 25.17164999153465, "p95_ms": 36.05407498544082, "device_peak_gib": 4.612262725830078, "device_min_gib": 4.612262725830078, "samples": 53, "added_device_peak_gib": 2.7510719299316406, "torch_peak_allocated_gib": 2.296936511993408, "torch_peak_reserved_gib": 2.48046875, "warmup_errors": [], "finished": "2026-10-01T14:23:05.862664+00:00", "predictions_sha256": "5e2dffb11553d157689ba443dbaf466b5f6e6b3fb325cdcbf3f0f46e9bdb24de", "timing_coverage": 1.0, "plottable": true, "config": "native" }, { "model": "laya", "name": "Laya", "code": "L", "suite": "fresh/emotion", "n": 200, "correct": 121, "accuracy": 0.605, "interval": [ 0.5358834915928536, 0.6701589898679203 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 25.400099926628172, "p95_ms": 34.99753504293039, "device_peak_gib": 4.615192413330078, "device_min_gib": 4.607868194580078, "samples": 53, "added_device_peak_gib": 2.7540016174316406, "torch_peak_allocated_gib": 2.296471118927002, "torch_peak_reserved_gib": 2.478515625, "warmup_errors": [], "finished": "2026-10-01T14:23:07.696746+00:00", "predictions_sha256": "42029a289a3c4f40a5375428215b649c3267454454b1323433499c433c6695c9", "timing_coverage": 1.0, "plottable": true, "config": "native" }, { "model": "laya", "name": "Laya", "code": "L", "suite": "fresh/spam", "n": 200, "correct": 200, "accuracy": 1.0, "interval": [ 0.9811546736227335, 1.0 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 26.230899849906564, "p95_ms": 33.9280349900946, "device_peak_gib": 4.614215850830078, "device_min_gib": 4.612262725830078, "samples": 53, "added_device_peak_gib": 2.7530250549316406, "torch_peak_allocated_gib": 2.298396587371826, "torch_peak_reserved_gib": 2.482421875, "warmup_errors": [], "finished": "2026-10-01T14:23:09.539161+00:00", "predictions_sha256": "df2894e903039cc5af0eb800c04ac9a07ead930271aaf5f33ec52cd5a38b6035", "timing_coverage": 1.0, "plottable": true, "config": "native" }, { "model": "laya", "name": "Laya", "code": "L", "suite": "fresh/phishing", "n": 200, "correct": 200, "accuracy": 1.0, "interval": [ 0.9811546736227335, 1.0 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 25.49100003670901, "p95_ms": 37.01317508239299, "device_peak_gib": 4.612262725830078, "device_min_gib": 4.612262725830078, "samples": 54, "added_device_peak_gib": 2.7510719299316406, "torch_peak_allocated_gib": 2.29823637008667, "torch_peak_reserved_gib": 2.48046875, "warmup_errors": [], "finished": "2026-10-01T14:23:11.389627+00:00", "predictions_sha256": "fe54ddc378f6f18a70c17b32c47fc6fe335fd31027b09f84ac76ba9194979a8f", "timing_coverage": 1.0, "plottable": true, "config": "native" }, { "model": "laya", "name": "Laya", "code": "L", "suite": "massive_en", "n": 2948, "correct": 1962, "accuracy": 0.6655359565807327, "interval": [ 0.6482990705972358, 0.6823419931643413 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 26.541699888184667, "p95_ms": 34.33489503804594, "device_peak_gib": 4.617145538330078, "device_min_gib": 4.609821319580078, "samples": 54, "added_device_peak_gib": 2.7559547424316406, "torch_peak_allocated_gib": 2.297150135040283, "torch_peak_reserved_gib": 2.48046875, "warmup_errors": [], "finished": "2026-10-01T14:23:13.272904+00:00", "predictions_sha256": "cf347b90ae66e7a5f6ca9b1a4bdc063a3d9aa9d957848acbc53e5438c2bcbcc3", "timing_coverage": 1.0, "plottable": true, "config": "native" }, { "model": "laya", "name": "Laya", "code": "L", "suite": "massive_nb", "n": 2948, "correct": 621, "accuracy": 0.21065128900949798, "interval": [ 0.19631283861441726, 0.22574284296626865 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 26.424849987961352, "p95_ms": 29.395264910999682, "device_peak_gib": 4.612262725830078, "device_min_gib": 4.612262725830078, "samples": 53, "added_device_peak_gib": 2.7510719299316406, "torch_peak_allocated_gib": 2.297243595123291, "torch_peak_reserved_gib": 2.48046875, "warmup_errors": [], "finished": "2026-10-01T14:23:15.133563+00:00", "predictions_sha256": "abfc2f1b64b80538874d397b26dd27ef18f350a24d824962dff6ebc2eeaac64c", "timing_coverage": 1.0, "plottable": true, "config": "native" }, { "model": "laya", "name": "Laya", "code": "L", "suite": "public_jevbench", "n": 231, "correct": 135, "accuracy": 0.5844155844155844, "interval": [ 0.5199890497317851, 0.6460804347444514 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 15, "median_ms": 26.933750021271408, "p95_ms": 37.81727516325191, "device_peak_gib": 4.633747100830078, "device_min_gib": 4.612262725830078, "samples": 56, "added_device_peak_gib": 2.7725563049316406, "torch_peak_allocated_gib": 2.3049721717834473, "torch_peak_reserved_gib": 2.501953125, "warmup_errors": [], "finished": "2026-10-01T14:23:17.066216+00:00", "predictions_sha256": "368d52afde52a587c25968882671deb7264f33960260f531b2e8fbe6ad5ec4e2", "timing_coverage": 1.0, "plottable": true, "config": "native" }, { "model": "laya", "name": "Laya", "code": "L", "suite": "kev_claim/devtools-v1", "n": 900, "correct": 439, "accuracy": 0.48777777777777775, "interval": [ 0.45572571368025916, 0.5174247222285053 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 6, "median_ms": 28.037550044246018, "p95_ms": 36.48177505237982, "device_peak_gib": 4.720661163330078, "device_min_gib": 4.711383819580078, "samples": 35, "added_device_peak_gib": 2.8594703674316406, "torch_peak_allocated_gib": 2.3164567947387695, "torch_peak_reserved_gib": 2.583984375, "warmup_errors": [], "finished": "2026-10-01T14:23:18.367861+00:00", "predictions_sha256": "c809d2499c026d9b19ae8a64f868fe4972212294ca4bb835d62c45c9db08e2cc", "timing_coverage": 1.0, "plottable": true, "config": "native" }, { "model": "laya", "name": "Laya", "code": "L", "suite": "industry/routing/en", "n": 40, "correct": 29, "accuracy": 0.725, "interval": [ 0.5716504419378943, 0.8389198375233267 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 25.235850014723837, "p95_ms": 33.4486750070937, "device_peak_gib": 4.612262725830078, "device_min_gib": 4.612262725830078, "samples": 32, "added_device_peak_gib": 2.7510719299316406, "torch_peak_allocated_gib": 2.296631336212158, "torch_peak_reserved_gib": 2.48046875, "warmup_errors": [], "finished": "2026-10-01T14:23:19.546781+00:00", "predictions_sha256": "67dcd8185e5a0a026a6327628ad2a4527fdac82178bc5e4c62fc07b470c9e327", "timing_coverage": 1.0, "plottable": true, "config": "native" }, { "model": "laya", "name": "Laya", "code": "L", "suite": "industry/routing/nb", "n": 40, "correct": 20, "accuracy": 0.5, "interval": [ 0.35199526933465386, 0.6480047306653461 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 26.614350033923984, "p95_ms": 35.21542489761487, "device_peak_gib": 4.612262725830078, "device_min_gib": 4.612262725830078, "samples": 33, "added_device_peak_gib": 2.7510719299316406, "torch_peak_allocated_gib": 2.2966885566711426, "torch_peak_reserved_gib": 2.48046875, "warmup_errors": [], "finished": "2026-10-01T14:23:20.767789+00:00", "predictions_sha256": "ef01dbd9831b9c1306d4c68ba85d68b18053639b6d2a114548f9cb4ce54254e7", "timing_coverage": 1.0, "plottable": true, "config": "native" }, { "model": "laya", "name": "Laya", "code": "L", "suite": "industry/documents/en", "n": 28, "correct": 25, "accuracy": 0.8928571428571429, "interval": [ 0.7280414845132785, 0.9628816331707 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 26.907449937425554, "p95_ms": 33.97032496286556, "device_peak_gib": 4.612262725830078, "device_min_gib": 4.612262725830078, "samples": 33, "added_device_peak_gib": 2.7510719299316406, "torch_peak_allocated_gib": 2.296665668487549, "torch_peak_reserved_gib": 2.48046875, "warmup_errors": [], "finished": "2026-10-01T14:23:21.973558+00:00", "predictions_sha256": "626a8c534d5231b7382736b0a951c0e920884911e3918cef212fb9213c96d703", "timing_coverage": 1.0, "plottable": true, "config": "native" }, { "model": "laya", "name": "Laya", "code": "L", "suite": "industry/documents/nb", "n": 28, "correct": 21, "accuracy": 0.75, "interval": [ 0.5664432968100104, 0.8732350508070668 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 25.62794997356832, "p95_ms": 34.6578499302268, "device_peak_gib": 4.612262725830078, "device_min_gib": 4.612262725830078, "samples": 33, "added_device_peak_gib": 2.7510719299316406, "torch_peak_allocated_gib": 2.296949863433838, "torch_peak_reserved_gib": 2.48046875, "warmup_errors": [], "finished": "2026-10-01T14:23:23.167862+00:00", "predictions_sha256": "382c98b05026b5d154ff563b9daea6e35a52546f3785714c4e7a9bd1bf4ba2d1", "timing_coverage": 1.0, "plottable": true, "config": "native" }, { "model": "nev-2b", "name": "Nev 2B Q8", "code": "NE", "suite": "fresh/news", "n": 200, "correct": 177, "accuracy": 0.885, "interval": [ 0.8333528310151243, 0.9221362676743804 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 36.62455000448972, "p95_ms": 56.37450494104995, "device_peak_gib": 4.1499786376953125, "device_min_gib": 4.1426544189453125, "samples": 79, "added_device_peak_gib": 2.275409698486328, "torch_peak_allocated_gib": 0.0, "torch_peak_reserved_gib": 0.0, "warmup_errors": [], "finished": "2026-10-01T14:43:51.157005+00:00", "predictions_sha256": "365a07675bdcb5008001826c4ba0d1286b5ba5bb2640abaab914ef21e46817fb", "timing_coverage": 1.0, "plottable": true, "config": "Q8_0 GGUF" }, { "model": "nev-2b", "name": "Nev 2B Q8", "code": "NE", "suite": "fresh/emotion", "n": 200, "correct": 123, "accuracy": 0.615, "interval": [ 0.5459986722024608, 0.6796669027307678 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 33.720399951562285, "p95_ms": 52.30315501103176, "device_peak_gib": 4.1450958251953125, "device_min_gib": 4.1450958251953125, "samples": 75, "added_device_peak_gib": 2.270526885986328, "torch_peak_allocated_gib": 0.0, "torch_peak_reserved_gib": 0.0, "warmup_errors": [], "finished": "2026-10-01T14:43:53.776244+00:00", "predictions_sha256": "e8c3bbd0d0049b27305428a7ecc40a0203e99f276d4d1e0dbd1111dfecd289ae", "timing_coverage": 1.0, "plottable": true, "config": "Q8_0 GGUF" }, { "model": "nev-2b", "name": "Nev 2B Q8", "code": "NE", "suite": "fresh/spam", "n": 200, "correct": 166, "accuracy": 0.83, "interval": [ 0.7718411637168737, 0.8757209208741302 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 33.173350035212934, "p95_ms": 61.086389934644096, "device_peak_gib": 4.1515655517578125, "device_min_gib": 4.1442413330078125, "samples": 78, "added_device_peak_gib": 2.276996612548828, "torch_peak_allocated_gib": 0.0, "torch_peak_reserved_gib": 0.0, "warmup_errors": [], "finished": "2026-10-01T14:43:56.523435+00:00", "predictions_sha256": "69139b45020bccfa8102b2d81d6b0a0883191f0855e12567c3233d2d2bf5c5d4", "timing_coverage": 1.0, "plottable": true, "config": "Q8_0 GGUF" }, { "model": "nev-2b", "name": "Nev 2B Q8", "code": "NE", "suite": "fresh/phishing", "n": 200, "correct": 136, "accuracy": 0.68, "interval": [ 0.6124810156588646, 0.7407346668453195 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 33.423650078475475, "p95_ms": 54.92840505903587, "device_peak_gib": 4.1466827392578125, "device_min_gib": 4.1466827392578125, "samples": 78, "added_device_peak_gib": 2.272113800048828, "torch_peak_allocated_gib": 0.0, "torch_peak_reserved_gib": 0.0, "warmup_errors": [], "finished": "2026-10-01T14:43:59.298339+00:00", "predictions_sha256": "72fafed890628a171046dc10b62ec4ab746967a84791fc30647b31878b8c5af2", "timing_coverage": 1.0, "plottable": true, "config": "Q8_0 GGUF" }, { "model": "nev-2b", "name": "Nev 2B Q8", "code": "NE", "suite": "massive_en", "n": 2948, "correct": 2449, "accuracy": 0.8307327001356852, "interval": [ 0.816767908713324, 0.8438366756463843 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 36.253750091418624, "p95_ms": 59.01105493539943, "device_peak_gib": 4.1515655517578125, "device_min_gib": 4.1442413330078125, "samples": 81, "added_device_peak_gib": 2.276996612548828, "torch_peak_allocated_gib": 0.0, "torch_peak_reserved_gib": 0.0, "warmup_errors": [], "finished": "2026-10-01T14:44:02.205840+00:00", "predictions_sha256": "ee2ba0f8c737aa5e46fbe6ac8dca141cbe896506518aec6f2b6e45ba117b3090", "timing_coverage": 1.0, "plottable": true, "config": "Q8_0 GGUF" }, { "model": "nev-2b", "name": "Nev 2B Q8", "code": "NE", "suite": "massive_nb", "n": 2948, "correct": 2183, "accuracy": 0.7405020352781547, "interval": [ 0.7243723204463967, 0.7560057824367145 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 36.321450024843216, "p95_ms": 61.47333501139656, "device_peak_gib": 4.1466827392578125, "device_min_gib": 4.1466827392578125, "samples": 82, "added_device_peak_gib": 2.272113800048828, "torch_peak_allocated_gib": 0.0, "torch_peak_reserved_gib": 0.0, "warmup_errors": [], "finished": "2026-10-01T14:44:05.080712+00:00", "predictions_sha256": "f3e4492011466b68394a8598bd0a3fea71542a1ea040960dc11ed0bee485e5f7", "timing_coverage": 1.0, "plottable": true, "config": "Q8_0 GGUF" }, { "model": "nev-2b", "name": "Nev 2B Q8", "code": "NE", "suite": "public_jevbench", "n": 231, "correct": 186, "accuracy": 0.8051948051948052, "interval": [ 0.7493033796887764, 0.8511016795714633 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 52.6474000653252, "p95_ms": 205.3510898724198, "device_peak_gib": 4.1530303955078125, "device_min_gib": 4.1457061767578125, "samples": 131, "added_device_peak_gib": 2.278461456298828, "torch_peak_allocated_gib": 0.0, "torch_peak_reserved_gib": 0.0, "warmup_errors": [], "finished": "2026-10-01T14:44:09.510684+00:00", "predictions_sha256": "780c8158ec09dcc1ede0b1eeb35af72e13612a0a7006026b5cc02542943eed3d", "timing_coverage": 1.0, "plottable": true, "config": "Q8_0 GGUF" }, { "model": "nev-2b", "name": "Nev 2B Q8", "code": "NE", "suite": "kev_claim/devtools-v1", "n": 900, "correct": 461, "accuracy": 0.5122222222222222, "interval": [ 0.48375, 0.5419764740379843 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 53.33129991777241, "p95_ms": 118.4470250736922, "device_peak_gib": 4.1481475830078125, "device_min_gib": 4.1481475830078125, "samples": 75, "added_device_peak_gib": 2.273578643798828, "torch_peak_allocated_gib": 0.0, "torch_peak_reserved_gib": 0.0, "warmup_errors": [], "finished": "2026-10-01T14:44:12.415881+00:00", "predictions_sha256": "20c07f4fe6a5a1c83e2b72b86467b129e329d470916d98b3b6b15f2ab5b85bda", "timing_coverage": 1.0, "plottable": true, "config": "Q8_0 GGUF" }, { "model": "nev-2b", "name": "Nev 2B Q8", "code": "NE", "suite": "industry/routing/en", "n": 40, "correct": 34, "accuracy": 0.85, "interval": [ 0.709276756335103, 0.9293881228267964 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 38.44619996380061, "p95_ms": 61.085599998477846, "device_peak_gib": 4.1530303955078125, "device_min_gib": 4.1457061767578125, "samples": 53, "added_device_peak_gib": 2.278461456298828, "torch_peak_allocated_gib": 0.0, "torch_peak_reserved_gib": 0.0, "warmup_errors": [], "finished": "2026-10-01T14:44:14.443362+00:00", "predictions_sha256": "7d20430dc5913ec939489e0256f0bd046366a76275e746bbcf47bd56f9b4495f", "timing_coverage": 1.0, "plottable": true, "config": "Q8_0 GGUF" }, { "model": "nev-2b", "name": "Nev 2B Q8", "code": "NE", "suite": "industry/routing/nb", "n": 40, "correct": 34, "accuracy": 0.85, "interval": [ 0.709276756335103, 0.9293881228267964 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 38.15709997434169, "p95_ms": 56.17852497380227, "device_peak_gib": 4.1481475830078125, "device_min_gib": 4.1481475830078125, "samples": 49, "added_device_peak_gib": 2.273578643798828, "torch_peak_allocated_gib": 0.0, "torch_peak_reserved_gib": 0.0, "warmup_errors": [], "finished": "2026-10-01T14:44:16.335567+00:00", "predictions_sha256": "d9dc18224e35821f5c24b02f4691dcb78a3edc83e23a28d34ed65afee4af5f6c", "timing_coverage": 1.0, "plottable": true, "config": "Q8_0 GGUF" }, { "model": "nev-2b", "name": "Nev 2B Q8", "code": "NE", "suite": "industry/documents/en", "n": 28, "correct": 28, "accuracy": 1.0, "interval": [ 0.8793566952341543, 0.9999999999999998 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 44.55879994202405, "p95_ms": 64.77045000065118, "device_peak_gib": 4.1481475830078125, "device_min_gib": 4.1481475830078125, "samples": 56, "added_device_peak_gib": 2.273578643798828, "torch_peak_allocated_gib": 0.0, "torch_peak_reserved_gib": 0.0, "warmup_errors": [], "finished": "2026-10-01T14:44:18.443706+00:00", "predictions_sha256": "6a01fa7ee34bf6d37dc1fbd9ef26b353b1fc8fe9ea9e1705de2493892366c422", "timing_coverage": 1.0, "plottable": true, "config": "Q8_0 GGUF" }, { "model": "nev-2b", "name": "Nev 2B Q8", "code": "NE", "suite": "industry/documents/nb", "n": 28, "correct": 27, "accuracy": 0.9642857142857143, "interval": [ 0.8228780225664668, 0.9936674801509623 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 37.27309999521822, "p95_ms": 52.771099959500134, "device_peak_gib": 4.1481475830078125, "device_min_gib": 4.1457061767578125, "samples": 46, "added_device_peak_gib": 2.273578643798828, "torch_peak_allocated_gib": 0.0, "torch_peak_reserved_gib": 0.0, "warmup_errors": [], "finished": "2026-10-01T14:44:20.304868+00:00", "predictions_sha256": "c9f70b92e9b9615fede9e20a7738827f70f7757eeb7cd32946034ffecead24a0", "timing_coverage": 1.0, "plottable": true, "config": "Q8_0 GGUF" }, { "model": "nimble-9b", "name": "Nimble 9B NF4", "code": "NI", "suite": "fresh/news", "n": 200, "correct": 176, "accuracy": 0.88, "interval": [ 0.8276574790170326, 0.9180200729362449 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 125.59045001398772, "p95_ms": 142.48327012173831, "device_peak_gib": 10.204696655273438, "device_min_gib": 10.197372436523438, "samples": 245, "added_device_peak_gib": 8.33017349243164, "torch_peak_allocated_gib": 7.972168445587158, "torch_peak_reserved_gib": 8.0546875, "warmup_errors": [], "finished": "2026-10-01T14:44:58.941036+00:00", "predictions_sha256": "8b505ec1e60bdc22fc8e900eb1d66a069500e54053db7a28424a06e723d6ed6b", "timing_coverage": 1.0, "plottable": true, "config": "NF4 base with bf16 compute and unmerged LoRA" }, { "model": "nimble-9b", "name": "Nimble 9B NF4", "code": "NI", "suite": "fresh/emotion", "n": 200, "correct": 118, "accuracy": 0.59, "interval": [ 0.5207645903369208, 0.6558432509151711 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 135.85990003775805, "p95_ms": 146.5798548539169, "device_peak_gib": 10.202743530273438, "device_min_gib": 10.195419311523438, "samples": 266, "added_device_peak_gib": 8.32822036743164, "torch_peak_allocated_gib": 7.9720587730407715, "torch_peak_reserved_gib": 8.052734375, "warmup_errors": [], "finished": "2026-10-01T14:45:08.320222+00:00", "predictions_sha256": "481c54d0677db150d3dd9071b3fa631f9cf90e710748d951d30194403da872f2", "timing_coverage": 1.0, "plottable": true, "config": "NF4 base with bf16 compute and unmerged LoRA" }, { "model": "nimble-9b", "name": "Nimble 9B NF4", "code": "NI", "suite": "fresh/spam", "n": 200, "correct": 165, "accuracy": 0.825, "interval": [ 0.7663556885175099, 0.8713948493372666 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 132.07289995625615, "p95_ms": 146.4576699538156, "device_peak_gib": 10.204696655273438, "device_min_gib": 10.197372436523438, "samples": 249, "added_device_peak_gib": 8.33017349243164, "torch_peak_allocated_gib": 7.979978084564209, "torch_peak_reserved_gib": 8.0546875, "warmup_errors": [], "finished": "2026-10-01T14:45:17.068810+00:00", "predictions_sha256": "6f55da8c748341edc68f75f8abd45f1dd31fbf46f87089e17b0f1ad90788287d", "timing_coverage": 1.0, "plottable": true, "config": "NF4 base with bf16 compute and unmerged LoRA" }, { "model": "nimble-9b", "name": "Nimble 9B NF4", "code": "NI", "suite": "fresh/phishing", "n": 200, "correct": 170, "accuracy": 0.85, "interval": [ 0.7939442071583334, 0.89286406437758 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 132.5845499522984, "p95_ms": 145.4209549468942, "device_peak_gib": 10.204696655273438, "device_min_gib": 10.197372436523438, "samples": 249, "added_device_peak_gib": 8.33017349243164, "torch_peak_allocated_gib": 7.977356910705566, "torch_peak_reserved_gib": 8.0546875, "warmup_errors": [], "finished": "2026-10-01T14:45:25.870304+00:00", "predictions_sha256": "d36df71ebeb6d628daff98dec90f3ab109b6d4e8c73e04b9dd3a992992788681", "timing_coverage": 1.0, "plottable": true, "config": "NF4 base with bf16 compute and unmerged LoRA" }, { "model": "nimble-9b", "name": "Nimble 9B NF4", "code": "NI", "suite": "massive_en", "n": 2948, "correct": 2411, "accuracy": 0.817842605156038, "interval": [ 0.8034989632345869, 0.8313589809141791 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 171.05414997786283, "p95_ms": 180.96960507100448, "device_peak_gib": 10.204696655273438, "device_min_gib": 10.197372436523438, "samples": 333, "added_device_peak_gib": 8.33017349243164, "torch_peak_allocated_gib": 7.989602088928223, "torch_peak_reserved_gib": 8.0546875, "warmup_errors": [], "finished": "2026-10-01T14:45:37.597704+00:00", "predictions_sha256": "cd8668b2417fcafd4bb1c56d72dd9c808238a52155acd67ebbbfc29aa3131d63", "timing_coverage": 1.0, "plottable": true, "config": "NF4 base with bf16 compute and unmerged LoRA" }, { "model": "nimble-9b", "name": "Nimble 9B NF4", "code": "NI", "suite": "massive_nb", "n": 2948, "correct": 2181, "accuracy": 0.7398236092265943, "interval": [ 0.7236813945294366, 0.7553416220267827 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 171.38705006800592, "p95_ms": 183.4796148585155, "device_peak_gib": 10.204696655273438, "device_min_gib": 10.197372436523438, "samples": 337, "added_device_peak_gib": 8.33017349243164, "torch_peak_allocated_gib": 7.989556312561035, "torch_peak_reserved_gib": 8.0546875, "warmup_errors": [], "finished": "2026-10-01T14:45:49.446852+00:00", "predictions_sha256": "a947d1ec13286f5c6c62be1b9471dff9951123045325da36adb0fad15d0482b3", "timing_coverage": 1.0, "plottable": true, "config": "NF4 base with bf16 compute and unmerged LoRA" }, { "model": "nimble-9b", "name": "Nimble 9B NF4", "code": "NI", "suite": "public_jevbench", "n": 231, "correct": 181, "accuracy": 0.7835497835497836, "interval": [ 0.7260365682646376, 0.8317865718991312 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 151.66884998325258, "p95_ms": 1188.742780056782, "device_peak_gib": 13.497665405273438, "device_min_gib": 10.197372436523438, "samples": 692, "added_device_peak_gib": 11.62314224243164, "torch_peak_allocated_gib": 9.883522033691406, "torch_peak_reserved_gib": 11.34765625, "warmup_errors": [], "finished": "2026-10-01T14:46:12.105358+00:00", "predictions_sha256": "9c2c24ab2d14f36f8d5849aadae8c4befe910cc0df29b732ec47d550fb9d9e92", "timing_coverage": 1.0, "plottable": true, "config": "NF4 base with bf16 compute and unmerged LoRA" }, { "model": "nimble-9b", "name": "Nimble 9B NF4", "code": "NI", "suite": "kev_claim/devtools-v1", "n": 900, "correct": 584, "accuracy": 0.6488888888888888, "interval": [ 0.6087704076621753, 0.6976509850307139 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 151.66085003875196, "p95_ms": 463.92785001080483, "device_peak_gib": 10.786727905273438, "device_min_gib": 10.555282592773438, "samples": 254, "added_device_peak_gib": 8.91220474243164, "torch_peak_allocated_gib": 8.271111011505127, "torch_peak_reserved_gib": 8.63671875, "warmup_errors": [], "finished": "2026-10-01T14:46:21.991838+00:00", "predictions_sha256": "72746e065f5d484f0ca07a33e5282618e6175a64ba9c4c97b413f887fe7d67fe", "timing_coverage": 1.0, "plottable": true, "config": "NF4 base with bf16 compute and unmerged LoRA" }, { "model": "nimble-9b", "name": "Nimble 9B NF4", "code": "NI", "suite": "industry/routing/en", "n": 40, "correct": 39, "accuracy": 0.975, "interval": [ 0.8711863103652591, 0.9955731684973186 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 134.51284996699542, "p95_ms": 143.74117500847206, "device_peak_gib": 10.202743530273438, "device_min_gib": 10.195419311523438, "samples": 158, "added_device_peak_gib": 8.32822036743164, "torch_peak_allocated_gib": 7.9723219871521, "torch_peak_reserved_gib": 8.052734375, "warmup_errors": [], "finished": "2026-10-01T14:46:28.006008+00:00", "predictions_sha256": "d095fc7194aaf970b2fdeeaf64d390de86481368d19e2d37795c9576fec0bf65", "timing_coverage": 1.0, "plottable": true, "config": "NF4 base with bf16 compute and unmerged LoRA" }, { "model": "nimble-9b", "name": "Nimble 9B NF4", "code": "NI", "suite": "industry/routing/nb", "n": 40, "correct": 39, "accuracy": 0.975, "interval": [ 0.8711863103652591, 0.9955731684973186 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 133.8287501130253, "p95_ms": 143.88767501804978, "device_peak_gib": 10.202743530273438, "device_min_gib": 10.195419311523438, "samples": 158, "added_device_peak_gib": 8.32822036743164, "torch_peak_allocated_gib": 7.973574161529541, "torch_peak_reserved_gib": 8.052734375, "warmup_errors": [], "finished": "2026-10-01T14:46:33.978446+00:00", "predictions_sha256": "698f5835279fb69cecbcc06a772fb3d1ff6b41fa3db17ceccb6f00e5f348647e", "timing_coverage": 1.0, "plottable": true, "config": "NF4 base with bf16 compute and unmerged LoRA" }, { "model": "nimble-9b", "name": "Nimble 9B NF4", "code": "NI", "suite": "industry/documents/en", "n": 28, "correct": 28, "accuracy": 1.0, "interval": [ 0.8793566952341543, 0.9999999999999998 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 135.33764996100217, "p95_ms": 145.53930016700178, "device_peak_gib": 10.202743530273438, "device_min_gib": 10.195419311523438, "samples": 160, "added_device_peak_gib": 8.32822036743164, "torch_peak_allocated_gib": 7.972245693206787, "torch_peak_reserved_gib": 8.052734375, "warmup_errors": [], "finished": "2026-10-01T14:46:40.027871+00:00", "predictions_sha256": "de6e4fb000be2b0dc9130966d8bc650dcbce99076c243f7aa2f3662808634892", "timing_coverage": 1.0, "plottable": true, "config": "NF4 base with bf16 compute and unmerged LoRA" }, { "model": "nimble-9b", "name": "Nimble 9B NF4", "code": "NI", "suite": "industry/documents/nb", "n": 28, "correct": 28, "accuracy": 1.0, "interval": [ 0.8793566952341543, 0.9999999999999998 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 135.81264996901155, "p95_ms": 146.9138499815017, "device_peak_gib": 10.202743530273438, "device_min_gib": 10.195419311523438, "samples": 161, "added_device_peak_gib": 8.32822036743164, "torch_peak_allocated_gib": 7.972168445587158, "torch_peak_reserved_gib": 8.052734375, "warmup_errors": [], "finished": "2026-10-01T14:46:46.088292+00:00", "predictions_sha256": "4c507aa666951a76c90753ac52204c5abf1861e3806a72eb5f7027bfb612a5e3", "timing_coverage": 1.0, "plottable": true, "config": "NF4 base with bf16 compute and unmerged LoRA" }, { "model": "plumb-4b", "name": "Plumb 4B", "code": "PL", "suite": "fresh/news", "n": 200, "correct": 171, "accuracy": 0.855, "interval": [ 0.7995133246921847, 0.8971064935799559 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 54.34254987630993, "p95_ms": 67.59221997344866, "device_peak_gib": 10.069931030273438, "device_min_gib": 10.062606811523438, "samples": 109, "added_device_peak_gib": 8.19540786743164, "torch_peak_allocated_gib": 7.898333549499512, "torch_peak_reserved_gib": 7.919921875, "warmup_errors": [], "finished": "2026-10-01T14:47:05.701831+00:00", "predictions_sha256": "220adac2a1067060d2ea6946ef94e5924ca858a24417f40bc60100a82d402d80", "timing_coverage": 1.0, "plottable": true, "config": "bf16" }, { "model": "plumb-4b", "name": "Plumb 4B", "code": "PL", "suite": "fresh/emotion", "n": 200, "correct": 116, "accuracy": 0.58, "interval": [ 0.5107207393958046, 0.6462640083838327 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 54.12480002269149, "p95_ms": 64.76515005342662, "device_peak_gib": 10.045516967773438, "device_min_gib": 10.045516967773438, "samples": 109, "added_device_peak_gib": 8.17099380493164, "torch_peak_allocated_gib": 7.894337177276611, "torch_peak_reserved_gib": 7.900390625, "warmup_errors": [], "finished": "2026-10-01T14:47:09.500308+00:00", "predictions_sha256": "502bbe499e595fdd21ddf56cef0d86ff93d20dbbdb0c15b52141e21778b3a86f", "timing_coverage": 1.0, "plottable": true, "config": "bf16" }, { "model": "plumb-4b", "name": "Plumb 4B", "code": "PL", "suite": "fresh/spam", "n": 200, "correct": 184, "accuracy": 0.92, "interval": [ 0.874010511732856, 0.9501594141102401 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 54.04839990660548, "p95_ms": 62.92427013395354, "device_peak_gib": 10.092391967773438, "device_min_gib": 10.070419311523438, "samples": 109, "added_device_peak_gib": 8.21786880493164, "torch_peak_allocated_gib": 7.920492172241211, "torch_peak_reserved_gib": 7.947265625, "warmup_errors": [], "finished": "2026-10-01T14:47:13.342327+00:00", "predictions_sha256": "a5a6405bd58f24dea31f5ed6bf132984631d68dc5a954c48d353caaf0fb2ab31", "timing_coverage": 1.0, "plottable": true, "config": "bf16" }, { "model": "plumb-4b", "name": "Plumb 4B", "code": "PL", "suite": "fresh/phishing", "n": 200, "correct": 183, "accuracy": 0.915, "interval": [ 0.8681041292843443, 0.9462542498225244 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 53.64065000321716, "p95_ms": 64.90954500623047, "device_peak_gib": 10.071884155273438, "device_min_gib": 10.064559936523438, "samples": 108, "added_device_peak_gib": 8.19736099243164, "torch_peak_allocated_gib": 7.905336380004883, "torch_peak_reserved_gib": 7.921875, "warmup_errors": [], "finished": "2026-10-01T14:47:17.114159+00:00", "predictions_sha256": "42494bf167a99dd774c0735883317894ca8af07d7fb47c1c8f6a23bb2366e031", "timing_coverage": 1.0, "plottable": true, "config": "bf16" }, { "model": "plumb-4b", "name": "Plumb 4B", "code": "PL", "suite": "massive_en", "n": 2948, "correct": 2438, "accuracy": 0.8270013568521031, "interval": [ 0.8129240920651353, 0.8402275174966687 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 173.9831999875605, "p95_ms": 183.52025500498712, "device_peak_gib": 10.089462280273438, "device_min_gib": 10.082138061523438, "samples": 335, "added_device_peak_gib": 8.21493911743164, "torch_peak_allocated_gib": 7.919917106628418, "torch_peak_reserved_gib": 7.939453125, "warmup_errors": [], "finished": "2026-10-01T14:47:28.947746+00:00", "predictions_sha256": "7610194853ef35066d00bbc36511e4d205d1af330f6053bf62cb616b0c06ad65", "timing_coverage": 1.0, "plottable": true, "config": "bf16" }, { "model": "plumb-4b", "name": "Plumb 4B", "code": "PL", "suite": "massive_nb", "n": 2948, "correct": 2174, "accuracy": 0.737449118046133, "interval": [ 0.7212635393635586, 0.7530166750485398 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 172.31234989594668, "p95_ms": 190.85651013301685, "device_peak_gib": 10.089462280273438, "device_min_gib": 10.082138061523438, "samples": 335, "added_device_peak_gib": 8.21493911743164, "torch_peak_allocated_gib": 7.920492172241211, "torch_peak_reserved_gib": 7.939453125, "warmup_errors": [], "finished": "2026-10-01T14:47:40.733403+00:00", "predictions_sha256": "cc098f5ec16b96ec10fd35c7f92fd52587efd2d0201a070c6797e2ac16d0162c", "timing_coverage": 1.0, "plottable": true, "config": "bf16" }, { "model": "plumb-4b", "name": "Plumb 4B", "code": "PL", "suite": "public_jevbench", "n": 231, "correct": 207, "accuracy": 0.8961038961038961, "interval": [ 0.8500656335984398, 0.9291834858669776 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 63.21749999187887, "p95_ms": 536.4041551132684, "device_peak_gib": 13.194931030273438, "device_min_gib": 10.065048217773438, "samples": 323, "added_device_peak_gib": 11.32040786743164, "torch_peak_allocated_gib": 9.793753147125244, "torch_peak_reserved_gib": 11.044921875, "warmup_errors": [], "finished": "2026-10-01T14:47:51.270024+00:00", "predictions_sha256": "8113eb8051131fbfe278efda0349622c6f91de694575a788f2e9a103558eb416", "timing_coverage": 1.0, "plottable": true, "config": "bf16" }, { "model": "plumb-4b", "name": "Plumb 4B", "code": "PL", "suite": "kev_claim/devtools-v1", "n": 900, "correct": 579, "accuracy": 0.6433333333333333, "interval": [ 0.597340833046687, 0.6859743696830483 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 65.95834996551275, "p95_ms": 172.4684999207966, "device_peak_gib": 10.645530700683594, "device_min_gib": 10.512107849121094, "samples": 100, "added_device_peak_gib": 8.618961334228516, "torch_peak_allocated_gib": 8.120352745056152, "torch_peak_reserved_gib": 8.34375, "warmup_errors": [], "finished": "2026-10-04T18:43:25.129062+00:00", "predictions_sha256": "38775df9485141347da7f444075b054ed902bf7e69ec435b283c9525065a320e", "timing_coverage": 1.0, "plottable": true, "config": "bf16" }, { "model": "plumb-4b", "name": "Plumb 4B", "code": "PL", "suite": "industry/routing/en", "n": 40, "correct": 38, "accuracy": 0.95, "interval": [ 0.8349612263085903, 0.9861793326138516 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 60.087749967351556, "p95_ms": 71.60602498333901, "device_peak_gib": 10.063095092773438, "device_min_gib": 10.063095092773438, "samples": 72, "added_device_peak_gib": 8.18857192993164, "torch_peak_allocated_gib": 7.898271560668945, "torch_peak_reserved_gib": 7.91796875, "warmup_errors": [], "finished": "2026-10-01T14:47:53.917277+00:00", "predictions_sha256": "60f3a2c0f27b8034283cc67e79cffeaecb1a5568acf569238d9536a3f65714c4", "timing_coverage": 1.0, "plottable": true, "config": "bf16" }, { "model": "plumb-4b", "name": "Plumb 4B", "code": "PL", "suite": "industry/routing/nb", "n": 40, "correct": 37, "accuracy": 0.925, "interval": [ 0.8013576647568946, 0.9741639742254119 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 56.73800001386553, "p95_ms": 73.14379996387288, "device_peak_gib": 10.069931030273438, "device_min_gib": 10.062606811523438, "samples": 68, "added_device_peak_gib": 8.19540786743164, "torch_peak_allocated_gib": 7.899028778076172, "torch_peak_reserved_gib": 7.919921875, "warmup_errors": [], "finished": "2026-10-01T14:47:56.496328+00:00", "predictions_sha256": "f1c1d02dc62516c141fd7d7f06f8184de34777cec4bbea2dc4921c4f99dfacbb", "timing_coverage": 1.0, "plottable": true, "config": "bf16" }, { "model": "plumb-4b", "name": "Plumb 4B", "code": "PL", "suite": "industry/documents/en", "n": 28, "correct": 27, "accuracy": 0.9642857142857143, "interval": [ 0.8228780225664668, 0.9936674801509623 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 59.48374990839511, "p95_ms": 76.57012500567362, "device_peak_gib": 10.067001342773438, "device_min_gib": 10.067001342773438, "samples": 73, "added_device_peak_gib": 8.19247817993164, "torch_peak_allocated_gib": 7.898556709289551, "torch_peak_reserved_gib": 7.921875, "warmup_errors": [], "finished": "2026-10-01T14:47:59.203874+00:00", "predictions_sha256": "cd28a6fb7590013114a880cac2979cb15b6566fddda96b982d50ad23dc18e844", "timing_coverage": 1.0, "plottable": true, "config": "bf16" }, { "model": "plumb-4b", "name": "Plumb 4B", "code": "PL", "suite": "industry/documents/nb", "n": 28, "correct": 26, "accuracy": 0.9285714285714286, "interval": [ 0.7735463743160095, 0.9801879358846942 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 58.635199908167124, "p95_ms": 74.14735003840178, "device_peak_gib": 10.067977905273438, "device_min_gib": 10.060653686523438, "samples": 71, "added_device_peak_gib": 8.19345474243164, "torch_peak_allocated_gib": 7.898438930511475, "torch_peak_reserved_gib": 7.91796875, "warmup_errors": [], "finished": "2026-10-01T14:48:01.788492+00:00", "predictions_sha256": "e316b6efd6ef4014117066ff44768231a5d1c182264e680efa534e114b77f183", "timing_coverage": 1.0, "plottable": true, "config": "bf16" }, { "model": "tev1", "name": "Tev1 4B", "code": "TE", "suite": "fresh/news", "n": 200, "correct": 175, "accuracy": 0.875, "interval": [ 0.8219857499741765, 0.9138802552428735 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 54.168849950656295, "p95_ms": 62.71319009829312, "device_peak_gib": 10.071884155273438, "device_min_gib": 10.062606811523438, "samples": 108, "added_device_peak_gib": 8.19736099243164, "torch_peak_allocated_gib": 7.90053653717041, "torch_peak_reserved_gib": 7.921875, "warmup_errors": [], "finished": "2026-10-01T14:48:20.800496+00:00", "predictions_sha256": "bf5dae7a9ef9cbd449f90c6916d294f23fa6f59b65a9d180c138d80de2dc786c", "timing_coverage": 1.0, "plottable": true, "config": "bf16" }, { "model": "tev1", "name": "Tev1 4B", "code": "TE", "suite": "fresh/emotion", "n": 200, "correct": 118, "accuracy": 0.59, "interval": [ 0.5207645903369208, 0.6558432509151711 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 54.21360011678189, "p95_ms": 67.36903997370973, "device_peak_gib": 10.071884155273438, "device_min_gib": 10.064559936523438, "samples": 109, "added_device_peak_gib": 8.19736099243164, "torch_peak_allocated_gib": 7.899068355560303, "torch_peak_reserved_gib": 7.921875, "warmup_errors": [], "finished": "2026-10-01T14:48:24.588421+00:00", "predictions_sha256": "891e6921ddaf943cae3f128ed7f786cf14509ed30f7975a78d7af239eec4af8b", "timing_coverage": 1.0, "plottable": true, "config": "bf16" }, { "model": "tev1", "name": "Tev1 4B", "code": "TE", "suite": "fresh/spam", "n": 200, "correct": 190, "accuracy": 0.95, "interval": [ 0.9104218518612239, 0.972617354399236 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 54.16374991182238, "p95_ms": 65.37364504765719, "device_peak_gib": 10.092391967773438, "device_min_gib": 10.072860717773438, "samples": 110, "added_device_peak_gib": 8.21786880493164, "torch_peak_allocated_gib": 7.922766208648682, "torch_peak_reserved_gib": 7.947265625, "warmup_errors": [], "finished": "2026-10-01T14:48:28.441816+00:00", "predictions_sha256": "659147d7efba911f70a29020489690dd8686a4891e9df9b974cdfeabae540c9f", "timing_coverage": 1.0, "plottable": true, "config": "bf16" }, { "model": "tev1", "name": "Tev1 4B", "code": "TE", "suite": "fresh/phishing", "n": 200, "correct": 175, "accuracy": 0.875, "interval": [ 0.8219857499741765, 0.9138802552428735 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 54.083799943327904, "p95_ms": 62.88180485134944, "device_peak_gib": 10.069931030273438, "device_min_gib": 10.062606811523438, "samples": 108, "added_device_peak_gib": 8.19540786743164, "torch_peak_allocated_gib": 7.908453464508057, "torch_peak_reserved_gib": 7.919921875, "warmup_errors": [], "finished": "2026-10-01T14:48:32.207709+00:00", "predictions_sha256": "a873a06377bd8e884f675b6e5526d8a294f9e9e6a4a65993d11de4c2f8e1ae08", "timing_coverage": 1.0, "plottable": true, "config": "bf16" }, { "model": "tev1", "name": "Tev1 4B", "code": "TE", "suite": "massive_en", "n": 2948, "correct": 2421, "accuracy": 0.8212347354138398, "interval": [ 0.8069881793407908, 0.8346451964424336 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 70.32759999856353, "p95_ms": 82.4728151434101, "device_peak_gib": 10.442977905273438, "device_min_gib": 10.433639526367188, "samples": 143, "added_device_peak_gib": 8.23770523071289, "torch_peak_allocated_gib": 7.942733287811279, "torch_peak_reserved_gib": 7.966796875, "warmup_errors": [], "finished": "2026-10-05T00:47:48.197741+00:00", "predictions_sha256": "38816ab84feed5af676952bbf6fcb90ca7ee10e254993c2c08e5685126d06d93", "timing_coverage": 1.0, "plottable": true, "config": "bf16" }, { "model": "tev1", "name": "Tev1 4B", "code": "TE", "suite": "massive_nb", "n": 2948, "correct": 2143, "accuracy": 0.7269335142469471, "interval": [ 0.7105629866559836, 0.7427133896892931 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 77.85110000986606, "p95_ms": 89.33886996237561, "device_peak_gib": 10.440902709960938, "device_min_gib": 10.430709838867188, "samples": 149, "added_device_peak_gib": 8.23563003540039, "torch_peak_allocated_gib": 7.943652629852295, "torch_peak_reserved_gib": 7.96484375, "warmup_errors": [], "finished": "2026-10-05T00:47:53.430286+00:00", "predictions_sha256": "c82f35bc9608a153a2b99adf07533471a5dcf41f63b0215d13749aa4312587d9", "timing_coverage": 1.0, "plottable": true, "config": "bf16" }, { "model": "tev1", "name": "Tev1 4B", "code": "TE", "suite": "public_jevbench", "n": 231, "correct": 174, "accuracy": 0.7532467532467533, "interval": [ 0.6938095945211326, 0.8043988589075769 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 63.193549984134734, "p95_ms": 604.8848951002583, "device_peak_gib": 13.554977416992188, "device_min_gib": 10.407516479492188, "samples": 329, "added_device_peak_gib": 11.34970474243164, "torch_peak_allocated_gib": 9.806066036224365, "torch_peak_reserved_gib": 11.07421875, "warmup_errors": [], "finished": "2026-10-05T00:48:04.197198+00:00", "predictions_sha256": "7481a42502729dc352e6cc99c5e3e90e9d29e6cde89deeb705116453a45797df", "timing_coverage": 1.0, "plottable": true, "config": "bf16" }, { "model": "tev1", "name": "Tev1 4B", "code": "TE", "suite": "kev_claim/devtools-v1", "n": 900, "correct": 584, "accuracy": 0.6488888888888888, "interval": [ 0.6084470125199947, 0.6957499313686875 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 66.34139991365373, "p95_ms": 178.10870002722368, "device_peak_gib": 10.784774780273438, "device_min_gib": 10.638839721679688, "samples": 104, "added_device_peak_gib": 8.57950210571289, "torch_peak_allocated_gib": 8.127182483673096, "torch_peak_reserved_gib": 8.30859375, "warmup_errors": [], "finished": "2026-10-05T00:48:08.186799+00:00", "predictions_sha256": "09192268a6ca9ccb82cd64ea07378a4ad2ba7eb9c2c46d6e94fa683fa274cb99", "timing_coverage": 1.0, "plottable": true, "config": "bf16" }, { "model": "tev1", "name": "Tev1 4B", "code": "TE", "suite": "industry/routing/en", "n": 40, "correct": 39, "accuracy": 0.975, "interval": [ 0.8711863103652591, 0.9955731684973186 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 53.983650053851306, "p95_ms": 63.1537000881508, "device_peak_gib": 10.065048217773438, "device_min_gib": 10.065048217773438, "samples": 66, "added_device_peak_gib": 8.19052505493164, "torch_peak_allocated_gib": 7.900622844696045, "torch_peak_reserved_gib": 7.919921875, "warmup_errors": [], "finished": "2026-10-01T14:48:34.653931+00:00", "predictions_sha256": "fd86fab64757b8a12c6af120276d99af6b67c8265c2940720cb1982641eaa4e6", "timing_coverage": 1.0, "plottable": true, "config": "bf16" }, { "model": "tev1", "name": "Tev1 4B", "code": "TE", "suite": "industry/routing/nb", "n": 40, "correct": 37, "accuracy": 0.925, "interval": [ 0.8013576647568946, 0.9741639742254119 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 54.05784992035478, "p95_ms": 73.5549500095658, "device_peak_gib": 10.069931030273438, "device_min_gib": 10.062606811523438, "samples": 68, "added_device_peak_gib": 8.19540786743164, "torch_peak_allocated_gib": 7.901787281036377, "torch_peak_reserved_gib": 7.919921875, "warmup_errors": [], "finished": "2026-10-01T14:48:37.152196+00:00", "predictions_sha256": "dd9b323f0a784ed9f8ffead182994e0dd540f564fb840312b9774e23f3bb1343", "timing_coverage": 1.0, "plottable": true, "config": "bf16" }, { "model": "tev1", "name": "Tev1 4B", "code": "TE", "suite": "industry/documents/en", "n": 28, "correct": 27, "accuracy": 0.9642857142857143, "interval": [ 0.8228780225664668, 0.9936674801509623 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 52.500750054605305, "p95_ms": 61.53679988346994, "device_peak_gib": 10.065048217773438, "device_min_gib": 10.065048217773438, "samples": 63, "added_device_peak_gib": 8.19052505493164, "torch_peak_allocated_gib": 7.90072774887085, "torch_peak_reserved_gib": 7.919921875, "warmup_errors": [], "finished": "2026-10-01T14:48:39.535396+00:00", "predictions_sha256": "8bcb36f261b9e64c7fb1913d015f4cc82fe7526cfb3451bf7a093ac2b90198cb", "timing_coverage": 1.0, "plottable": true, "config": "bf16" }, { "model": "tev1", "name": "Tev1 4B", "code": "TE", "suite": "industry/documents/nb", "n": 28, "correct": 28, "accuracy": 1.0, "interval": [ 0.8793566952341543, 0.9999999999999998 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 54.28115010727197, "p95_ms": 64.89102484192699, "device_peak_gib": 10.069931030273438, "device_min_gib": 10.062606811523438, "samples": 66, "added_device_peak_gib": 8.19540786743164, "torch_peak_allocated_gib": 7.900609016418457, "torch_peak_reserved_gib": 7.919921875, "warmup_errors": [], "finished": "2026-10-01T14:48:41.981342+00:00", "predictions_sha256": "68e65dbd4e1aa274f98f0b5ef55290ec41c16f1117229e2852f58acc3dc593ad", "timing_coverage": 1.0, "plottable": true, "config": "bf16" }, { "model": "von", "name": "Von", "code": "V", "suite": "fresh/news", "n": 200, "correct": 165, "accuracy": 0.825, "interval": [ 0.7663556885175099, 0.8713948493372666 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 23.08009995613247, "p95_ms": 39.053575007710606, "device_peak_gib": 5.2798919677734375, "device_min_gib": 5.2798919677734375, "samples": 50, "added_device_peak_gib": 3.4053688049316406, "torch_peak_allocated_gib": 1.5106873512268066, "torch_peak_reserved_gib": 3.134765625, "warmup_errors": [], "finished": "2026-10-01T14:48:52.313220+00:00", "predictions_sha256": "f9a41b99750b9c391c40ebfa975203d115b55356fd966ded7b97f42289d63f01", "timing_coverage": 1.0, "plottable": true, "config": "native" }, { "model": "von", "name": "Von", "code": "V", "suite": "fresh/emotion", "n": 200, "correct": 177, "accuracy": 0.885, "interval": [ 0.8333528310151243, 0.9221362676743804 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 18.73360003810376, "p95_ms": 29.050899983849376, "device_peak_gib": 3.9156341552734375, "device_min_gib": 3.9083099365234375, "samples": 40, "added_device_peak_gib": 2.0411109924316406, "torch_peak_allocated_gib": 1.507309913635254, "torch_peak_reserved_gib": 1.765625, "warmup_errors": [], "finished": "2026-10-01T14:48:53.714134+00:00", "predictions_sha256": "9411a903ce84b7a03c262280e1d12c84c91890242a4bdcd343e8e84ba0996351", "timing_coverage": 1.0, "plottable": true, "config": "native" }, { "model": "von", "name": "Von", "code": "V", "suite": "fresh/spam", "n": 200, "correct": 125, "accuracy": 0.625, "interval": [ 0.5561430369897816, 0.6891456314159018 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 37.7057499717921, "p95_ms": 47.878044960089, "device_peak_gib": 3.9146575927734375, "device_min_gib": 3.9146575927734375, "samples": 88, "added_device_peak_gib": 2.0401344299316406, "torch_peak_allocated_gib": 1.5322909355163574, "torch_peak_reserved_gib": 1.76953125, "warmup_errors": [], "finished": "2026-10-01T14:48:56.777915+00:00", "predictions_sha256": "acc3f377a2cb7c3585009cd78437fdf06d3498723618259f2891c6827c9b6a7c", "timing_coverage": 1.0, "plottable": true, "config": "native" }, { "model": "von", "name": "Von", "code": "V", "suite": "fresh/phishing", "n": 200, "correct": 152, "accuracy": 0.76, "interval": [ 0.6962665902079583, 0.8139338400758631 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 17.53039995674044, "p95_ms": 30.022935196757313, "device_peak_gib": 3.9146575927734375, "device_min_gib": 3.9146575927734375, "samples": 39, "added_device_peak_gib": 2.0401344299316406, "torch_peak_allocated_gib": 1.514617919921875, "torch_peak_reserved_gib": 1.76953125, "warmup_errors": [], "finished": "2026-10-01T14:48:58.102032+00:00", "predictions_sha256": "6228bb0499fed15d8a2021a6cc15ae184463a60409e358a8414042d7172356f2", "timing_coverage": 1.0, "plottable": true, "config": "native" }, { "model": "von", "name": "Von", "code": "V", "suite": "massive_en", "n": 2948, "correct": 1665, "accuracy": 0.5647896879240163, "interval": [ 0.5468199427314454, 0.5825908014867128 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 18.06829997804016, "p95_ms": 28.156674990896136, "device_peak_gib": 3.9175872802734375, "device_min_gib": 3.9102630615234375, "samples": 40, "added_device_peak_gib": 2.0430641174316406, "torch_peak_allocated_gib": 1.5103259086608887, "torch_peak_reserved_gib": 1.767578125, "warmup_errors": [], "finished": "2026-10-01T14:48:59.469525+00:00", "predictions_sha256": "c7ea06c0f8b4f6aa103d08a740da4f77eb2369a9fc3d2e9a04a9182a068723d1", "timing_coverage": 1.0, "plottable": true, "config": "native" }, { "model": "von", "name": "Von", "code": "V", "suite": "massive_nb", "n": 2948, "correct": 394, "accuracy": 0.13364993215739485, "interval": [ 0.12184212154942771, 0.14641126192904913 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 18.826449988409877, "p95_ms": 26.216674968600262, "device_peak_gib": 3.9127044677734375, "device_min_gib": 3.9127044677734375, "samples": 40, "added_device_peak_gib": 2.0381813049316406, "torch_peak_allocated_gib": 1.510580062866211, "torch_peak_reserved_gib": 1.767578125, "warmup_errors": [], "finished": "2026-10-01T14:49:00.843944+00:00", "predictions_sha256": "40857841b4b349b9bed81ad488e53d6f41e24594d2364494a843f4b2cafaf69f", "timing_coverage": 1.0, "plottable": true, "config": "native" }, { "model": "von", "name": "Von", "code": "V", "suite": "public_jevbench", "n": 231, "correct": 144, "accuracy": 0.6233766233766234, "interval": [ 0.5593544603348791, 0.6833624785150049 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 23.256400017999113, "p95_ms": 2626.4995648525646, "device_peak_gib": 4.0855560302734375, "device_min_gib": 3.9146575927734375, "samples": 534, "added_device_peak_gib": 2.2110328674316406, "torch_peak_allocated_gib": 1.7126879692077637, "torch_peak_reserved_gib": 1.935546875, "warmup_errors": [], "finished": "2026-10-01T14:49:17.659526+00:00", "predictions_sha256": "7e82652e548819dd3b2d00fb927b65a8d35394edf53b63c50e36f25f68942a08", "timing_coverage": 1.0, "plottable": true, "config": "native" }, { "model": "von", "name": "Von", "code": "V", "suite": "kev_claim/devtools-v1", "n": 900, "correct": 389, "accuracy": 0.43222222222222223, "interval": [ 0.3821133085227448, 0.47703601026669923 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 43.24234998784959, "p95_ms": 90.39389988174662, "device_peak_gib": 3.9146575927734375, "device_min_gib": 3.9146575927734375, "samples": 57, "added_device_peak_gib": 2.0401344299316406, "torch_peak_allocated_gib": 1.55656099319458, "torch_peak_reserved_gib": 1.76953125, "warmup_errors": [], "finished": "2026-10-01T14:49:19.779440+00:00", "predictions_sha256": "27298c8367ec6625d663e21e7ad30bbc616d737f75709802641a66b8e2d8600d", "timing_coverage": 1.0, "plottable": true, "config": "native" }, { "model": "von", "name": "Von", "code": "V", "suite": "industry/routing/en", "n": 40, "correct": 34, "accuracy": 0.85, "interval": [ 0.709276756335103, 0.9293881228267964 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 17.91165000759065, "p95_ms": 24.554149946197867, "device_peak_gib": 3.9127044677734375, "device_min_gib": 3.9127044677734375, "samples": 24, "added_device_peak_gib": 2.0381813049316406, "torch_peak_allocated_gib": 1.5093417167663574, "torch_peak_reserved_gib": 1.767578125, "warmup_errors": [], "finished": "2026-10-01T14:49:20.665940+00:00", "predictions_sha256": "909bac8b068ce4e7a9d4124c8f73864aba54e65be5b5f2a2228b3adc1f06ac06", "timing_coverage": 1.0, "plottable": true, "config": "native" }, { "model": "von", "name": "Von", "code": "V", "suite": "industry/routing/nb", "n": 40, "correct": 15, "accuracy": 0.375, "interval": [ 0.24222979167233266, 0.5296756086269889 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 18.15035007894039, "p95_ms": 26.931900007184595, "device_peak_gib": 3.9156341552734375, "device_min_gib": 3.9083099365234375, "samples": 24, "added_device_peak_gib": 2.0411109924316406, "torch_peak_allocated_gib": 1.5106148719787598, "torch_peak_reserved_gib": 1.765625, "warmup_errors": [], "finished": "2026-10-01T14:49:21.530978+00:00", "predictions_sha256": "72cea3b40a1bb6dde47d42a44d8e309fd1a49859aede84f192fb7619c4ba930d", "timing_coverage": 1.0, "plottable": true, "config": "native" }, { "model": "von", "name": "Von", "code": "V", "suite": "industry/documents/en", "n": 28, "correct": 26, "accuracy": 0.9285714285714286, "interval": [ 0.7735463743160095, 0.9801879358846942 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 17.26950006559491, "p95_ms": 26.163000031374395, "device_peak_gib": 3.9127044677734375, "device_min_gib": 3.9127044677734375, "samples": 24, "added_device_peak_gib": 2.0381813049316406, "torch_peak_allocated_gib": 1.510507583618164, "torch_peak_reserved_gib": 1.767578125, "warmup_errors": [], "finished": "2026-10-01T14:49:22.410166+00:00", "predictions_sha256": "39acfa8d090c5df6b3fda98a6311b5e3c57482685e53fa375bd752915760e01f", "timing_coverage": 1.0, "plottable": true, "config": "native" }, { "model": "von", "name": "Von", "code": "V", "suite": "industry/documents/nb", "n": 28, "correct": 12, "accuracy": 0.42857142857142855, "interval": [ 0.26508547124123694, 0.6092921437253123 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 17.491700127720833, "p95_ms": 26.03072498459369, "device_peak_gib": 3.9107513427734375, "device_min_gib": 3.9107513427734375, "samples": 23, "added_device_peak_gib": 2.0362281799316406, "torch_peak_allocated_gib": 1.5109047889709473, "torch_peak_reserved_gib": 1.765625, "warmup_errors": [], "finished": "2026-10-01T14:49:23.246954+00:00", "predictions_sha256": "327f0f1012012b2b7c76ac2c4a0505600ff990a56a1953b6deafce23a1f8fbd2", "timing_coverage": 1.0, "plottable": true, "config": "native" }, { "model": "wald-4b-v12", "name": "Wald 4B v1.2", "code": "WA", "suite": "fresh/news", "n": 200, "correct": 170, "accuracy": 0.85, "interval": [ 0.7939442071583334, 0.89286406437758 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 57.243599905632436, "p95_ms": 66.14972505485639, "device_peak_gib": 10.065048217773438, "device_min_gib": 10.065048217773438, "samples": 114, "added_device_peak_gib": 8.19052505493164, "torch_peak_allocated_gib": 7.900611400604248, "torch_peak_reserved_gib": 7.919921875, "warmup_errors": [], "finished": "2026-10-01T14:49:42.896378+00:00", "predictions_sha256": "ff9048b2a0c6bc336bdfc85f3fe8ef4816bb406bfd977e361d2f173776d0b909", "timing_coverage": 1.0, "plottable": true, "config": "bf16" }, { "model": "wald-4b-v12", "name": "Wald 4B v1.2", "code": "WA", "suite": "fresh/emotion", "n": 200, "correct": 121, "accuracy": 0.605, "interval": [ 0.5358834915928536, 0.6701589898679203 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 59.50570001732558, "p95_ms": 69.7240351117216, "device_peak_gib": 10.056259155273438, "device_min_gib": 10.031845092773438, "samples": 118, "added_device_peak_gib": 8.18173599243164, "torch_peak_allocated_gib": 7.88443660736084, "torch_peak_reserved_gib": 7.908203125, "warmup_errors": [], "finished": "2026-10-01T14:49:47.022300+00:00", "predictions_sha256": "4aed26495449f5219c8ed9952c9f5e4dcc7743930892f32ede7272f5253fec57", "timing_coverage": 1.0, "plottable": true, "config": "bf16" }, { "model": "wald-4b-v12", "name": "Wald 4B v1.2", "code": "WA", "suite": "fresh/spam", "n": 200, "correct": 184, "accuracy": 0.92, "interval": [ 0.874010511732856, 0.9501594141102401 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 60.29175000730902, "p95_ms": 71.61099503282452, "device_peak_gib": 10.153915405273438, "device_min_gib": 10.092391967773438, "samples": 121, "added_device_peak_gib": 8.27939224243164, "torch_peak_allocated_gib": 7.956739902496338, "torch_peak_reserved_gib": 8.00390625, "warmup_errors": [], "finished": "2026-10-01T14:49:51.257196+00:00", "predictions_sha256": "879c1faf0670af5456b02bc6afb1a219d470075e4fa053c6c0d95cd7ccdd978a", "timing_coverage": 1.0, "plottable": true, "config": "bf16" }, { "model": "wald-4b-v12", "name": "Wald 4B v1.2", "code": "WA", "suite": "fresh/phishing", "n": 200, "correct": 181, "accuracy": 0.905, "interval": [ 0.8563983883518006, 0.9383368972826135 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 57.89635004475713, "p95_ms": 64.99812486581504, "device_peak_gib": 10.089462280273438, "device_min_gib": 10.082138061523438, "samples": 114, "added_device_peak_gib": 8.21493911743164, "torch_peak_allocated_gib": 7.923110008239746, "torch_peak_reserved_gib": 7.939453125, "warmup_errors": [], "finished": "2026-10-01T14:49:55.259152+00:00", "predictions_sha256": "5de47d539bc6d67f0ec37d99316497ac5ee9695fa33ef8b117011d9313a2ae31", "timing_coverage": 1.0, "plottable": true, "config": "bf16" }, { "model": "wald-4b-v12", "name": "Wald 4B v1.2", "code": "WA", "suite": "massive_en", "n": 2948, "correct": 2507, "accuracy": 0.8504070556309362, "interval": [ 0.8370761654409344, 0.8628259223986334 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 57.09370004478842, "p95_ms": 68.42383001931012, "device_peak_gib": 10.067001342773438, "device_min_gib": 10.067001342773438, "samples": 114, "added_device_peak_gib": 8.19247817993164, "torch_peak_allocated_gib": 7.898597240447998, "torch_peak_reserved_gib": 7.921875, "warmup_errors": [], "finished": "2026-10-01T14:49:59.318005+00:00", "predictions_sha256": "c0919a891e34a00aaf40df696248f51c2f50c47de7e430afae05d556c40f5187", "timing_coverage": 1.0, "plottable": true, "config": "bf16" }, { "model": "wald-4b-v12", "name": "Wald 4B v1.2", "code": "WA", "suite": "massive_nb", "n": 2948, "correct": 2379, "accuracy": 0.8069877883310719, "interval": [ 0.7923453834065616, 0.8208311795119368 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 57.730650063604116, "p95_ms": 69.24646007828414, "device_peak_gib": 10.203842163085938, "device_min_gib": 10.203720092773438, "samples": 117, "added_device_peak_gib": 8.19162368774414, "torch_peak_allocated_gib": 7.900162696838379, "torch_peak_reserved_gib": 7.921875, "warmup_errors": [], "finished": "2026-10-04T18:22:40.470252+00:00", "predictions_sha256": "24b0d3a70798fbb121f50f8ba8e76d7fd287119f770bbc670e4cce9b5616ef8a", "timing_coverage": 1.0, "plottable": true, "config": "bf16" }, { "model": "wald-4b-v12", "name": "Wald 4B v1.2", "code": "WA", "suite": "public_jevbench", "n": 231, "correct": 197, "accuracy": 0.8528138528138528, "interval": [ 0.8013637559728725, 0.8927215252996031 ], "accuracy_failures": 10, "complete": true, "calls": 60, "unique_cases": 20, "answered": 54, "failures": 6, "abstentions": 0, "truncated_calls": 0, "median_ms": 68.23229999281466, "p95_ms": 1051.2826500111255, "device_peak_gib": 14.526229858398438, "device_min_gib": 10.240951538085938, "samples": 436, "added_device_peak_gib": 12.51401138305664, "torch_peak_allocated_gib": 11.785543441772461, "torch_peak_reserved_gib": 12.244140625, "warmup_errors": [], "finished": "2026-10-04T18:22:54.547911+00:00", "predictions_sha256": "1570828f1ed25f5909aed7752b33e4f19216433f32e32c36b32c915839c0e234", "timing_coverage": 0.9, "plottable": true, "config": "bf16" }, { "model": "wald-4b-v12", "name": "Wald 4B v1.2", "code": "WA", "suite": "kev_claim/devtools-v1", "n": 900, "correct": 595, "accuracy": 0.6611111111111111, "interval": [ 0.620132422925719, 0.7117335822534592 ], "accuracy_failures": 1, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 79.96945001650602, "p95_ms": 282.77979989070445, "device_peak_gib": 10.892196655273438, "device_min_gib": 10.423446655273438, "samples": 136, "added_device_peak_gib": 8.87997817993164, "torch_peak_allocated_gib": 8.489532470703125, "torch_peak_reserved_gib": 8.609375, "warmup_errors": [], "finished": "2026-10-04T18:22:59.873577+00:00", "predictions_sha256": "8feb72e43cf3c1445b9ebaefc78f02329857d5a4542d9f7b1b676b5d28ef2113", "timing_coverage": 1.0, "plottable": true, "config": "bf16" }, { "model": "wald-4b-v12", "name": "Wald 4B v1.2", "code": "WA", "suite": "industry/routing/en", "n": 40, "correct": 38, "accuracy": 0.95, "interval": [ 0.8349612263085903, 0.9861793326138516 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 57.168400031514466, "p95_ms": 67.4928500666283, "device_peak_gib": 10.050399780273438, "device_min_gib": 10.043075561523438, "samples": 70, "added_device_peak_gib": 8.17587661743164, "torch_peak_allocated_gib": 7.889847755432129, "torch_peak_reserved_gib": 7.900390625, "warmup_errors": [], "finished": "2026-10-01T14:50:01.919942+00:00", "predictions_sha256": "a1e3af96a1d71e70d79b6f9bdae124ec24ed5a25aea1acd04fc01ac1e2ff4868", "timing_coverage": 1.0, "plottable": true, "config": "bf16" }, { "model": "wald-4b-v12", "name": "Wald 4B v1.2", "code": "WA", "suite": "industry/routing/nb", "n": 40, "correct": 36, "accuracy": 0.9, "interval": [ 0.7694822477247771, 0.9604204713173936 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 56.63964990526438, "p95_ms": 63.23100003646687, "device_peak_gib": 10.045516967773438, "device_min_gib": 10.045516967773438, "samples": 68, "added_device_peak_gib": 8.17099380493164, "torch_peak_allocated_gib": 7.8926472663879395, "torch_peak_reserved_gib": 7.900390625, "warmup_errors": [], "finished": "2026-10-01T14:50:04.464798+00:00", "predictions_sha256": "3f3ea7f3a32ed30f041ae363f23ddfff54ac7ab82d6d8f8ccb4969b46479ba30", "timing_coverage": 1.0, "plottable": true, "config": "bf16" }, { "model": "wald-4b-v12", "name": "Wald 4B v1.2", "code": "WA", "suite": "industry/documents/en", "n": 28, "correct": 28, "accuracy": 1.0, "interval": [ 0.8793566952341543, 0.9999999999999998 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 57.187350117601454, "p95_ms": 77.45527510996908, "device_peak_gib": 10.052352905273438, "device_min_gib": 10.045028686523438, "samples": 70, "added_device_peak_gib": 8.17782974243164, "torch_peak_allocated_gib": 7.895297050476074, "torch_peak_reserved_gib": 7.90234375, "warmup_errors": [], "finished": "2026-10-01T14:50:07.073571+00:00", "predictions_sha256": "8f960fcb1f58ef66f07e5a26c336fbfc7665c184eb6fb66889421c14d3bd9534", "timing_coverage": 1.0, "plottable": true, "config": "bf16" }, { "model": "wald-4b-v12", "name": "Wald 4B v1.2", "code": "WA", "suite": "industry/documents/nb", "n": 28, "correct": 28, "accuracy": 1.0, "interval": [ 0.8793566952341543, 0.9999999999999998 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 55.485400022007525, "p95_ms": 64.22727496828884, "device_peak_gib": 10.045516967773438, "device_min_gib": 10.045516967773438, "samples": 67, "added_device_peak_gib": 8.17099380493164, "torch_peak_allocated_gib": 7.895090103149414, "torch_peak_reserved_gib": 7.900390625, "warmup_errors": [], "finished": "2026-10-01T14:50:09.588913+00:00", "predictions_sha256": "eb2cb6263b4ea04d7fbede67b44bf9e157d0107d05c4a0e85f83131f4cdbd643", "timing_coverage": 1.0, "plottable": true, "config": "bf16" }, { "model": "winnow-12b-q8", "name": "Winnow 12B Q8", "code": "W", "suite": "fresh/news", "n": 200, "correct": 170, "accuracy": 0.85, "interval": [ 0.7939442071583334, 0.89286406437758 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 70.2879000455141, "p95_ms": 81.24559503048658, "device_peak_gib": 14.925224304199219, "device_min_gib": 14.917900085449219, "samples": 139, "added_device_peak_gib": 13.063667297363281, "torch_peak_allocated_gib": 0.0, "torch_peak_reserved_gib": 0.0, "warmup_errors": [], "finished": "2026-10-01T14:25:57.822238+00:00", "predictions_sha256": "3038cb24c61300d8066c75f46fbeed3a9aa479648bdb8ef115dc32e33293f74a", "timing_coverage": 1.0, "plottable": true, "config": "publisher Q8_0 GGUF" }, { "model": "winnow-12b-q8", "name": "Winnow 12B Q8", "code": "W", "suite": "fresh/emotion", "n": 200, "correct": 120, "accuracy": 0.6, "interval": [ 0.53083672039262, 0.6653942143319266 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 67.66979990061373, "p95_ms": 70.36983490688725, "device_peak_gib": 14.931083679199219, "device_min_gib": 14.920341491699219, "samples": 134, "added_device_peak_gib": 13.069526672363281, "torch_peak_allocated_gib": 0.0, "torch_peak_reserved_gib": 0.0, "warmup_errors": [], "finished": "2026-10-01T14:26:02.521122+00:00", "predictions_sha256": "e09c5ca043d114532e82314ebcfbb86c7499c6da5bcacea782f3254e82592f53", "timing_coverage": 1.0, "plottable": true, "config": "publisher Q8_0 GGUF" }, { "model": "winnow-12b-q8", "name": "Winnow 12B Q8", "code": "W", "suite": "fresh/spam", "n": 200, "correct": 187, "accuracy": 0.935, "interval": [ 0.8919809207009312, 0.961623645350847 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 78.43510003294796, "p95_ms": 94.07567491289224, "device_peak_gib": 14.931083679199219, "device_min_gib": 14.923759460449219, "samples": 152, "added_device_peak_gib": 13.069526672363281, "torch_peak_allocated_gib": 0.0, "torch_peak_reserved_gib": 0.0, "warmup_errors": [], "finished": "2026-10-01T14:26:07.829694+00:00", "predictions_sha256": "1e62441477369af58694b368d651a1d26246b952a06d3dd437c77a6bc654d6f9", "timing_coverage": 1.0, "plottable": true, "config": "publisher Q8_0 GGUF" }, { "model": "winnow-12b-q8", "name": "Winnow 12B Q8", "code": "W", "suite": "fresh/phishing", "n": 200, "correct": 182, "accuracy": 0.91, "interval": [ 0.8622342812328346, 0.9423125511378069 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 82.06104999408126, "p95_ms": 91.07474009506403, "device_peak_gib": 14.931083679199219, "device_min_gib": 14.923759460449219, "samples": 155, "added_device_peak_gib": 13.069526672363281, "torch_peak_allocated_gib": 0.0, "torch_peak_reserved_gib": 0.0, "warmup_errors": [], "finished": "2026-10-01T14:26:13.261340+00:00", "predictions_sha256": "1da11499c50dc52223baaea894a4af174d5a73f9891e310f28fac8e727171a22", "timing_coverage": 1.0, "plottable": true, "config": "publisher Q8_0 GGUF" }, { "model": "winnow-12b-q8", "name": "Winnow 12B Q8", "code": "W", "suite": "public_jevbench", "n": 231, "correct": 199, "accuracy": 0.8614718614718615, "interval": [ 0.8109826332087084, 0.9001354157023556 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 90.9514999948442, "p95_ms": 769.4086801027877, "device_peak_gib": 14.934989929199219, "device_min_gib": 14.926200866699219, "samples": 474, "added_device_peak_gib": 13.073432922363281, "torch_peak_allocated_gib": 0.0, "torch_peak_reserved_gib": 0.0, "warmup_errors": [], "finished": "2026-10-01T14:26:28.620055+00:00", "predictions_sha256": "7ced0fbd5b37a4acf1f2b425b4403e489b2a10702b18723bb9cec28e07ff5e9a", "timing_coverage": 1.0, "plottable": true, "config": "publisher Q8_0 GGUF" }, { "model": "decision2-nox-4b", "name": "Decision 2.0 Nox 4B", "code": "NX", "suite": "fresh/news", "n": 200, "correct": 174, "accuracy": 0.87, "interval": [ 0.8163364812775159, 0.9097179772033067 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 55.27444998733699, "p95_ms": 65.91447499813512, "device_peak_gib": 11.495223999023438, "device_min_gib": 11.487899780273438, "samples": 111, "added_device_peak_gib": 9.48404312133789, "torch_peak_allocated_gib": 9.192676067352295, "torch_peak_reserved_gib": 9.208984375, "warmup_errors": [], "finished": "2026-10-04T18:24:40.627221+00:00", "predictions_sha256": "13651c4a1dd3437d4dfe66dbd0638ee813a023cb4f53502d18dc5a55cd01092c", "timing_coverage": 1.0, "plottable": true, "config": "Publisher BF16-exact linear residency; FP32 embeddings, norms and decision head; BF16 autocast" }, { "model": "decision2-nox-4b", "name": "Decision 2.0 Nox 4B", "code": "NX", "suite": "fresh/emotion", "n": 200, "correct": 118, "accuracy": 0.59, "interval": [ 0.5207645903369208, 0.6558432509151711 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 57.63270007446408, "p95_ms": 70.6892749411054, "device_peak_gib": 11.510025024414062, "device_min_gib": 11.495315551757812, "samples": 117, "added_device_peak_gib": 9.498844146728516, "torch_peak_allocated_gib": 9.185922622680664, "torch_peak_reserved_gib": 9.189453125, "warmup_errors": [], "finished": "2026-10-04T18:24:44.671482+00:00", "predictions_sha256": "290eb67feb09b0fe6f4aa9b8ca3009b9925898187da66df4bb260cfc11aca080", "timing_coverage": 1.0, "plottable": true, "config": "Publisher BF16-exact linear residency; FP32 embeddings, norms and decision head; BF16 autocast" }, { "model": "decision2-nox-4b", "name": "Decision 2.0 Nox 4B", "code": "NX", "suite": "fresh/spam", "n": 200, "correct": 186, "accuracy": 0.93, "interval": [ 0.8859451911438936, 0.9578478281716571 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 55.74794998392463, "p95_ms": 64.67301006196067, "device_peak_gib": 11.536270141601562, "device_min_gib": 11.513931274414062, "samples": 111, "added_device_peak_gib": 9.525089263916016, "torch_peak_allocated_gib": 9.203571796417236, "torch_peak_reserved_gib": 9.23828125, "warmup_errors": [], "finished": "2026-10-04T18:24:48.566139+00:00", "predictions_sha256": "2a70a197c20255af357a1309ea354210543186a6e58d997f6e3976ff4f132c5c", "timing_coverage": 1.0, "plottable": true, "config": "Publisher BF16-exact linear residency; FP32 embeddings, norms and decision head; BF16 autocast" }, { "model": "decision2-nox-4b", "name": "Decision 2.0 Nox 4B", "code": "NX", "suite": "fresh/phishing", "n": 200, "correct": 185, "accuracy": 0.925, "interval": [ 0.8799563897637055, 0.954025082815618 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 53.511800011619925, "p95_ms": 63.55751000810414, "device_peak_gib": 11.508590698242188, "device_min_gib": 11.493331909179688, "samples": 107, "added_device_peak_gib": 9.49740982055664, "torch_peak_allocated_gib": 9.200542449951172, "torch_peak_reserved_gib": 9.21484375, "warmup_errors": [], "finished": "2026-10-04T18:24:52.400413+00:00", "predictions_sha256": "64227a6dec372f610b02deef5a1fed82baf70dc3c8da0c4de132d817910f4e9d", "timing_coverage": 1.0, "plottable": true, "config": "Publisher BF16-exact linear residency; FP32 embeddings, norms and decision head; BF16 autocast" }, { "model": "decision2-nox-4b", "name": "Decision 2.0 Nox 4B", "code": "NX", "suite": "massive_en", "n": 2948, "correct": 2512, "accuracy": 0.8521031207598372, "interval": [ 0.838830303940018, 0.8644594997167794 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 68.77249991521239, "p95_ms": 80.014070100151, "device_peak_gib": 11.528915405273438, "device_min_gib": 11.520797729492188, "samples": 136, "added_device_peak_gib": 9.51773452758789, "torch_peak_allocated_gib": 9.218977928161621, "torch_peak_reserved_gib": 9.2421875, "warmup_errors": [], "finished": "2026-10-04T18:24:57.171812+00:00", "predictions_sha256": "f6ee85d950dc07863b4e4507d2e7716591531ebaeca6101871a04ea95f21df88", "timing_coverage": 1.0, "plottable": true, "config": "Publisher BF16-exact linear residency; FP32 embeddings, norms and decision head; BF16 autocast" }, { "model": "decision2-nox-4b", "name": "Decision 2.0 Nox 4B", "code": "NX", "suite": "massive_nb", "n": 2948, "correct": 2367, "accuracy": 0.8029172320217096, "interval": [ 0.7881672986123502, 0.816878746344798 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 69.38680005259812, "p95_ms": 81.0673299827613, "device_peak_gib": 11.543746948242188, "device_min_gib": 11.543075561523438, "samples": 137, "added_device_peak_gib": 9.53256607055664, "torch_peak_allocated_gib": 9.238706588745117, "torch_peak_reserved_gib": 9.26171875, "warmup_errors": [], "finished": "2026-10-04T18:25:01.998950+00:00", "predictions_sha256": "12f024ecb92018183101f21fe004d3aa3af120d04592b0808ae3fbe1bb5cde9a", "timing_coverage": 1.0, "plottable": true, "config": "Publisher BF16-exact linear residency; FP32 embeddings, norms and decision head; BF16 autocast" }, { "model": "decision2-nox-4b", "name": "Decision 2.0 Nox 4B", "code": "NX", "suite": "public_jevbench", "n": 231, "correct": 185, "accuracy": 0.8008658008658008, "interval": [ 0.7446321747693682, 0.8472565006715772 ], "accuracy_failures": 0, "complete": true, "calls": 60, "unique_cases": 20, "answered": 60, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 62.03684990759939, "p95_ms": 461.8937648250721, "device_peak_gib": 12.235031127929688, "device_min_gib": 11.494308471679688, "samples": 288, "added_device_peak_gib": 10.22385025024414, "torch_peak_allocated_gib": 9.666880130767822, "torch_peak_reserved_gib": 9.94921875, "warmup_errors": [], "finished": "2026-10-04T18:25:11.396695+00:00", "predictions_sha256": "fb25c11b78c33c48a9e1c77f281dcbd7c33eec9b741473b8080bf32c4956bd47", "timing_coverage": 1.0, "plottable": true, "config": "Publisher BF16-exact linear residency; FP32 embeddings, norms and decision head; BF16 autocast" }, { "model": "decision2-nox-4b", "name": "Decision 2.0 Nox 4B", "code": "NX", "suite": "kev_claim/devtools-v1", "n": 900, "correct": 588, "accuracy": 0.6533333333333333, "interval": [ 0.6137904392174773, 0.6972258381776191 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 65.0545998942107, "p95_ms": 167.7291999803856, "device_peak_gib": 11.671676635742188, "device_min_gib": 11.664352416992188, "samples": 98, "added_device_peak_gib": 9.66049575805664, "torch_peak_allocated_gib": 9.343379020690918, "torch_peak_reserved_gib": 9.384765625, "warmup_errors": [], "finished": "2026-10-04T18:25:15.226170+00:00", "predictions_sha256": "35a79ef0474eac9d849684cedbf9c1c03f5a94e192f702c84441d1be1233dec1", "timing_coverage": 1.0, "plottable": true, "config": "Publisher BF16-exact linear residency; FP32 embeddings, norms and decision head; BF16 autocast" }, { "model": "decision2-nox-4b", "name": "Decision 2.0 Nox 4B", "code": "NX", "suite": "industry/routing/en", "n": 40, "correct": 38, "accuracy": 0.95, "interval": [ 0.8349612263085903, 0.9861793326138516 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 54.41600002814084, "p95_ms": 65.66842494066805, "device_peak_gib": 11.490707397460938, "device_min_gib": 11.489913940429688, "samples": 67, "added_device_peak_gib": 9.47952651977539, "torch_peak_allocated_gib": 9.188786029815674, "torch_peak_reserved_gib": 9.208984375, "warmup_errors": [], "finished": "2026-10-04T18:25:17.742414+00:00", "predictions_sha256": "de9c5984d6b191f7a59e7e34a46afb6b2bcbae4c749f0ae8a892db7b55fb3033", "timing_coverage": 1.0, "plottable": true, "config": "Publisher BF16-exact linear residency; FP32 embeddings, norms and decision head; BF16 autocast" }, { "model": "decision2-nox-4b", "name": "Decision 2.0 Nox 4B", "code": "NX", "suite": "industry/routing/nb", "n": 40, "correct": 35, "accuracy": 0.875, "interval": [ 0.738878801611489, 0.945404997490546 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 53.29694994725287, "p95_ms": 65.78482501208782, "device_peak_gib": 11.497543334960938, "device_min_gib": 11.490219116210938, "samples": 66, "added_device_peak_gib": 9.48636245727539, "torch_peak_allocated_gib": 9.19394302368164, "torch_peak_reserved_gib": 9.2109375, "warmup_errors": [], "finished": "2026-10-04T18:25:20.192497+00:00", "predictions_sha256": "32312f8dc80bc0c4301e27afcb7140e02e3832322be844649c4ccb7697c0034f", "timing_coverage": 1.0, "plottable": true, "config": "Publisher BF16-exact linear residency; FP32 embeddings, norms and decision head; BF16 autocast" }, { "model": "decision2-nox-4b", "name": "Decision 2.0 Nox 4B", "code": "NX", "suite": "industry/documents/en", "n": 28, "correct": 28, "accuracy": 1.0, "interval": [ 0.8793566952341543, 0.9999999999999998 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 55.79084996134043, "p95_ms": 66.11465010792017, "device_peak_gib": 11.491012573242188, "device_min_gib": 11.490219116210938, "samples": 68, "added_device_peak_gib": 9.47983169555664, "torch_peak_allocated_gib": 9.189348220825195, "torch_peak_reserved_gib": 9.208984375, "warmup_errors": [], "finished": "2026-10-04T18:25:22.687994+00:00", "predictions_sha256": "1c5c598845cbb1fc60fbaf06a17597ee88314424be5ce5f335e58d4057ee19ce", "timing_coverage": 1.0, "plottable": true, "config": "Publisher BF16-exact linear residency; FP32 embeddings, norms and decision head; BF16 autocast" }, { "model": "decision2-nox-4b", "name": "Decision 2.0 Nox 4B", "code": "NX", "suite": "industry/documents/nb", "n": 28, "correct": 28, "accuracy": 1.0, "interval": [ 0.8793566952341543, 0.9999999999999998 ], "accuracy_failures": 0, "complete": true, "calls": 36, "unique_cases": 12, "answered": 36, "failures": 0, "abstentions": 0, "truncated_calls": 0, "median_ms": 54.970550001598895, "p95_ms": 64.69399994239211, "device_peak_gib": 11.491012573242188, "device_min_gib": 11.490036010742188, "samples": 67, "added_device_peak_gib": 9.47983169555664, "torch_peak_allocated_gib": 9.192676067352295, "torch_peak_reserved_gib": 9.208984375, "warmup_errors": [], "finished": "2026-10-04T18:25:25.190304+00:00", "predictions_sha256": "91fbd34f6fd05778bba245a3118f3d6290f738a4f0856578b3148e37f6063498", "timing_coverage": 1.0, "plottable": true, "config": "Publisher BF16-exact linear residency; FP32 embeddings, norms and decision head; BF16 autocast" } ], "pending": [ { "model": "jevk5", "suite": "kev_claim/devtools-v1", "reason": "Developer-tool timing unavailable: three fresh workers failed native/batched probability parity during fixed warm-up (maximum difference 0.047281). Native-serial diagnostics changed 1 of 12 saved decisions, so alternative-path latency is withheld from comparison with the original accuracy score. No timing value is estimated; existing accuracy records are preserved." }, { "model": "winnow-12b-q8", "suite": "massive_en", "reason": "Timing incomplete" }, { "model": "winnow-12b-q8", "suite": "massive_nb", "reason": "Timing incomplete" }, { "model": "winnow-12b-q8", "suite": "kev_claim/devtools-v1", "reason": "Timing incomplete" }, { "model": "winnow-12b-q8", "suite": "industry/routing/en", "reason": "Timing incomplete" }, { "model": "winnow-12b-q8", "suite": "industry/routing/nb", "reason": "Timing incomplete" }, { "model": "winnow-12b-q8", "suite": "industry/documents/en", "reason": "Timing incomplete" }, { "model": "winnow-12b-q8", "suite": "industry/documents/nb", "reason": "Timing incomplete" } ], "plot_rule": "Only complete timing blocks with >=90% answered calls are plotted. Median covers answered calls. Asterisk flags timing refusals/errors/abstentions or native truncation. Excluded blocks and full counts remain in this file.", "limits": "Accuracy and timing use different sample sizes. Industry cases are authored and not expert-validated. Public JevBench Wilson intervals are descriptive; related cases and unknown training exposure limit inference. Device-level VRAM deltas include external processes and desktop noise, so do not directly compare bubble areas to the earlier PyTorch-reserved-memory chart." }