{"data":[{"project":"default","id":"app.email_spam-harness-20260917-jev","benchmark":"app.email_spam","split":"harness","system":"jev","version":"typesafe/jev-1.13-20260917","baseline":true,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"e95c6ee6362fd66d6d80c711095e03f30bbeeffd","questions":400,"answered":400,"when":"","ended":"2026-09-25T02:55:39Z","by":"","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.9775,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"brier","value":0.0379,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"ece","value":0.0604,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"aurc","value":0.0007,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":400,"of":400}],"notes":"preds sha256:866ee077035cc9923b82789c1691a58c64cb7eaffe382f24d5905e517561eba5","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"app.email_spam-harness-a5.df1c16a2-kai","benchmark":"app.email_spam","split":"harness","system":"kai","version":"a5@df1c16a2fef7ec93f2b34a6a194f6dc3c2eee70e936473b802151543c5d46e83","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"611059b60f64ab686d6475cd76778b3fac52cd98","questions":400,"answered":400,"when":"","ended":"2026-09-27T21:09:54Z","by":"","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.995,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"brier","value":0.0041,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"ece","value":0.0036,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"aurc","value":0,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":400,"of":400}],"notes":"device metal; dtype bf16; preds sha256:f28954731ec991ecbed3c1fbfc45980eeffef7744d8c12bcfc4cb160cb6f3eac; scores sha256:071b556a8eafd2c19d4f9a948b4f10251c9b253107c6afb21ad959ef72af4a77","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"app.email_spam-harness-a6.a211cc70-kai","benchmark":"app.email_spam","split":"harness","system":"kai","version":"a6@a211cc70103825bfbc9bfb0fbe970f8bf08da2efe0953cd8045a3cd07397f4bf","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"a1a4a73d524cecf8038e30c408ad30b66e71d490","questions":400,"answered":400,"when":"","ended":"2026-09-28T03:18:42Z","by":"","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.9975,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"brier","value":0.0041,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"ece","value":0.003,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"aurc","value":0,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":400,"of":400}],"notes":"device metal; dtype bf16; preds sha256:43b1bd476a72a536ee96fbc934a065a808fdb35395e694d9af873e462b3d8866; scores sha256:3760df4b2c4cdcccb225bbb446b2ddcff998e09ab723e36ee466f771dc355a7e","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"app.email_spam-harness-a7.0834a74f-kai","benchmark":"app.email_spam","split":"harness","system":"kai","version":"a7@0834a74f2d140642a453373da09e5a128e3d2c8d9e8bb1dcaf86d7210a4ecdfc","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"a1a4a73d524cecf8038e30c408ad30b66e71d490","questions":400,"answered":400,"when":"","ended":"2026-09-28T12:24:45Z","by":"","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.9975,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"brier","value":0.0047,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"ece","value":0.003,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"aurc","value":0,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":400,"of":400}],"notes":"device metal; dtype bf16; preds sha256:25bfec2f5ad49bc050cc9015161c7ef2dbf07f599f35e9dad153b4a5b6f7b543; scores sha256:ddca48bcf9816e7df2a187251d9f0f00de3b2fc24f22805b34b7bd650a048c81","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"app.email_spam-harness-a8.0cfeef05-kai","benchmark":"app.email_spam","split":"harness","system":"kai","version":"a8@0cfeef05c879d7f24bbde275a6c23e85e80b6baf06cf9311d529010688d4a697","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"a1a4a73d524cecf8038e30c408ad30b66e71d490","questions":400,"answered":400,"when":"","ended":"2026-09-28T16:33:11Z","by":"","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.995,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"brier","value":0.008,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"ece","value":0.0037,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"aurc","value":0,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":400,"of":400}],"notes":"device metal; dtype bf16; preds sha256:d0a81ccbdc9eb7305fc196d7ad2c6be8c1df1ce5966ac964dabeeea231ae41d6; scores sha256:86b58e1f53447a8fb3b7963ea69f2bee40ffa97b9bda7b74b038064188998f10","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"app.email_spam-harness-english-laya","benchmark":"app.email_spam","split":"harness","system":"laya","version":"convaiinnovations/laya@55cf4c4ebb4ebe31b2550e8bdf3bd21b99753851","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"735e240323cd7c9f0c880b7d8c62d89f40be2164","questions":400,"answered":400,"when":"","ended":"2026-09-25T17:33:56Z","by":"","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.9975,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"brier","value":0.0052,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"ece","value":0.0109,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"aurc","value":0,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":400,"of":400}],"notes":"runtime laya 0.3.20; device cpu; dtype f32; preds sha256:45d01564e4ae1c2e6a0bcd7b8e81f70cc2b57df168e6b884ad042a81a4283d03","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"app.email_spam-harness-kai-1.0834a74f-kai","benchmark":"app.email_spam","split":"harness","system":"kai","version":"kai-1@0834a74f2d140642a453373da09e5a128e3d2c8d9e8bb1dcaf86d7210a4ecdfc","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"2c74e2d61addc885e847358975fb866b03cf17dc","questions":400,"answered":400,"when":"","ended":"2026-10-03T00:06:45Z","by":"","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.9975,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"brier","value":0.0047,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"ece","value":0.003,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"aurc","value":0,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":400,"of":400}],"notes":"device cuda; dtype bf16; preds sha256:9fe4a808809d63c37446a87aed506ba891ceb148c5c8d7c9947e17e3d27bf54e; scores sha256:1ec811cfb085ac209c7b094b408b00ef95d3d3c73b7ae7bdf6c52fc314d112e0","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"app.email_spam-harness-multilingual-laya","benchmark":"app.email_spam","split":"harness","system":"laya","version":"convaiinnovations/laya-multilingual@e4e9ddf21a7b1903b7acffd8814ad4307bf63a67","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"735e240323cd7c9f0c880b7d8c62d89f40be2164","questions":400,"answered":400,"when":"","ended":"2026-09-25T17:33:56Z","by":"","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.9975,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"brier","value":0.0049,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"ece","value":0.0026,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"aurc","value":0,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":400,"of":400}],"notes":"runtime laya 0.3.20; device cpu; dtype f32; preds sha256:45d01564e4ae1c2e6a0bcd7b8e81f70cc2b57df168e6b884ad042a81a4283d03","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"app.email_spam-harness-routed-laya","benchmark":"app.email_spam","split":"harness","system":"laya","version":"convaiinnovations/laya@55cf4c4ebb4ebe31b2550e8bdf3bd21b99753851+convaiinnovations/laya-multilingual@e4e9ddf21a7b1903b7acffd8814ad4307bf63a67","baseline":true,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"735e240323cd7c9f0c880b7d8c62d89f40be2164","questions":400,"answered":400,"when":"","ended":"2026-09-25T17:33:56Z","by":"","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.9975,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"brier","value":0.0052,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"ece","value":0.0108,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"aurc","value":0,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":400,"of":400}],"notes":"runtime laya 0.3.20; device cpu; dtype f32; preds sha256:45d01564e4ae1c2e6a0bcd7b8e81f70cc2b57df168e6b884ad042a81a4283d03","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"app.email_spam-harness-typed-decisions-laya","benchmark":"app.email_spam","split":"harness","system":"laya","version":"convaiinnovations/laya-typed-decisions@1a793eb568e6718f15941d08f85432581df534e3","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"735e240323cd7c9f0c880b7d8c62d89f40be2164","questions":400,"answered":400,"when":"","ended":"2026-09-25T17:33:56Z","by":"","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.9675,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"brier","value":0.0888,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"ece","value":0.1144,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"aurc","value":0.0017,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":400,"of":400}],"notes":"runtime laya 0.3.20; device cpu; dtype f32; preds sha256:45d01564e4ae1c2e6a0bcd7b8e81f70cc2b57df168e6b884ad042a81a4283d03","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"app.guardrails_jailbreak-harness-20260917-jev","benchmark":"app.guardrails_jailbreak","split":"harness","system":"jev","version":"typesafe/jev-1.13-20260917","baseline":true,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"e95c6ee6362fd66d6d80c711095e03f30bbeeffd","questions":400,"answered":400,"when":"","ended":"2026-09-25T02:55:39Z","by":"","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.94,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"brier","value":0.1044,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"ece","value":0.0469,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"aurc","value":0.012,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":400,"of":400}],"notes":"preds sha256:9b380114224f310d4f4c9b028933960c9348281ed4eb270c1588d77ca6c67564","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"app.guardrails_jailbreak-harness-a5.df1c16a2-kai","benchmark":"app.guardrails_jailbreak","split":"harness","system":"kai","version":"a5@df1c16a2fef7ec93f2b34a6a194f6dc3c2eee70e936473b802151543c5d46e83","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"611059b60f64ab686d6475cd76778b3fac52cd98","questions":400,"answered":400,"when":"","ended":"2026-09-27T21:09:54Z","by":"","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.915,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"brier","value":0.1432,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"ece","value":0.055,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"aurc","value":0.0184,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":400,"of":400}],"notes":"device metal; dtype bf16; preds sha256:f28954731ec991ecbed3c1fbfc45980eeffef7744d8c12bcfc4cb160cb6f3eac; scores sha256:071b556a8eafd2c19d4f9a948b4f10251c9b253107c6afb21ad959ef72af4a77","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"app.guardrails_jailbreak-harness-a6.a211cc70-kai","benchmark":"app.guardrails_jailbreak","split":"harness","system":"kai","version":"a6@a211cc70103825bfbc9bfb0fbe970f8bf08da2efe0953cd8045a3cd07397f4bf","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"a1a4a73d524cecf8038e30c408ad30b66e71d490","questions":400,"answered":400,"when":"","ended":"2026-09-28T03:18:42Z","by":"","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.92,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"brier","value":0.1411,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"ece","value":0.0622,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"aurc","value":0.015,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":400,"of":400}],"notes":"device metal; dtype bf16; preds sha256:43b1bd476a72a536ee96fbc934a065a808fdb35395e694d9af873e462b3d8866; scores sha256:3760df4b2c4cdcccb225bbb446b2ddcff998e09ab723e36ee466f771dc355a7e","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"app.guardrails_jailbreak-harness-a7.0834a74f-kai","benchmark":"app.guardrails_jailbreak","split":"harness","system":"kai","version":"a7@0834a74f2d140642a453373da09e5a128e3d2c8d9e8bb1dcaf86d7210a4ecdfc","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"a1a4a73d524cecf8038e30c408ad30b66e71d490","questions":400,"answered":400,"when":"","ended":"2026-09-28T12:24:45Z","by":"","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.92,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"brier","value":0.1473,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"ece","value":0.0716,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"aurc","value":0.0242,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":400,"of":400}],"notes":"device metal; dtype bf16; preds sha256:25bfec2f5ad49bc050cc9015161c7ef2dbf07f599f35e9dad153b4a5b6f7b543; scores sha256:ddca48bcf9816e7df2a187251d9f0f00de3b2fc24f22805b34b7bd650a048c81","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"app.guardrails_jailbreak-harness-a8.0cfeef05-kai","benchmark":"app.guardrails_jailbreak","split":"harness","system":"kai","version":"a8@0cfeef05c879d7f24bbde275a6c23e85e80b6baf06cf9311d529010688d4a697","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"a1a4a73d524cecf8038e30c408ad30b66e71d490","questions":400,"answered":400,"when":"","ended":"2026-09-28T16:33:11Z","by":"","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.92,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"brier","value":0.1527,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"ece","value":0.074,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"aurc","value":0.0282,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":400,"of":400}],"notes":"device metal; dtype bf16; preds sha256:d0a81ccbdc9eb7305fc196d7ad2c6be8c1df1ce5966ac964dabeeea231ae41d6; scores sha256:86b58e1f53447a8fb3b7963ea69f2bee40ffa97b9bda7b74b038064188998f10","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"app.guardrails_jailbreak-harness-english-laya","benchmark":"app.guardrails_jailbreak","split":"harness","system":"laya","version":"convaiinnovations/laya@55cf4c4ebb4ebe31b2550e8bdf3bd21b99753851","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"735e240323cd7c9f0c880b7d8c62d89f40be2164","questions":400,"answered":400,"when":"","ended":"2026-09-25T17:33:56Z","by":"","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.7075,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"brier","value":0.5389,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"ece","value":0.2591,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"aurc","value":0.225,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":400,"of":400}],"notes":"runtime laya 0.3.20; device cpu; dtype f32; preds sha256:b4176c9138e1a9c97d681c2f6cb14d853a055964d84a9a55c0b2e0c4568a035a","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"app.guardrails_jailbreak-harness-kai-1.0834a74f-kai","benchmark":"app.guardrails_jailbreak","split":"harness","system":"kai","version":"kai-1@0834a74f2d140642a453373da09e5a128e3d2c8d9e8bb1dcaf86d7210a4ecdfc","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"2c74e2d61addc885e847358975fb866b03cf17dc","questions":400,"answered":400,"when":"","ended":"2026-10-03T00:06:45Z","by":"","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.9175,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"brier","value":0.1476,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"ece","value":0.0741,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"aurc","value":0.0235,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":400,"of":400}],"notes":"device cuda; dtype bf16; preds sha256:6f80a4c8de05e9bc6f4481f5121235ca07c69be2ec8d9f2ede7c8d3df6864add; scores sha256:1ec811cfb085ac209c7b094b408b00ef95d3d3c73b7ae7bdf6c52fc314d112e0","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"app.guardrails_jailbreak-harness-multilingual-laya","benchmark":"app.guardrails_jailbreak","split":"harness","system":"laya","version":"convaiinnovations/laya-multilingual@e4e9ddf21a7b1903b7acffd8814ad4307bf63a67","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"735e240323cd7c9f0c880b7d8c62d89f40be2164","questions":400,"answered":400,"when":"","ended":"2026-09-25T17:33:56Z","by":"","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.805,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"brier","value":0.3347,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"ece","value":0.1528,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"aurc","value":0.1372,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"zero_prob","value":0.0025,"lo":null,"hi":null,"n":400,"of":400}],"notes":"runtime laya 0.3.20; device cpu; dtype f32; preds sha256:b4176c9138e1a9c97d681c2f6cb14d853a055964d84a9a55c0b2e0c4568a035a","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"app.guardrails_jailbreak-harness-routed-laya","benchmark":"app.guardrails_jailbreak","split":"harness","system":"laya","version":"convaiinnovations/laya@55cf4c4ebb4ebe31b2550e8bdf3bd21b99753851+convaiinnovations/laya-multilingual@e4e9ddf21a7b1903b7acffd8814ad4307bf63a67","baseline":true,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"735e240323cd7c9f0c880b7d8c62d89f40be2164","questions":400,"answered":400,"when":"","ended":"2026-09-25T17:33:56Z","by":"","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.705,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"brier","value":0.5427,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"ece","value":0.2617,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"aurc","value":0.2304,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"zero_prob","value":0.0025,"lo":null,"hi":null,"n":400,"of":400}],"notes":"runtime laya 0.3.20; device cpu; dtype f32; preds sha256:b4176c9138e1a9c97d681c2f6cb14d853a055964d84a9a55c0b2e0c4568a035a","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"app.guardrails_jailbreak-harness-typed-decisions-laya","benchmark":"app.guardrails_jailbreak","split":"harness","system":"laya","version":"convaiinnovations/laya-typed-decisions@1a793eb568e6718f15941d08f85432581df534e3","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"735e240323cd7c9f0c880b7d8c62d89f40be2164","questions":400,"answered":400,"when":"","ended":"2026-09-25T17:33:56Z","by":"","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.7625,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"brier","value":0.3365,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"ece","value":0.0869,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"aurc","value":0.1134,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":400,"of":400}],"notes":"runtime laya 0.3.20; device cpu; dtype f32; preds sha256:b4176c9138e1a9c97d681c2f6cb14d853a055964d84a9a55c0b2e0c4568a035a","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"app.model_routing_domain-harness-20260917-jev","benchmark":"app.model_routing_domain","split":"harness","system":"jev","version":"typesafe/jev-1.13-20260917","baseline":true,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"e95c6ee6362fd66d6d80c711095e03f30bbeeffd","questions":399,"answered":399,"when":"","ended":"2026-09-25T02:55:39Z","by":"","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.9774,"lo":null,"hi":null,"n":399,"of":399},{"category":"all","metric":"brier","value":0.04,"lo":null,"hi":null,"n":399,"of":399},{"category":"all","metric":"ece","value":0.0138,"lo":null,"hi":null,"n":399,"of":399},{"category":"all","metric":"aurc","value":0.0028,"lo":null,"hi":null,"n":399,"of":399},{"category":"all","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":399,"of":399}],"notes":"preds sha256:79583326c68d6ef33a1d11acb9e158f11a1df842a8579f40fd37d471798b6ed2","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"app.model_routing_domain-harness-a5.df1c16a2-kai","benchmark":"app.model_routing_domain","split":"harness","system":"kai","version":"a5@df1c16a2fef7ec93f2b34a6a194f6dc3c2eee70e936473b802151543c5d46e83","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"611059b60f64ab686d6475cd76778b3fac52cd98","questions":399,"answered":399,"when":"","ended":"2026-09-27T21:09:54Z","by":"","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":1,"lo":null,"hi":null,"n":399,"of":399},{"category":"all","metric":"brier","value":0,"lo":null,"hi":null,"n":399,"of":399},{"category":"all","metric":"ece","value":0.0018,"lo":null,"hi":null,"n":399,"of":399},{"category":"all","metric":"aurc","value":0,"lo":null,"hi":null,"n":399,"of":399},{"category":"all","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":399,"of":399}],"notes":"device metal; dtype bf16; preds sha256:f28954731ec991ecbed3c1fbfc45980eeffef7744d8c12bcfc4cb160cb6f3eac; scores sha256:071b556a8eafd2c19d4f9a948b4f10251c9b253107c6afb21ad959ef72af4a77","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"app.model_routing_domain-harness-a6.a211cc70-kai","benchmark":"app.model_routing_domain","split":"harness","system":"kai","version":"a6@a211cc70103825bfbc9bfb0fbe970f8bf08da2efe0953cd8045a3cd07397f4bf","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"a1a4a73d524cecf8038e30c408ad30b66e71d490","questions":399,"answered":399,"when":"","ended":"2026-09-28T03:18:42Z","by":"","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":1,"lo":null,"hi":null,"n":399,"of":399},{"category":"all","metric":"brier","value":0,"lo":null,"hi":null,"n":399,"of":399},{"category":"all","metric":"ece","value":0.0004,"lo":null,"hi":null,"n":399,"of":399},{"category":"all","metric":"aurc","value":0,"lo":null,"hi":null,"n":399,"of":399},{"category":"all","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":399,"of":399}],"notes":"device metal; dtype bf16; preds sha256:43b1bd476a72a536ee96fbc934a065a808fdb35395e694d9af873e462b3d8866; scores sha256:3760df4b2c4cdcccb225bbb446b2ddcff998e09ab723e36ee466f771dc355a7e","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"app.model_routing_domain-harness-a7.0834a74f-kai","benchmark":"app.model_routing_domain","split":"harness","system":"kai","version":"a7@0834a74f2d140642a453373da09e5a128e3d2c8d9e8bb1dcaf86d7210a4ecdfc","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"a1a4a73d524cecf8038e30c408ad30b66e71d490","questions":399,"answered":399,"when":"","ended":"2026-09-28T12:24:45Z","by":"","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":1,"lo":null,"hi":null,"n":399,"of":399},{"category":"all","metric":"brier","value":0,"lo":null,"hi":null,"n":399,"of":399},{"category":"all","metric":"ece","value":0.0003,"lo":null,"hi":null,"n":399,"of":399},{"category":"all","metric":"aurc","value":0,"lo":null,"hi":null,"n":399,"of":399},{"category":"all","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":399,"of":399}],"notes":"device metal; dtype bf16; preds sha256:25bfec2f5ad49bc050cc9015161c7ef2dbf07f599f35e9dad153b4a5b6f7b543; scores sha256:ddca48bcf9816e7df2a187251d9f0f00de3b2fc24f22805b34b7bd650a048c81","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"app.model_routing_domain-harness-a8.0cfeef05-kai","benchmark":"app.model_routing_domain","split":"harness","system":"kai","version":"a8@0cfeef05c879d7f24bbde275a6c23e85e80b6baf06cf9311d529010688d4a697","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"a1a4a73d524cecf8038e30c408ad30b66e71d490","questions":399,"answered":399,"when":"","ended":"2026-09-28T16:33:11Z","by":"","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":1,"lo":null,"hi":null,"n":399,"of":399},{"category":"all","metric":"brier","value":0.0006,"lo":null,"hi":null,"n":399,"of":399},{"category":"all","metric":"ece","value":0.0012,"lo":null,"hi":null,"n":399,"of":399},{"category":"all","metric":"aurc","value":0,"lo":null,"hi":null,"n":399,"of":399},{"category":"all","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":399,"of":399}],"notes":"device metal; dtype bf16; preds sha256:d0a81ccbdc9eb7305fc196d7ad2c6be8c1df1ce5966ac964dabeeea231ae41d6; scores sha256:86b58e1f53447a8fb3b7963ea69f2bee40ffa97b9bda7b74b038064188998f10","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"app.model_routing_domain-harness-english-laya","benchmark":"app.model_routing_domain","split":"harness","system":"laya","version":"convaiinnovations/laya@55cf4c4ebb4ebe31b2550e8bdf3bd21b99753851","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"735e240323cd7c9f0c880b7d8c62d89f40be2164","questions":399,"answered":399,"when":"","ended":"2026-09-25T17:33:56Z","by":"","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.6391,"lo":null,"hi":null,"n":399,"of":399},{"category":"all","metric":"brier","value":0.4938,"lo":null,"hi":null,"n":399,"of":399},{"category":"all","metric":"ece","value":0.0889,"lo":null,"hi":null,"n":399,"of":399},{"category":"all","metric":"aurc","value":0.1758,"lo":null,"hi":null,"n":399,"of":399},{"category":"all","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":399,"of":399}],"notes":"runtime laya 0.3.20; device cpu; dtype f32; preds sha256:1213bbf65e2ff9a8b0b38354dccec05a5ecd5486beac525b3296db61eaac618f","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"app.model_routing_domain-harness-kai-1.0834a74f-kai","benchmark":"app.model_routing_domain","split":"harness","system":"kai","version":"kai-1@0834a74f2d140642a453373da09e5a128e3d2c8d9e8bb1dcaf86d7210a4ecdfc","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"2c74e2d61addc885e847358975fb866b03cf17dc","questions":399,"answered":399,"when":"","ended":"2026-10-03T00:06:45Z","by":"","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":1,"lo":null,"hi":null,"n":399,"of":399},{"category":"all","metric":"brier","value":0,"lo":null,"hi":null,"n":399,"of":399},{"category":"all","metric":"ece","value":0.0004,"lo":null,"hi":null,"n":399,"of":399},{"category":"all","metric":"aurc","value":0,"lo":null,"hi":null,"n":399,"of":399},{"category":"all","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":399,"of":399}],"notes":"device cuda; dtype bf16; preds sha256:5a17369685d00c5b34a4cc06f22225a53cb8877f936c980fc57484803585d1c3; scores sha256:1ec811cfb085ac209c7b094b408b00ef95d3d3c73b7ae7bdf6c52fc314d112e0","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"app.model_routing_domain-harness-multilingual-laya","benchmark":"app.model_routing_domain","split":"harness","system":"laya","version":"convaiinnovations/laya-multilingual@e4e9ddf21a7b1903b7acffd8814ad4307bf63a67","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"735e240323cd7c9f0c880b7d8c62d89f40be2164","questions":399,"answered":399,"when":"","ended":"2026-09-25T17:33:56Z","by":"","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.4411,"lo":null,"hi":null,"n":399,"of":399},{"category":"all","metric":"brier","value":0.7607,"lo":null,"hi":null,"n":399,"of":399},{"category":"all","metric":"ece","value":0.1957,"lo":null,"hi":null,"n":399,"of":399},{"category":"all","metric":"aurc","value":0.4835,"lo":null,"hi":null,"n":399,"of":399},{"category":"all","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":399,"of":399}],"notes":"runtime laya 0.3.20; device cpu; dtype f32; preds sha256:1213bbf65e2ff9a8b0b38354dccec05a5ecd5486beac525b3296db61eaac618f","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"app.model_routing_domain-harness-routed-laya","benchmark":"app.model_routing_domain","split":"harness","system":"laya","version":"convaiinnovations/laya@55cf4c4ebb4ebe31b2550e8bdf3bd21b99753851+convaiinnovations/laya-multilingual@e4e9ddf21a7b1903b7acffd8814ad4307bf63a67","baseline":true,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"735e240323cd7c9f0c880b7d8c62d89f40be2164","questions":399,"answered":399,"when":"","ended":"2026-09-25T17:33:56Z","by":"","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.6391,"lo":null,"hi":null,"n":399,"of":399},{"category":"all","metric":"brier","value":0.4938,"lo":null,"hi":null,"n":399,"of":399},{"category":"all","metric":"ece","value":0.0889,"lo":null,"hi":null,"n":399,"of":399},{"category":"all","metric":"aurc","value":0.1758,"lo":null,"hi":null,"n":399,"of":399},{"category":"all","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":399,"of":399}],"notes":"runtime laya 0.3.20; device cpu; dtype f32; preds sha256:1213bbf65e2ff9a8b0b38354dccec05a5ecd5486beac525b3296db61eaac618f","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"app.model_routing_domain-harness-typed-decisions-laya","benchmark":"app.model_routing_domain","split":"harness","system":"laya","version":"convaiinnovations/laya-typed-decisions@1a793eb568e6718f15941d08f85432581df534e3","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"735e240323cd7c9f0c880b7d8c62d89f40be2164","questions":399,"answered":399,"when":"","ended":"2026-09-25T17:33:56Z","by":"","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.6591,"lo":null,"hi":null,"n":399,"of":399},{"category":"all","metric":"brier","value":0.503,"lo":null,"hi":null,"n":399,"of":399},{"category":"all","metric":"ece","value":0.1058,"lo":null,"hi":null,"n":399,"of":399},{"category":"all","metric":"aurc","value":0.2484,"lo":null,"hi":null,"n":399,"of":399},{"category":"all","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":399,"of":399}],"notes":"runtime laya 0.3.20; device cpu; dtype f32; preds sha256:1213bbf65e2ff9a8b0b38354dccec05a5ecd5486beac525b3296db61eaac618f","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"app.moderation_toxicity-harness-20260917-jev","benchmark":"app.moderation_toxicity","split":"harness","system":"jev","version":"typesafe/jev-1.13-20260917","baseline":true,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"e95c6ee6362fd66d6d80c711095e03f30bbeeffd","questions":400,"answered":400,"when":"","ended":"2026-09-25T02:55:39Z","by":"","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.6625,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"brier","value":0.4316,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"ece","value":0.1768,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"aurc","value":0.1382,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":400,"of":400}],"notes":"preds sha256:bce8d3fbe33f008b8f718b403a1ad6347133e88c7c1da4cb6770a359d5466d9b","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"app.moderation_toxicity-harness-a5.df1c16a2-kai","benchmark":"app.moderation_toxicity","split":"harness","system":"kai","version":"a5@df1c16a2fef7ec93f2b34a6a194f6dc3c2eee70e936473b802151543c5d46e83","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"611059b60f64ab686d6475cd76778b3fac52cd98","questions":400,"answered":400,"when":"","ended":"2026-09-27T21:09:54Z","by":"","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.82,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"brier","value":0.3393,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"ece","value":0.1673,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"aurc","value":0.1066,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"zero_prob","value":0.0025,"lo":null,"hi":null,"n":400,"of":400}],"notes":"device metal; dtype bf16; preds sha256:f28954731ec991ecbed3c1fbfc45980eeffef7744d8c12bcfc4cb160cb6f3eac; scores sha256:071b556a8eafd2c19d4f9a948b4f10251c9b253107c6afb21ad959ef72af4a77","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"app.moderation_toxicity-harness-a6.a211cc70-kai","benchmark":"app.moderation_toxicity","split":"harness","system":"kai","version":"a6@a211cc70103825bfbc9bfb0fbe970f8bf08da2efe0953cd8045a3cd07397f4bf","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"a1a4a73d524cecf8038e30c408ad30b66e71d490","questions":400,"answered":400,"when":"","ended":"2026-09-28T03:18:42Z","by":"","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.8075,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"brier","value":0.3691,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"ece","value":0.187,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"aurc","value":0.1125,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"zero_prob","value":0.0025,"lo":null,"hi":null,"n":400,"of":400}],"notes":"device metal; dtype bf16; preds sha256:43b1bd476a72a536ee96fbc934a065a808fdb35395e694d9af873e462b3d8866; scores sha256:3760df4b2c4cdcccb225bbb446b2ddcff998e09ab723e36ee466f771dc355a7e","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"app.moderation_toxicity-harness-a7.0834a74f-kai","benchmark":"app.moderation_toxicity","split":"harness","system":"kai","version":"a7@0834a74f2d140642a453373da09e5a128e3d2c8d9e8bb1dcaf86d7210a4ecdfc","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"a1a4a73d524cecf8038e30c408ad30b66e71d490","questions":400,"answered":400,"when":"","ended":"2026-09-28T12:24:45Z","by":"","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.8025,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"brier","value":0.3841,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"ece","value":0.1912,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"aurc","value":0.1339,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"zero_prob","value":0.025,"lo":null,"hi":null,"n":400,"of":400}],"notes":"device metal; dtype bf16; preds sha256:25bfec2f5ad49bc050cc9015161c7ef2dbf07f599f35e9dad153b4a5b6f7b543; scores sha256:ddca48bcf9816e7df2a187251d9f0f00de3b2fc24f22805b34b7bd650a048c81","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"app.moderation_toxicity-harness-a8.0cfeef05-kai","benchmark":"app.moderation_toxicity","split":"harness","system":"kai","version":"a8@0cfeef05c879d7f24bbde275a6c23e85e80b6baf06cf9311d529010688d4a697","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"a1a4a73d524cecf8038e30c408ad30b66e71d490","questions":400,"answered":400,"when":"","ended":"2026-09-28T16:33:11Z","by":"","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.82,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"brier","value":0.348,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"ece","value":0.1718,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"aurc","value":0.1129,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"zero_prob","value":0.01,"lo":null,"hi":null,"n":400,"of":400}],"notes":"device metal; dtype bf16; preds sha256:d0a81ccbdc9eb7305fc196d7ad2c6be8c1df1ce5966ac964dabeeea231ae41d6; scores sha256:86b58e1f53447a8fb3b7963ea69f2bee40ffa97b9bda7b74b038064188998f10","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"app.moderation_toxicity-harness-english-laya","benchmark":"app.moderation_toxicity","split":"harness","system":"laya","version":"convaiinnovations/laya@55cf4c4ebb4ebe31b2550e8bdf3bd21b99753851","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"735e240323cd7c9f0c880b7d8c62d89f40be2164","questions":400,"answered":400,"when":"","ended":"2026-09-25T17:33:56Z","by":"","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.53,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"brier","value":0.6015,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"ece","value":0.2961,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"aurc","value":0.2811,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":400,"of":400}],"notes":"runtime laya 0.3.20; device cpu; dtype f32; preds sha256:f762d7933516f86904b1b6d1749c9963c4462af32bf04d32a4f4893fa11038bf","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"app.moderation_toxicity-harness-kai-1.0834a74f-kai","benchmark":"app.moderation_toxicity","split":"harness","system":"kai","version":"kai-1@0834a74f2d140642a453373da09e5a128e3d2c8d9e8bb1dcaf86d7210a4ecdfc","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"2c74e2d61addc885e847358975fb866b03cf17dc","questions":400,"answered":400,"when":"","ended":"2026-10-03T00:06:45Z","by":"","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.8025,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"brier","value":0.3855,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"ece","value":0.1954,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"aurc","value":0.1339,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"zero_prob","value":0.025,"lo":null,"hi":null,"n":400,"of":400}],"notes":"device cuda; dtype bf16; preds sha256:b32408f970c4588513e7e968a387392d11620468bd3b5a8e9ac46d782bbea3a4; scores sha256:1ec811cfb085ac209c7b094b408b00ef95d3d3c73b7ae7bdf6c52fc314d112e0","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"app.moderation_toxicity-harness-multilingual-laya","benchmark":"app.moderation_toxicity","split":"harness","system":"laya","version":"convaiinnovations/laya-multilingual@e4e9ddf21a7b1903b7acffd8814ad4307bf63a67","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"735e240323cd7c9f0c880b7d8c62d89f40be2164","questions":400,"answered":400,"when":"","ended":"2026-09-25T17:33:56Z","by":"","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.535,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"brier","value":0.7377,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"ece","value":0.3729,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"aurc","value":0.2516,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":400,"of":400}],"notes":"runtime laya 0.3.20; device cpu; dtype f32; preds sha256:f762d7933516f86904b1b6d1749c9963c4462af32bf04d32a4f4893fa11038bf","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"app.moderation_toxicity-harness-routed-laya","benchmark":"app.moderation_toxicity","split":"harness","system":"laya","version":"convaiinnovations/laya@55cf4c4ebb4ebe31b2550e8bdf3bd21b99753851+convaiinnovations/laya-multilingual@e4e9ddf21a7b1903b7acffd8814ad4307bf63a67","baseline":true,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"735e240323cd7c9f0c880b7d8c62d89f40be2164","questions":400,"answered":400,"when":"","ended":"2026-09-25T17:33:56Z","by":"","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.53,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"brier","value":0.6015,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"ece","value":0.2967,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"aurc","value":0.2811,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":400,"of":400}],"notes":"runtime laya 0.3.20; device cpu; dtype f32; preds sha256:f762d7933516f86904b1b6d1749c9963c4462af32bf04d32a4f4893fa11038bf","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"app.moderation_toxicity-harness-typed-decisions-laya","benchmark":"app.moderation_toxicity","split":"harness","system":"laya","version":"convaiinnovations/laya-typed-decisions@1a793eb568e6718f15941d08f85432581df534e3","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"735e240323cd7c9f0c880b7d8c62d89f40be2164","questions":400,"answered":400,"when":"","ended":"2026-09-25T17:33:56Z","by":"","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.53,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"brier","value":0.5315,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"ece","value":0.244,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"aurc","value":0.2713,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":400,"of":400}],"notes":"runtime laya 0.3.20; device cpu; dtype f32; preds sha256:f762d7933516f86904b1b6d1749c9963c4462af32bf04d32a4f4893fa11038bf","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"app.phishing-harness-20260917-jev","benchmark":"app.phishing","split":"harness","system":"jev","version":"typesafe/jev-1.13-20260917","baseline":true,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"e95c6ee6362fd66d6d80c711095e03f30bbeeffd","questions":400,"answered":400,"when":"","ended":"2026-09-25T02:55:39Z","by":"","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.9,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"brier","value":0.1523,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"ece","value":0.0388,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"aurc","value":0.0208,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":400,"of":400}],"notes":"preds sha256:350ab0dba82eac868412ecfff93b72e578dd75b496ea32bf31cb96489a1373d7","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"app.phishing-harness-a5.df1c16a2-kai","benchmark":"app.phishing","split":"harness","system":"kai","version":"a5@df1c16a2fef7ec93f2b34a6a194f6dc3c2eee70e936473b802151543c5d46e83","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"611059b60f64ab686d6475cd76778b3fac52cd98","questions":400,"answered":400,"when":"","ended":"2026-09-27T21:09:54Z","by":"","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.99,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"brier","value":0.0197,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"ece","value":0.0097,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"aurc","value":0.0003,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":400,"of":400}],"notes":"device metal; dtype bf16; preds sha256:f28954731ec991ecbed3c1fbfc45980eeffef7744d8c12bcfc4cb160cb6f3eac; scores sha256:071b556a8eafd2c19d4f9a948b4f10251c9b253107c6afb21ad959ef72af4a77","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"app.phishing-harness-a6.a211cc70-kai","benchmark":"app.phishing","split":"harness","system":"kai","version":"a6@a211cc70103825bfbc9bfb0fbe970f8bf08da2efe0953cd8045a3cd07397f4bf","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"a1a4a73d524cecf8038e30c408ad30b66e71d490","questions":400,"answered":400,"when":"","ended":"2026-09-28T03:18:42Z","by":"","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.9925,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"brier","value":0.0149,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"ece","value":0.0073,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"aurc","value":0.0002,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":400,"of":400}],"notes":"device metal; dtype bf16; preds sha256:43b1bd476a72a536ee96fbc934a065a808fdb35395e694d9af873e462b3d8866; scores sha256:3760df4b2c4cdcccb225bbb446b2ddcff998e09ab723e36ee466f771dc355a7e","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"app.phishing-harness-a7.0834a74f-kai","benchmark":"app.phishing","split":"harness","system":"kai","version":"a7@0834a74f2d140642a453373da09e5a128e3d2c8d9e8bb1dcaf86d7210a4ecdfc","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"a1a4a73d524cecf8038e30c408ad30b66e71d490","questions":400,"answered":400,"when":"","ended":"2026-09-28T12:24:45Z","by":"","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.99,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"brier","value":0.02,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"ece","value":0.01,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"aurc","value":0.0002,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":400,"of":400}],"notes":"device metal; dtype bf16; preds sha256:25bfec2f5ad49bc050cc9015161c7ef2dbf07f599f35e9dad153b4a5b6f7b543; scores sha256:ddca48bcf9816e7df2a187251d9f0f00de3b2fc24f22805b34b7bd650a048c81","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"app.phishing-harness-a8.0cfeef05-kai","benchmark":"app.phishing","split":"harness","system":"kai","version":"a8@0cfeef05c879d7f24bbde275a6c23e85e80b6baf06cf9311d529010688d4a697","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"a1a4a73d524cecf8038e30c408ad30b66e71d490","questions":400,"answered":400,"when":"","ended":"2026-09-28T16:33:11Z","by":"","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.9925,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"brier","value":0.0152,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"ece","value":0.008,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"aurc","value":0.0002,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":400,"of":400}],"notes":"device metal; dtype bf16; preds sha256:d0a81ccbdc9eb7305fc196d7ad2c6be8c1df1ce5966ac964dabeeea231ae41d6; scores sha256:86b58e1f53447a8fb3b7963ea69f2bee40ffa97b9bda7b74b038064188998f10","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"app.phishing-harness-english-laya","benchmark":"app.phishing","split":"harness","system":"laya","version":"convaiinnovations/laya@55cf4c4ebb4ebe31b2550e8bdf3bd21b99753851","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"735e240323cd7c9f0c880b7d8c62d89f40be2164","questions":400,"answered":400,"when":"","ended":"2026-09-25T17:33:56Z","by":"","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.98,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"brier","value":0.0292,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"ece","value":0.0121,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"aurc","value":0.0007,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":400,"of":400}],"notes":"runtime laya 0.3.20; device cpu; dtype f32; preds sha256:4ca6f4caa290788b507a735d641595bc5af4df5fbe9bb773e55337a2ff554587","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"app.phishing-harness-kai-1.0834a74f-kai","benchmark":"app.phishing","split":"harness","system":"kai","version":"kai-1@0834a74f2d140642a453373da09e5a128e3d2c8d9e8bb1dcaf86d7210a4ecdfc","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"2c74e2d61addc885e847358975fb866b03cf17dc","questions":400,"answered":400,"when":"","ended":"2026-10-03T00:06:45Z","by":"","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.99,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"brier","value":0.02,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"ece","value":0.01,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"aurc","value":0.0002,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":400,"of":400}],"notes":"device cuda; dtype bf16; preds sha256:7138ccb18a6da0ee563487551b92417f89886751b634839585d2a2519647369b; scores sha256:1ec811cfb085ac209c7b094b408b00ef95d3d3c73b7ae7bdf6c52fc314d112e0","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"app.phishing-harness-multilingual-laya","benchmark":"app.phishing","split":"harness","system":"laya","version":"convaiinnovations/laya-multilingual@e4e9ddf21a7b1903b7acffd8814ad4307bf63a67","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"735e240323cd7c9f0c880b7d8c62d89f40be2164","questions":400,"answered":400,"when":"","ended":"2026-09-25T17:33:56Z","by":"","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.9925,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"brier","value":0.0138,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"ece","value":0.0063,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"aurc","value":0.0002,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":400,"of":400}],"notes":"runtime laya 0.3.20; device cpu; dtype f32; preds sha256:4ca6f4caa290788b507a735d641595bc5af4df5fbe9bb773e55337a2ff554587","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"app.phishing-harness-routed-laya","benchmark":"app.phishing","split":"harness","system":"laya","version":"convaiinnovations/laya@55cf4c4ebb4ebe31b2550e8bdf3bd21b99753851+convaiinnovations/laya-multilingual@e4e9ddf21a7b1903b7acffd8814ad4307bf63a67","baseline":true,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"735e240323cd7c9f0c880b7d8c62d89f40be2164","questions":400,"answered":400,"when":"","ended":"2026-09-25T17:33:56Z","by":"","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.9825,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"brier","value":0.026,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"ece","value":0.0101,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"aurc","value":0.0005,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":400,"of":400}],"notes":"runtime laya 0.3.20; device cpu; dtype f32; preds sha256:4ca6f4caa290788b507a735d641595bc5af4df5fbe9bb773e55337a2ff554587","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"app.phishing-harness-typed-decisions-laya","benchmark":"app.phishing","split":"harness","system":"laya","version":"convaiinnovations/laya-typed-decisions@1a793eb568e6718f15941d08f85432581df534e3","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"735e240323cd7c9f0c880b7d8c62d89f40be2164","questions":400,"answered":400,"when":"","ended":"2026-09-25T17:33:56Z","by":"","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.94,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"brier","value":0.1062,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"ece","value":0.0762,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"aurc","value":0.0146,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":400,"of":400}],"notes":"runtime laya 0.3.20; device cpu; dtype f32; preds sha256:4ca6f4caa290788b507a735d641595bc5af4df5fbe9bb773e55337a2ff554587","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"app.rag_relevance-harness-20260917-jev","benchmark":"app.rag_relevance","split":"harness","system":"jev","version":"typesafe/jev-1.13-20260917","baseline":true,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"e95c6ee6362fd66d6d80c711095e03f30bbeeffd","questions":400,"answered":400,"when":"","ended":"2026-09-25T02:55:39Z","by":"","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.62,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"brier","value":0.6282,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"ece","value":0.2788,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"aurc","value":0.3309,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":400,"of":400}],"notes":"preds sha256:90a32f1ab938513b505e27cbe005ec504a4f8d8cea1c46b64fb26a011bcc5677","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"app.rag_relevance-harness-a5.df1c16a2-kai","benchmark":"app.rag_relevance","split":"harness","system":"kai","version":"a5@df1c16a2fef7ec93f2b34a6a194f6dc3c2eee70e936473b802151543c5d46e83","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"611059b60f64ab686d6475cd76778b3fac52cd98","questions":400,"answered":400,"when":"","ended":"2026-09-27T21:09:54Z","by":"","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.6675,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"brier","value":0.4184,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"ece","value":0.0373,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"aurc","value":0.2264,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":400,"of":400}],"notes":"device metal; dtype bf16; preds sha256:f28954731ec991ecbed3c1fbfc45980eeffef7744d8c12bcfc4cb160cb6f3eac; scores sha256:071b556a8eafd2c19d4f9a948b4f10251c9b253107c6afb21ad959ef72af4a77","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"app.rag_relevance-harness-a6.a211cc70-kai","benchmark":"app.rag_relevance","split":"harness","system":"kai","version":"a6@a211cc70103825bfbc9bfb0fbe970f8bf08da2efe0953cd8045a3cd07397f4bf","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"a1a4a73d524cecf8038e30c408ad30b66e71d490","questions":400,"answered":400,"when":"","ended":"2026-09-28T03:18:42Z","by":"","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.6525,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"brier","value":0.4128,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"ece","value":0.0522,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"aurc","value":0.2184,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":400,"of":400}],"notes":"device metal; dtype bf16; preds sha256:43b1bd476a72a536ee96fbc934a065a808fdb35395e694d9af873e462b3d8866; scores sha256:3760df4b2c4cdcccb225bbb446b2ddcff998e09ab723e36ee466f771dc355a7e","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"app.rag_relevance-harness-a7.0834a74f-kai","benchmark":"app.rag_relevance","split":"harness","system":"kai","version":"a7@0834a74f2d140642a453373da09e5a128e3d2c8d9e8bb1dcaf86d7210a4ecdfc","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"a1a4a73d524cecf8038e30c408ad30b66e71d490","questions":400,"answered":400,"when":"","ended":"2026-09-28T12:24:45Z","by":"","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.675,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"brier","value":0.419,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"ece","value":0.0295,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"aurc","value":0.232,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":400,"of":400}],"notes":"device metal; dtype bf16; preds sha256:25bfec2f5ad49bc050cc9015161c7ef2dbf07f599f35e9dad153b4a5b6f7b543; scores sha256:ddca48bcf9816e7df2a187251d9f0f00de3b2fc24f22805b34b7bd650a048c81","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"app.rag_relevance-harness-a8.0cfeef05-kai","benchmark":"app.rag_relevance","split":"harness","system":"kai","version":"a8@0cfeef05c879d7f24bbde275a6c23e85e80b6baf06cf9311d529010688d4a697","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"a1a4a73d524cecf8038e30c408ad30b66e71d490","questions":400,"answered":400,"when":"","ended":"2026-09-28T16:33:11Z","by":"","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.69,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"brier","value":0.4088,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"ece","value":0.0315,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"aurc","value":0.2302,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":400,"of":400}],"notes":"device metal; dtype bf16; preds sha256:d0a81ccbdc9eb7305fc196d7ad2c6be8c1df1ce5966ac964dabeeea231ae41d6; scores sha256:86b58e1f53447a8fb3b7963ea69f2bee40ffa97b9bda7b74b038064188998f10","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"app.rag_relevance-harness-english-laya","benchmark":"app.rag_relevance","split":"harness","system":"laya","version":"convaiinnovations/laya@55cf4c4ebb4ebe31b2550e8bdf3bd21b99753851","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"735e240323cd7c9f0c880b7d8c62d89f40be2164","questions":400,"answered":400,"when":"","ended":"2026-09-25T17:33:56Z","by":"","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.625,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"brier","value":0.4656,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"ece","value":0.1175,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"aurc","value":0.2795,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":400,"of":400}],"notes":"runtime laya 0.3.20; device cpu; dtype f32; preds sha256:ff9000a8f88dc13c42afb13dc191a2d7930c4d294c55e0f85e7b6d5d3af7d7db","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"app.rag_relevance-harness-kai-1.0834a74f-kai","benchmark":"app.rag_relevance","split":"harness","system":"kai","version":"kai-1@0834a74f2d140642a453373da09e5a128e3d2c8d9e8bb1dcaf86d7210a4ecdfc","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"2c74e2d61addc885e847358975fb866b03cf17dc","questions":400,"answered":400,"when":"","ended":"2026-10-03T00:06:45Z","by":"","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.68,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"brier","value":0.4189,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"ece","value":0.0277,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"aurc","value":0.2323,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":400,"of":400}],"notes":"device cuda; dtype bf16; preds sha256:22e5c20c15382e4a9981dc541779fbe6f31aa4abbb5eb10f40b61f58dc8b698e; scores sha256:1ec811cfb085ac209c7b094b408b00ef95d3d3c73b7ae7bdf6c52fc314d112e0","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"app.rag_relevance-harness-multilingual-laya","benchmark":"app.rag_relevance","split":"harness","system":"laya","version":"convaiinnovations/laya-multilingual@e4e9ddf21a7b1903b7acffd8814ad4307bf63a67","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"735e240323cd7c9f0c880b7d8c62d89f40be2164","questions":400,"answered":400,"when":"","ended":"2026-09-25T17:33:56Z","by":"","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.6725,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"brier","value":0.4859,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"ece","value":0.1689,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"aurc","value":0.2619,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":400,"of":400}],"notes":"runtime laya 0.3.20; device cpu; dtype f32; preds sha256:ff9000a8f88dc13c42afb13dc191a2d7930c4d294c55e0f85e7b6d5d3af7d7db","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"app.rag_relevance-harness-routed-laya","benchmark":"app.rag_relevance","split":"harness","system":"laya","version":"convaiinnovations/laya@55cf4c4ebb4ebe31b2550e8bdf3bd21b99753851+convaiinnovations/laya-multilingual@e4e9ddf21a7b1903b7acffd8814ad4307bf63a67","baseline":true,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"735e240323cd7c9f0c880b7d8c62d89f40be2164","questions":400,"answered":400,"when":"","ended":"2026-09-25T17:33:56Z","by":"","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.625,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"brier","value":0.4656,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"ece","value":0.1175,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"aurc","value":0.2795,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":400,"of":400}],"notes":"runtime laya 0.3.20; device cpu; dtype f32; preds sha256:ff9000a8f88dc13c42afb13dc191a2d7930c4d294c55e0f85e7b6d5d3af7d7db","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"app.rag_relevance-harness-typed-decisions-laya","benchmark":"app.rag_relevance","split":"harness","system":"laya","version":"convaiinnovations/laya-typed-decisions@1a793eb568e6718f15941d08f85432581df534e3","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"735e240323cd7c9f0c880b7d8c62d89f40be2164","questions":400,"answered":400,"when":"","ended":"2026-09-25T17:33:56Z","by":"","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.625,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"brier","value":0.4458,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"ece","value":0.0419,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"aurc","value":0.2798,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":400,"of":400}],"notes":"runtime laya 0.3.20; device cpu; dtype f32; preds sha256:ff9000a8f88dc13c42afb13dc191a2d7930c4d294c55e0f85e7b6d5d3af7d7db","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"app.support_triage-harness-20260917-jev","benchmark":"app.support_triage","split":"harness","system":"jev","version":"typesafe/jev-1.13-20260917","baseline":true,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"e95c6ee6362fd66d6d80c711095e03f30bbeeffd","questions":400,"answered":400,"when":"","ended":"2026-09-25T02:55:39Z","by":"","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.365,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"brier","value":1.0557,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"ece","value":0.4827,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"aurc","value":0.4868,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"zero_prob","value":0.335,"lo":null,"hi":null,"n":400,"of":400}],"notes":"preds sha256:4af3a3a23d05ea9b3a4249287581ecaa0dbfe2662c4e6f0da5d4a5214b485aed","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"app.support_triage-harness-a5.df1c16a2-kai","benchmark":"app.support_triage","split":"harness","system":"kai","version":"a5@df1c16a2fef7ec93f2b34a6a194f6dc3c2eee70e936473b802151543c5d46e83","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"611059b60f64ab686d6475cd76778b3fac52cd98","questions":400,"answered":400,"when":"","ended":"2026-09-27T21:09:54Z","by":"","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.49,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"brier","value":0.7047,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"ece","value":0.1605,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"aurc","value":0.3218,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":400,"of":400}],"notes":"device metal; dtype bf16; preds sha256:f28954731ec991ecbed3c1fbfc45980eeffef7744d8c12bcfc4cb160cb6f3eac; scores sha256:071b556a8eafd2c19d4f9a948b4f10251c9b253107c6afb21ad959ef72af4a77","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"app.support_triage-harness-a6.a211cc70-kai","benchmark":"app.support_triage","split":"harness","system":"kai","version":"a6@a211cc70103825bfbc9bfb0fbe970f8bf08da2efe0953cd8045a3cd07397f4bf","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"a1a4a73d524cecf8038e30c408ad30b66e71d490","questions":400,"answered":400,"when":"","ended":"2026-09-28T03:18:42Z","by":"","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.435,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"brier","value":0.9616,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"ece","value":0.4337,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"aurc","value":0.388,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"zero_prob","value":0.025,"lo":null,"hi":null,"n":400,"of":400}],"notes":"device metal; dtype bf16; preds sha256:43b1bd476a72a536ee96fbc934a065a808fdb35395e694d9af873e462b3d8866; scores sha256:3760df4b2c4cdcccb225bbb446b2ddcff998e09ab723e36ee466f771dc355a7e","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"app.support_triage-harness-a7.0834a74f-kai","benchmark":"app.support_triage","split":"harness","system":"kai","version":"a7@0834a74f2d140642a453373da09e5a128e3d2c8d9e8bb1dcaf86d7210a4ecdfc","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"a1a4a73d524cecf8038e30c408ad30b66e71d490","questions":400,"answered":400,"when":"","ended":"2026-09-28T12:24:45Z","by":"","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.4425,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"brier","value":0.975,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"ece","value":0.4413,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"aurc","value":0.399,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"zero_prob","value":0.025,"lo":null,"hi":null,"n":400,"of":400}],"notes":"device metal; dtype bf16; preds sha256:25bfec2f5ad49bc050cc9015161c7ef2dbf07f599f35e9dad153b4a5b6f7b543; scores sha256:ddca48bcf9816e7df2a187251d9f0f00de3b2fc24f22805b34b7bd650a048c81","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"app.support_triage-harness-a8.0cfeef05-kai","benchmark":"app.support_triage","split":"harness","system":"kai","version":"a8@0cfeef05c879d7f24bbde275a6c23e85e80b6baf06cf9311d529010688d4a697","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"a1a4a73d524cecf8038e30c408ad30b66e71d490","questions":400,"answered":400,"when":"","ended":"2026-09-28T16:33:11Z","by":"","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.435,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"brier","value":0.9452,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"ece","value":0.4248,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"aurc","value":0.3622,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"zero_prob","value":0.0225,"lo":null,"hi":null,"n":400,"of":400}],"notes":"device metal; dtype bf16; preds sha256:d0a81ccbdc9eb7305fc196d7ad2c6be8c1df1ce5966ac964dabeeea231ae41d6; scores sha256:86b58e1f53447a8fb3b7963ea69f2bee40ffa97b9bda7b74b038064188998f10","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"app.support_triage-harness-english-laya","benchmark":"app.support_triage","split":"harness","system":"laya","version":"convaiinnovations/laya@55cf4c4ebb4ebe31b2550e8bdf3bd21b99753851","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"735e240323cd7c9f0c880b7d8c62d89f40be2164","questions":400,"answered":400,"when":"","ended":"2026-09-25T17:33:56Z","by":"","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.5025,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"brier","value":0.6438,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"ece","value":0.0907,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"aurc","value":0.3208,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":400,"of":400}],"notes":"runtime laya 0.3.20; device cpu; dtype f32; preds sha256:6d50a06c335837fd1f1126c26558f4e5fc2a7154d3415f19d750a73664be2d6a","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"app.support_triage-harness-kai-1.0834a74f-kai","benchmark":"app.support_triage","split":"harness","system":"kai","version":"kai-1@0834a74f2d140642a453373da09e5a128e3d2c8d9e8bb1dcaf86d7210a4ecdfc","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"2c74e2d61addc885e847358975fb866b03cf17dc","questions":400,"answered":400,"when":"","ended":"2026-10-03T00:06:45Z","by":"","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.44,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"brier","value":0.9766,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"ece","value":0.4441,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"aurc","value":0.401,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"zero_prob","value":0.025,"lo":null,"hi":null,"n":400,"of":400}],"notes":"device cuda; dtype bf16; preds sha256:1c93dcfeb484857e3ee16a52695b96956f140672a690420485b89c6890ec1d84; scores sha256:1ec811cfb085ac209c7b094b408b00ef95d3d3c73b7ae7bdf6c52fc314d112e0","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"app.support_triage-harness-multilingual-laya","benchmark":"app.support_triage","split":"harness","system":"laya","version":"convaiinnovations/laya-multilingual@e4e9ddf21a7b1903b7acffd8814ad4307bf63a67","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"735e240323cd7c9f0c880b7d8c62d89f40be2164","questions":400,"answered":400,"when":"","ended":"2026-09-25T17:33:56Z","by":"","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.54,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"brier","value":0.5721,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"ece","value":0.0862,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"aurc","value":0.2401,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":400,"of":400}],"notes":"runtime laya 0.3.20; device cpu; dtype f32; preds sha256:6d50a06c335837fd1f1126c26558f4e5fc2a7154d3415f19d750a73664be2d6a","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"app.support_triage-harness-routed-laya","benchmark":"app.support_triage","split":"harness","system":"laya","version":"convaiinnovations/laya@55cf4c4ebb4ebe31b2550e8bdf3bd21b99753851+convaiinnovations/laya-multilingual@e4e9ddf21a7b1903b7acffd8814ad4307bf63a67","baseline":true,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"735e240323cd7c9f0c880b7d8c62d89f40be2164","questions":400,"answered":400,"when":"","ended":"2026-09-25T17:33:56Z","by":"","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.5025,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"brier","value":0.6438,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"ece","value":0.0907,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"aurc","value":0.3208,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":400,"of":400}],"notes":"runtime laya 0.3.20; device cpu; dtype f32; preds sha256:6d50a06c335837fd1f1126c26558f4e5fc2a7154d3415f19d750a73664be2d6a","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"app.support_triage-harness-typed-decisions-laya","benchmark":"app.support_triage","split":"harness","system":"laya","version":"convaiinnovations/laya-typed-decisions@1a793eb568e6718f15941d08f85432581df534e3","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"735e240323cd7c9f0c880b7d8c62d89f40be2164","questions":400,"answered":400,"when":"","ended":"2026-09-25T17:33:56Z","by":"","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.505,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"brier","value":0.6603,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"ece","value":0.1149,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"aurc","value":0.3274,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":400,"of":400}],"notes":"runtime laya 0.3.20; device cpu; dtype f32; preds sha256:6d50a06c335837fd1f1126c26558f4e5fc2a7154d3415f19d750a73664be2d6a","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"cap.cardinality-cases-jev-1.13-jev","benchmark":"cap.cardinality","split":"cases","system":"jev","version":"typesafe/jev-1.13","baseline":true,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"","store_digest":"","facts_digest":"","commit":"735e240323cd7c9f0c880b7d8c62d89f40be2164","questions":null,"answered":null,"when":"","ended":"2026-09-27T10:05:18Z","by":"ra","completion":"unknown","measures":[{"category":"all","metric":"k1000.accall","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k1000.bi.accall","value":0.64,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k1000.bi.accuracy","value":0.64,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k1000.bi.brier","value":0.5481,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k1000.bi.ece","value":0.2198,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k1000.bi.nll","value":9.5674,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k1000.bi.recall1","value":0.64,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k1000.bi.recall20","value":0.655,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k1000.bi.recall5","value":0.655,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k1000.bi.rejected","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k1000.bi.unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k1000.bi.usd","value":0.0071,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k1000.rejected","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k1000.sl.accall","value":0.035,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k1000.sl.accuracy","value":0.035,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k1000.sl.brier","value":1.4346,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k1000.sl.ece","value":0.5853,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k1000.sl.nll","value":26.5488,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k1000.sl.recall1","value":0.035,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k1000.sl.recall20","value":0.04,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k1000.sl.recall5","value":0.04,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k1000.sl.rejected","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k1000.sl.unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k1000.sl.usd","value":0.0068,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k1000.unanswered","value":200,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k1000.usd","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k10000.accall","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k10000.bi.accall","value":0.4,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k10000.bi.accuracy","value":0.4,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k10000.bi.brier","value":0.9115,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k10000.bi.ece","value":0.3643,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k10000.bi.nll","value":16.4607,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k10000.bi.recall1","value":0.4,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k10000.bi.recall20","value":0.405,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k10000.bi.recall5","value":0.405,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k10000.bi.rejected","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k10000.bi.unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k10000.bi.usd","value":0.007,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k10000.rejected","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k10000.sl.accall","value":0.015,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k10000.sl.accuracy","value":0.015,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k10000.sl.brier","value":1.4585,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k10000.sl.ece","value":0.5989,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k10000.sl.nll","value":27.2166,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k10000.sl.recall1","value":0.015,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k10000.sl.recall20","value":0.015,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k10000.sl.recall5","value":0.015,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k10000.sl.rejected","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k10000.sl.unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k10000.sl.usd","value":0.0069,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k10000.unanswered","value":200,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k10000.usd","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k100000.accall","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k100000.bi.accall","value":0.26,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k100000.bi.accuracy","value":0.26,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k100000.bi.brier","value":1.0509,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k100000.bi.ece","value":0.3938,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k100000.bi.nll","value":20.4628,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k100000.bi.recall1","value":0.26,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k100000.bi.recall20","value":0.26,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k100000.bi.recall5","value":0.26,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k100000.bi.rejected","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k100000.bi.unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k100000.bi.usd","value":0.0017,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k100000.rejected","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k100000.sl.accall","value":0.02,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k100000.sl.accuracy","value":0.02,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k100000.sl.brier","value":1.4745,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k100000.sl.ece","value":0.6256,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k100000.sl.nll","value":27.0784,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k100000.sl.recall1","value":0.02,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k100000.sl.recall20","value":0.02,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k100000.sl.recall5","value":0.02,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k100000.sl.rejected","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k100000.sl.unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k100000.sl.usd","value":0.0017,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k100000.unanswered","value":50,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k100000.usd","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k150.accall","value":0.94,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k150.accuracy","value":0.94,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k150.bi.accall","value":0.825,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k150.bi.accuracy","value":0.825,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k150.bi.brier","value":0.2934,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k150.bi.ece","value":0.1044,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k150.bi.nll","value":4.1037,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k150.bi.recall1","value":0.825,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k150.bi.recall20","value":0.855,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k150.bi.recall5","value":0.855,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k150.bi.rejected","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k150.bi.unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k150.bi.usd","value":0.0097,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k150.brier","value":0.1074,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k150.ece","value":0.0317,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k150.nll","value":0.4346,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k150.recall1","value":0.9375,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k150.recall20","value":0.99,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k150.recall5","value":0.99,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k150.rejected","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k150.sl.accall","value":0.1925,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k150.sl.accuracy","value":0.1925,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k150.sl.brier","value":1.2677,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k150.sl.ece","value":0.5504,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k150.sl.nll","value":22.189,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k150.sl.recall1","value":0.1925,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k150.sl.recall20","value":0.1975,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k150.sl.recall5","value":0.1975,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k150.sl.rejected","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k150.sl.unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k150.sl.usd","value":0.0102,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k150.unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k150.usd","value":0.041,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k16.accall","value":0.8525,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k16.accuracy","value":0.8525,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k16.brier","value":0.2212,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k16.ece","value":0.067,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k16.nll","value":1.1469,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k16.recall1","value":0.8525,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k16.recall20","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k16.recall5","value":0.965,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k16.rejected","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k16.unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k16.usd","value":0.0097,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k4.accall","value":0.905,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k4.accuracy","value":0.905,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k4.brier","value":0.1402,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k4.ece","value":0.0465,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k4.nll","value":0.4411,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k4.recall1","value":0.905,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k4.recall20","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k4.recall5","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k4.rejected","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k4.unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k4.usd","value":0.0062,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k77.accall","value":0.8425,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k77.accuracy","value":0.8425,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k77.bi.accall","value":0.8075,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k77.bi.accuracy","value":0.8075,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k77.bi.brier","value":0.2997,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k77.bi.ece","value":0.0736,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k77.bi.nll","value":2.2036,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k77.bi.recall1","value":0.805,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k77.bi.recall20","value":0.9325,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k77.bi.recall5","value":0.93,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k77.bi.rejected","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k77.bi.unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k77.bi.usd","value":0.0109,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k77.brier","value":0.2574,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k77.ece","value":0.0682,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k77.nll","value":1.2902,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k77.recall1","value":0.8425,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k77.recall20","value":0.9675,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k77.recall5","value":0.96,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k77.rejected","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k77.sl.accall","value":0.0775,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k77.sl.accuracy","value":0.0775,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k77.sl.brier","value":1.4965,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k77.sl.ece","value":0.6704,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k77.sl.nll","value":25.4291,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k77.sl.recall1","value":0.0775,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k77.sl.recall20","value":0.08,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k77.sl.recall5","value":0.0775,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k77.sl.rejected","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k77.sl.unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k77.sl.usd","value":0.011,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k77.unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k77.usd","value":0.0272,"lo":null,"hi":null,"n":null,"of":null}],"notes":"","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"cap.cardinality-cases-kai-1-kai","benchmark":"cap.cardinality","split":"cases","system":"kai","version":"kai-1@0834a74f2d140642a453373da09e5a128e3d2c8d9e8bb1dcaf86d7210a4ecdfc","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"","store_digest":"","facts_digest":"","commit":"5138293ef73d16d2f6300d3cd3386a8ed11fe5bc","questions":null,"answered":null,"when":"","ended":"2026-10-03T00:20:27Z","by":"dgx","completion":"unknown","measures":[{"category":"all","metric":"k1000.accall","value":0.005,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k1000.accuracy","value":0.005,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k1000.bi.accall","value":0.03,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k1000.bi.accuracy","value":0.03,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k1000.bi.brier","value":1.0824,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k1000.bi.ece","value":0.253,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k1000.bi.nll","value":11.9627,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k1000.bi.recall1","value":0.03,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k1000.bi.recall20","value":0.655,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k1000.bi.recall5","value":0.135,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k1000.bi.rejected","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k1000.bi.unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k1000.brier","value":0.9982,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k1000.ece","value":0.0261,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k1000.nll","value":6.6967,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k1000.recall1","value":0.005,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k1000.recall20","value":0.1,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k1000.recall5","value":0.035,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k1000.rejected","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k1000.sl.accall","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k1000.sl.accuracy","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k1000.sl.brier","value":1.118,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k1000.sl.ece","value":0.2307,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k1000.sl.nll","value":26.6337,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k1000.sl.recall1","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k1000.sl.recall20","value":0.04,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k1000.sl.recall5","value":0.025,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k1000.sl.rejected","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k1000.sl.unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k1000.unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k10000.accall","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k10000.accuracy","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k10000.bi.accall","value":0.02,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k10000.bi.accuracy","value":0.02,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k10000.bi.brier","value":1.0923,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k10000.bi.ece","value":0.2318,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k10000.bi.nll","value":17.9489,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k10000.bi.recall1","value":0.02,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k10000.bi.recall20","value":0.405,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k10000.bi.recall5","value":0.075,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k10000.bi.rejected","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k10000.bi.unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k10000.brier","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k10000.ece","value":0.01,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k10000.nll","value":8.7704,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k10000.recall1","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k10000.recall20","value":0.035,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k10000.recall5","value":0.01,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k10000.rejected","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k10000.sl.accall","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k10000.sl.accuracy","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k10000.sl.brier","value":1.1302,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k10000.sl.ece","value":0.2365,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k10000.sl.nll","value":27.2571,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k10000.sl.recall1","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k10000.sl.recall20","value":0.015,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k10000.sl.recall5","value":0.01,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k10000.sl.rejected","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k10000.sl.unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k10000.unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k100000.accall","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k100000.accuracy","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k100000.bi.accall","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k100000.bi.accuracy","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k100000.bi.brier","value":1.0965,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k100000.bi.ece","value":0.1965,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k100000.bi.nll","value":21.4707,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k100000.bi.recall1","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k100000.bi.recall20","value":0.26,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k100000.bi.recall5","value":0.04,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k100000.bi.rejected","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k100000.bi.unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k100000.brier","value":1.001,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k100000.ece","value":0.0047,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k100000.nll","value":11.2211,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k100000.recall1","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k100000.recall20","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k100000.recall5","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k100000.rejected","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k100000.sl.accall","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k100000.sl.accuracy","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k100000.sl.brier","value":1.1271,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k100000.sl.ece","value":0.2351,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k100000.sl.nll","value":27.1246,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k100000.sl.recall1","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k100000.sl.recall20","value":0.02,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k100000.sl.recall5","value":0.02,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k100000.sl.rejected","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k100000.sl.unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k100000.unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k150.accall","value":0.9075,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k150.accuracy","value":0.9075,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k150.bi.accall","value":0.8025,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k150.bi.accuracy","value":0.8025,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k150.bi.brier","value":0.3219,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k150.bi.ece","value":0.1376,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k150.bi.nll","value":4.2481,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k150.bi.recall1","value":0.8025,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k150.bi.recall20","value":0.8575,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k150.bi.recall5","value":0.8525,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k150.bi.rejected","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k150.bi.unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k150.brier","value":0.15,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k150.ece","value":0.0409,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k150.nll","value":0.5465,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k150.recall1","value":0.9075,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k150.recall20","value":0.9875,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k150.recall5","value":0.9675,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k150.rejected","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k150.sl.accall","value":0.19,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k150.sl.accuracy","value":0.19,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k150.sl.brier","value":1.3418,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k150.sl.ece","value":0.6058,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k150.sl.nll","value":22.1634,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k150.sl.recall1","value":0.19,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k150.sl.recall20","value":0.2,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k150.sl.recall5","value":0.1975,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k150.sl.rejected","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k150.sl.unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k150.unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k16.accall","value":0.935,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k16.accuracy","value":0.935,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k16.brier","value":0.1135,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k16.ece","value":0.0516,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k16.nll","value":0.5631,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k16.recall1","value":0.935,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k16.recall20","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k16.recall5","value":0.99,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k16.rejected","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k16.unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k4.accall","value":0.9675,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k4.accuracy","value":0.9675,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k4.brier","value":0.0564,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k4.ece","value":0.028,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k4.nll","value":0.2471,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k4.recall1","value":0.9675,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k4.recall20","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k4.recall5","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k4.rejected","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k4.unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k77.accall","value":0.9075,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k77.accuracy","value":0.9075,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k77.bi.accall","value":0.8875,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k77.bi.accuracy","value":0.8875,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k77.bi.brier","value":0.1987,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k77.bi.ece","value":0.09,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k77.bi.nll","value":1.7013,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k77.bi.recall1","value":0.8875,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k77.bi.recall20","value":0.965,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k77.bi.recall5","value":0.9525,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k77.bi.rejected","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k77.bi.unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k77.brier","value":0.1638,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k77.ece","value":0.0747,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k77.nll","value":0.7448,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k77.recall1","value":0.9075,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k77.recall20","value":0.9925,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k77.recall5","value":0.9725,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k77.rejected","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k77.sl.accall","value":0.075,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k77.sl.accuracy","value":0.075,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k77.sl.brier","value":1.568,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k77.sl.ece","value":0.7244,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k77.sl.nll","value":25.4405,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k77.sl.recall1","value":0.075,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k77.sl.recall20","value":0.08,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k77.sl.recall5","value":0.0775,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k77.sl.rejected","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k77.sl.unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k77.unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null}],"notes":"device cuda; dtype f32; host dgx; corrected: ended and commit as hanzo-inc/kai 5138293 results/cardinality/cardinality.json records them, runs 2026-10-03 00:11:42Z and 00:20:27Z on weights 0834a74f","revision":"corrected","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"cap.cardinality-cases-routed-laya","benchmark":"cap.cardinality","split":"cases","system":"laya","version":"convaiinnovations/laya@55cf4c4ebb4ebe31b2550e8bdf3bd21b99753851+convaiinnovations/laya-multilingual@e4e9ddf21a7b1903b7acffd8814ad4307bf63a67","baseline":true,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"","store_digest":"","facts_digest":"","commit":"735e240323cd7c9f0c880b7d8c62d89f40be2164","questions":null,"answered":null,"when":"","ended":"2026-09-27T10:47:57Z","by":"ra","completion":"unknown","measures":[{"category":"all","metric":"k1000.accall","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k1000.bi.accall","value":0.425,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k1000.bi.accuracy","value":0.425,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k1000.bi.brier","value":0.9681,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k1000.bi.ece","value":0.4263,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k1000.bi.nll","value":10.9297,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k1000.bi.recall1","value":0.425,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k1000.bi.recall20","value":0.655,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k1000.bi.recall5","value":0.58,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k1000.bi.unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k1000.optionscut","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k1000.rejected","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k1000.sl.accall","value":0.03,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k1000.sl.accuracy","value":0.03,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k1000.sl.brier","value":1.4916,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k1000.sl.ece","value":0.6185,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k1000.sl.nll","value":26.605,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k1000.sl.recall1","value":0.03,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k1000.sl.recall20","value":0.04,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k1000.sl.recall5","value":0.04,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k1000.sl.unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k1000.statecut","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k1000.unanswered","value":200,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k1000.wide.accall","value":0.02,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k1000.wide.accuracy","value":0.02,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k1000.wide.brier","value":1.1571,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k1000.wide.ece","value":0.284,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k1000.wide.nll","value":8.4174,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k1000.wide.optionscut","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k1000.wide.recall1","value":0.02,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k1000.wide.recall20","value":0.135,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k1000.wide.recall5","value":0.055,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k1000.wide.rejected","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k1000.wide.statecut","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k1000.wide.unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k10000.accall","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k10000.bi.accall","value":0.205,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k10000.bi.accuracy","value":0.205,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k10000.bi.brier","value":1.3592,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k10000.bi.ece","value":0.6352,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k10000.bi.nll","value":17.5739,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k10000.bi.recall1","value":0.205,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k10000.bi.recall20","value":0.405,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k10000.bi.recall5","value":0.35,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k10000.bi.unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k10000.optionscut","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k10000.rejected","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k10000.sl.accall","value":0.01,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k10000.sl.accuracy","value":0.01,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k10000.sl.brier","value":1.4228,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k10000.sl.ece","value":0.5454,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k10000.sl.nll","value":27.2361,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k10000.sl.recall1","value":0.01,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k10000.sl.recall20","value":0.015,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k10000.sl.recall5","value":0.015,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k10000.sl.unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k10000.statecut","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k10000.unanswered","value":200,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k100000.accall","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k100000.bi.accall","value":0.04,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k100000.bi.accuracy","value":0.04,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k100000.bi.brier","value":1.6497,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k100000.bi.ece","value":0.7569,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k100000.bi.nll","value":22.0356,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k100000.bi.recall1","value":0.04,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k100000.bi.recall20","value":0.26,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k100000.bi.recall5","value":0.14,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k100000.bi.unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k100000.optionscut","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k100000.rejected","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k100000.sl.accall","value":0.02,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k100000.sl.accuracy","value":0.02,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k100000.sl.brier","value":1.4937,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k100000.sl.ece","value":0.6194,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k100000.sl.nll","value":27.0784,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k100000.sl.recall1","value":0.02,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k100000.sl.recall20","value":0.02,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k100000.sl.recall5","value":0.02,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k100000.sl.unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k100000.statecut","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k100000.unanswered","value":50,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k150.accall","value":0.61,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k150.accuracy","value":0.61,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k150.bi.accall","value":0.7775,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k150.bi.accuracy","value":0.7775,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k150.bi.brier","value":0.395,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k150.bi.ece","value":0.1881,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k150.bi.nll","value":4.5791,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k150.bi.recall1","value":0.7775,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k150.bi.recall20","value":0.8575,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k150.bi.recall5","value":0.8575,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k150.bi.unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k150.brier","value":0.6982,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k150.ece","value":0.3258,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k150.nll","value":4.1294,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k150.optionscut","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k150.recall1","value":0.61,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k150.recall20","value":0.975,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k150.recall5","value":0.92,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k150.rejected","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k150.sl.accall","value":0.1925,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k150.sl.accuracy","value":0.1925,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k150.sl.brier","value":1.3844,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k150.sl.ece","value":0.6286,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k150.sl.nll","value":22.1297,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k150.sl.recall1","value":0.1925,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k150.sl.recall20","value":0.2,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k150.sl.recall5","value":0.2,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k150.sl.unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k150.statecut","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k150.unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k150.wide.accall","value":0.6575,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k150.wide.accuracy","value":0.6575,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k150.wide.brier","value":0.62,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k150.wide.ece","value":0.2878,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k150.wide.nll","value":3.6941,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k150.wide.optionscut","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k150.wide.recall1","value":0.6575,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k150.wide.recall20","value":0.9825,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k150.wide.recall5","value":0.95,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k150.wide.rejected","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k150.wide.statecut","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k150.wide.unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k16.accall","value":0.67,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k16.accuracy","value":0.67,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k16.brier","value":0.5738,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k16.ece","value":0.25,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k16.nll","value":2.4747,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k16.optionscut","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k16.recall1","value":0.67,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k16.recall20","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k16.recall5","value":0.8925,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k16.rejected","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k16.statecut","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k16.unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k4.accall","value":0.7775,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k4.accuracy","value":0.7775,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k4.brier","value":0.3229,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k4.ece","value":0.0603,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k4.nll","value":0.6512,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k4.optionscut","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k4.recall1","value":0.7775,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k4.recall20","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k4.recall5","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k4.rejected","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k4.statecut","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k4.unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k77.accall","value":0.425,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k77.accuracy","value":0.425,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k77.bi.accall","value":0.57,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k77.bi.accuracy","value":0.57,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k77.bi.brier","value":0.7035,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k77.bi.ece","value":0.2981,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k77.bi.nll","value":3.5176,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k77.bi.recall1","value":0.57,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k77.bi.recall20","value":0.965,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k77.bi.recall5","value":0.8175,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k77.bi.unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k77.brier","value":0.9246,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k77.ece","value":0.3809,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k77.nll","value":4.7968,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k77.optionscut","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k77.recall1","value":0.425,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k77.recall20","value":0.8775,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k77.recall5","value":0.6975,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k77.rejected","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k77.sl.accall","value":0.075,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k77.sl.accuracy","value":0.075,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k77.sl.brier","value":1.3681,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k77.sl.ece","value":0.55,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k77.sl.nll","value":25.4353,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k77.sl.recall1","value":0.075,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k77.sl.recall20","value":0.08,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k77.sl.recall5","value":0.08,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k77.sl.unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k77.statecut","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k77.unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k77.wide.accall","value":0.5325,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k77.wide.accuracy","value":0.5325,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k77.wide.brier","value":0.7646,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k77.wide.ece","value":0.3024,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k77.wide.nll","value":3.1271,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k77.wide.optionscut","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k77.wide.recall1","value":0.5325,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k77.wide.recall20","value":0.9125,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k77.wide.recall5","value":0.8275,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k77.wide.rejected","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k77.wide.statecut","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"k77.wide.unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null}],"notes":"","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"cap.conformal-cases-jev-1.13-jev","benchmark":"cap.conformal","split":"cases","system":"jev","version":"typesafe/jev-1.13","baseline":true,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"","store_digest":"","facts_digest":"","commit":"735e240323cd7c9f0c880b7d8c62d89f40be2164","questions":null,"answered":null,"when":"","ended":"2026-09-27T14:44:27Z","by":"ra","completion":"unknown","measures":[{"category":"all","metric":"apps.accuracy","value":0.7739,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"apps.aurc","value":0.1185,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"apps.brier","value":0.3587,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"apps.conf90.cover","value":0.9075,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"apps.conf90.size","value":1.5938,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"apps.conf95.cover","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"apps.conf95.size","value":11.4111,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"apps.ece","value":0.1209,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"apps.n","value":3999,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"apps.nll","value":2.0313,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"apps.risk50","value":0.1047,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"apps.risk75","value":0.1471,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"apps.risk90","value":0.1916,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"apps.unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"aurc","value":0.061,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"choice.accuracy","value":0.885,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"choice.aurc","value":0.0304,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"choice.brier","value":0.1815,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"choice.conf90.cover","value":0.9054,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"choice.conf90.size","value":1.0577,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"choice.conf95.cover","value":0.9537,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"choice.conf95.size","value":1.3462,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"choice.ece","value":0.0446,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"choice.n","value":1600,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"choice.nll","value":0.8282,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"choice.risk50","value":0.0145,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"choice.risk75","value":0.0383,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"choice.risk90","value":0.0733,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"choice.unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"cov90","value":0.9007,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"cov95","value":0.9522,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"ece","value":0.0492,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"joint.accuracy","value":0.837,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"joint.aurc","value":0.0438,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"joint.brier","value":0.2224,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"joint.conf90.cover","value":0.9032,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"joint.conf90.size","value":1.1401,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"joint.conf95.cover","value":0.953,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"joint.conf95.size","value":1.3056,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"joint.ece","value":0.0375,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"joint.n","value":2000,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"joint.nll","value":0.3634,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"joint.risk50","value":0.0096,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"joint.risk75","value":0.0899,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"joint.risk90","value":0.1308,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"joint.unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"massive.accuracy","value":0.8902,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"massive.aurc","value":0.0162,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"massive.brier","value":0.1606,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"massive.conf90.cover","value":0.9027,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"massive.conf90.size","value":1.0288,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"massive.conf95.cover","value":0.9538,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"massive.conf95.size","value":1.4405,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"massive.ece","value":0.0184,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"massive.n","value":5099,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"massive.nll","value":0.7755,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"massive.risk50","value":0.0007,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"massive.risk75","value":0.023,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"massive.risk90","value":0.0547,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"massive.unanswered","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"orig.accuracy","value":0.8204,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"orig.aurc","value":0.061,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"orig.brier","value":0.2689,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"orig.conf90.cover","value":0.9007,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"orig.conf90.size","value":1.3043,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"orig.conf95.cover","value":0.9522,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"orig.conf95.size","value":1.9188,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"orig.ece","value":0.0492,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"orig.n","value":11098,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"orig.nll","value":1.2156,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"orig.risk50","value":0.0475,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"orig.risk75","value":0.0918,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"orig.risk90","value":0.1365,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"orig.unanswered","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"size90","value":1.3043,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"size95","value":1.9188,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"td.accuracy","value":0.7355,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"td.aurc","value":0.1277,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"td.brier","value":0.3658,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"td.conf90.cover","value":0.9005,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"td.conf90.size","value":1.5369,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"td.conf95.cover","value":0.9521,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"td.conf95.size","value":1.9118,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"td.ece","value":0.0472,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"td.n","value":2000,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"td.nll","value":0.7067,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"td.risk50","value":0.1142,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"td.risk75","value":0.1859,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"td.risk90","value":0.2314,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"td.unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null}],"notes":"","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"cap.conformal-cases-routed-laya","benchmark":"cap.conformal","split":"cases","system":"laya","version":"convaiinnovations/laya@55cf4c4ebb4ebe31b2550e8bdf3bd21b99753851+convaiinnovations/laya-multilingual@e4e9ddf21a7b1903b7acffd8814ad4307bf63a67","baseline":true,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"","store_digest":"","facts_digest":"","commit":"735e240323cd7c9f0c880b7d8c62d89f40be2164","questions":null,"answered":null,"when":"","ended":"2026-09-27T14:44:27Z","by":"ra","completion":"unknown","measures":[{"category":"all","metric":"apps.accuracy","value":0.6952,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"apps.aurc","value":0.1489,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"apps.brier","value":0.448,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"apps.conf90.cover","value":0.9004,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"apps.conf90.size","value":2.1446,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"apps.conf95.cover","value":0.9501,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"apps.conf95.size","value":4.391,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"apps.ece","value":0.1455,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"apps.n","value":3999,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"apps.nll","value":1.3018,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"apps.risk50","value":0.1406,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"apps.risk75","value":0.2071,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"apps.risk90","value":0.2659,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"apps.unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"aurc","value":0.2999,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"choice.accuracy","value":0.6206,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"choice.aurc","value":0.2383,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"choice.brier","value":0.6299,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"choice.conf90.cover","value":0.9057,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"choice.conf90.size","value":14.6707,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"choice.conf95.cover","value":0.9544,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"choice.conf95.size","value":26.0801,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"choice.ece","value":0.2255,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"choice.n","value":1600,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"choice.nll","value":3.013,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"choice.risk50","value":0.2712,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"choice.risk75","value":0.3042,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"choice.risk90","value":0.3451,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"choice.unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"cov90","value":0.9015,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"cov95","value":0.9505,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"ece","value":0.202,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"joint.accuracy","value":0.336,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"joint.aurc","value":0.6183,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"joint.brier","value":0.7183,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"joint.conf90.cover","value":0.8979,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"joint.conf90.size","value":2.9477,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"joint.conf95.cover","value":0.9457,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"joint.conf95.size","value":3.1041,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"joint.ece","value":0.0908,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"joint.n","value":2000,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"joint.nll","value":1.2587,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"joint.risk50","value":0.626,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"joint.risk75","value":0.644,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"joint.risk90","value":0.6628,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"joint.unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"massive.accuracy","value":0.3822,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"massive.aurc","value":0.4376,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"massive.brier","value":0.987,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"massive.conf90.cover","value":0.9015,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"massive.conf90.size","value":13.9462,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"massive.conf95.cover","value":0.9498,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"massive.conf95.size","value":16.6606,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"massive.ece","value":0.4096,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"massive.n","value":5100,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"massive.nll","value":3.8282,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"massive.risk50","value":0.4651,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"massive.risk75","value":0.5535,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"massive.risk90","value":0.5891,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"massive.unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"orig.accuracy","value":0.5641,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"orig.aurc","value":0.2999,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"orig.brier","value":0.687,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"orig.conf90.cover","value":0.9015,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"orig.conf90.size","value":6.8007,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"orig.conf95.cover","value":0.9505,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"orig.conf95.size","value":9.6469,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"orig.ece","value":0.202,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"orig.n","value":11099,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"orig.nll","value":2.3555,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"orig.risk50","value":0.3296,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"orig.risk75","value":0.3871,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"orig.risk90","value":0.4064,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"orig.unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"size90","value":6.8007,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"size95","value":9.6469,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"td.accuracy","value":0.766,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"td.aurc","value":0.0988,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"td.brier","value":0.3996,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"td.conf90.cover","value":0.9009,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"td.conf90.size","value":1.4342,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"td.conf95.cover","value":0.9514,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"td.conf95.size","value":1.6718,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"td.ece","value":0.2132,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"td.n","value":2000,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"td.nll","value":0.7069,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"td.risk50","value":0.097,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"td.risk75","value":0.1647,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"td.risk90","value":0.2044,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"td.unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null}],"notes":"","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"cap.deploy-cases-jev-1.13-jev","benchmark":"cap.deploy","split":"cases","system":"jev","version":"typesafe/jev-1.13","baseline":true,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"","store_digest":"","facts_digest":"","commit":"735e240323cd7c9f0c880b7d8c62d89f40be2164","questions":null,"answered":null,"when":"","ended":"2026-09-27T14:44:27Z","by":"ra","completion":"unknown","measures":[{"category":"all","metric":"fee","value":0.0001,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"leaves","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"offline","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"selfhosted","value":0,"lo":null,"hi":null,"n":null,"of":null}],"notes":"pinned: model name","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"cap.deploy-cases-routed-laya","benchmark":"cap.deploy","split":"cases","system":"laya","version":"convaiinnovations/laya@55cf4c4ebb4ebe31b2550e8bdf3bd21b99753851+convaiinnovations/laya-multilingual@e4e9ddf21a7b1903b7acffd8814ad4307bf63a67","baseline":true,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"","store_digest":"","facts_digest":"","commit":"735e240323cd7c9f0c880b7d8c62d89f40be2164","questions":null,"answered":null,"when":"","ended":"2026-09-27T14:44:27Z","by":"ra","completion":"unknown","measures":[{"category":"all","metric":"fee","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"leaves","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"offline","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"selfhosted","value":1,"lo":null,"hi":null,"n":null,"of":null}],"notes":"pinned: weights sha256","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"cap.invariance-cases-a6.a211cc70-kai","benchmark":"cap.invariance","split":"cases","system":"kai","version":"a6@a211cc70103825bfbc9bfb0fbe970f8bf08da2efe0953cd8045a3cd07397f4bf","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"","store_digest":"","facts_digest":"","commit":"a1a4a73d524cecf8038e30c408ad30b66e71d490","questions":null,"answered":null,"when":"","ended":"2026-09-28T03:23:23Z","by":"dgx","completion":"unknown","measures":[{"category":"all","metric":"alias.code.acc","value":0.955,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"alias.code.flip","value":0.02,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"alias.snake.acc","value":0.94,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"alias.snake.flip","value":0.025,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"alias.text.acc","value":0.95,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"perm.acc.max","value":0.95,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"perm.acc.min","value":0.95,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"perm.compared","value":200,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"perm.flip","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"perm.maxdp","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"perm.tv","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"swap.acc","value":0.8625,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"swap.desc.acc","value":0.8075,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"swap.desc.compared","value":400,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"swap.desc.gap","value":0.0928,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"swap.desc.maxgap","value":0.5067,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"swap.desc.mirror","value":0.915,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"swap.neg.acc","value":0.4575,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"swap.neg.compared","value":400,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"swap.neg.gap","value":0.4312,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"swap.neg.maxgap","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"swap.neg.mirror","value":0.43,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null}],"notes":"","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"cap.invariance-cases-jev-1.13-jev","benchmark":"cap.invariance","split":"cases","system":"jev","version":"typesafe/jev-1.13","baseline":true,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"","store_digest":"","facts_digest":"","commit":"a1a4a73d524cecf8038e30c408ad30b66e71d490","questions":null,"answered":null,"when":"","ended":"2026-09-27T09:52:25Z","by":"ra","completion":"unknown","measures":[{"category":"all","metric":"alias.code.acc","value":0.855,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"alias.code.flip","value":0.035,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"alias.snake.acc","value":0.845,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"alias.snake.flip","value":0.01,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"alias.text.acc","value":0.85,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"perm.acc.max","value":0.87,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"perm.acc.min","value":0.845,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"perm.compared","value":200,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"perm.flip","value":0.06,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"perm.maxdp","value":0.47,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"perm.tv","value":0.0332,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"swap.acc","value":0.82,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"swap.desc.acc","value":0.1925,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"swap.desc.compared","value":400,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"swap.desc.gap","value":0.5522,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"swap.desc.maxgap","value":0.98,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"swap.desc.mirror","value":0.0625,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"swap.neg.acc","value":0.8325,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"swap.neg.compared","value":400,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"swap.neg.gap","value":0.0928,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"swap.neg.maxgap","value":0.49,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"swap.neg.mirror","value":0.8825,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"usd","value":0.0666,"lo":null,"hi":null,"n":null,"of":null}],"notes":"","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"cap.invariance-cases-routed-laya","benchmark":"cap.invariance","split":"cases","system":"laya","version":"convaiinnovations/laya@55cf4c4ebb4ebe31b2550e8bdf3bd21b99753851+convaiinnovations/laya-multilingual@e4e9ddf21a7b1903b7acffd8814ad4307bf63a67","baseline":true,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"","store_digest":"","facts_digest":"","commit":"a1a4a73d524cecf8038e30c408ad30b66e71d490","questions":null,"answered":null,"when":"","ended":"2026-09-27T09:27:04Z","by":"ra","completion":"unknown","measures":[{"category":"all","metric":"alias.code.acc","value":0.65,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"alias.code.flip","value":0.205,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"alias.snake.acc","value":0.61,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"alias.snake.flip","value":0.21,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"alias.text.acc","value":0.625,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"perm.acc.max","value":0.65,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"perm.acc.min","value":0.62,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"perm.compared","value":200,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"perm.flip","value":0.255,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"perm.maxdp","value":0.996,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"perm.tv","value":0.1445,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"swap.acc","value":0.815,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"swap.desc.acc","value":0.31,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"swap.desc.compared","value":400,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"swap.desc.gap","value":0.5103,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"swap.desc.maxgap","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"swap.desc.mirror","value":0.205,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"swap.neg.acc","value":0.41,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"swap.neg.compared","value":400,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"swap.neg.gap","value":0.4644,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"swap.neg.maxgap","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"swap.neg.mirror","value":0.3,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null}],"notes":"","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"cap.joint-cases-a6.a211cc70-kai","benchmark":"cap.joint","split":"cases","system":"kai","version":"a6@a211cc70103825bfbc9bfb0fbe970f8bf08da2efe0953cd8045a3cd07397f4bf","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"","store_digest":"","facts_digest":"","commit":"a1a4a73d524cecf8038e30c408ad30b66e71d490","questions":null,"answered":null,"when":"","ended":"2026-09-28T03:23:08Z","by":"dgx","completion":"unknown","measures":[{"category":"all","metric":"acc","value":0.306,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"exact","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"maintenance.acc","value":0.323,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"maintenance.exact","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"maintenance.proj.acc","value":0.332,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"maintenance.proj.exact","value":0.015,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"maintenance.violate","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"pass1.acc","value":0.306,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"pass1.asked","value":2000,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"pass1.exact","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"pass1.frozen","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"pass1.maintenance.acc","value":0.323,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"pass1.maintenance.exact","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"pass1.maintenance.frozen","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"pass1.maintenance.proj.acc","value":0.314,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"pass1.maintenance.proj.exact","value":0.015,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"pass1.maintenance.violate","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"pass1.proj.acc","value":0.315,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"pass1.proj.exact","value":0.0075,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"pass1.trade.acc","value":0.289,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"pass1.trade.exact","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"pass1.trade.frozen","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"pass1.trade.proj.acc","value":0.316,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"pass1.trade.proj.exact","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"pass1.trade.violate","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"pass1.violate","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"pass2.acc","value":0.306,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"pass2.asked","value":7200,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"pass2.exact","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"pass2.frozen","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"pass2.maintenance.acc","value":0.323,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"pass2.maintenance.exact","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"pass2.maintenance.frozen","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"pass2.maintenance.proj.acc","value":0.332,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"pass2.maintenance.proj.exact","value":0.015,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"pass2.maintenance.violate","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"pass2.proj.acc","value":0.3405,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"pass2.proj.exact","value":0.01,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"pass2.trade.acc","value":0.289,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"pass2.trade.exact","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"pass2.trade.frozen","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"pass2.trade.proj.acc","value":0.349,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"pass2.trade.proj.exact","value":0.005,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"pass2.trade.violate","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"pass2.violate","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"pass3.acc","value":0.306,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"pass3.asked","value":7200,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"pass3.exact","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"pass3.frozen","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"pass3.maintenance.acc","value":0.323,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"pass3.maintenance.exact","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"pass3.maintenance.frozen","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"pass3.maintenance.proj.acc","value":0.332,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"pass3.maintenance.proj.exact","value":0.015,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"pass3.maintenance.violate","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"pass3.proj.acc","value":0.3405,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"pass3.proj.exact","value":0.01,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"pass3.trade.acc","value":0.289,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"pass3.trade.exact","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"pass3.trade.frozen","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"pass3.trade.proj.acc","value":0.349,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"pass3.trade.proj.exact","value":0.005,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"pass3.trade.violate","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"pass3.violate","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"proj.acc","value":0.3405,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"proj.exact","value":0.01,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"trade.acc","value":0.289,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"trade.exact","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"trade.proj.acc","value":0.349,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"trade.proj.exact","value":0.005,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"trade.violate","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"violate","value":1,"lo":null,"hi":null,"n":null,"of":null}],"notes":"","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"cap.joint-cases-jev-1.13-jev","benchmark":"cap.joint","split":"cases","system":"jev","version":"typesafe/jev-1.13","baseline":true,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"","store_digest":"","facts_digest":"","commit":"a1a4a73d524cecf8038e30c408ad30b66e71d490","questions":null,"answered":null,"when":"","ended":"2026-09-27T10:08:23Z","by":"ra","completion":"unknown","measures":[{"category":"all","metric":"acc","value":0.837,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"acc.action","value":0.78,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"acc.evidence","value":0.915,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"acc.fault","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"acc.option","value":0.965,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"acc.readiness","value":0.825,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"acc.review","value":0.6,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"acc.risk","value":0.93,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"acc.severity","value":0.93,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"exact","value":0.4375,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"maintenance.acc","value":0.826,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"maintenance.exact","value":0.435,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"maintenance.proj.acc","value":0.889,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"maintenance.proj.exact","value":0.545,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"maintenance.violate","value":0.29,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"proj.acc","value":0.8795,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"proj.acc.action","value":0.96,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"proj.acc.evidence","value":0.935,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"proj.acc.fault","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"proj.acc.option","value":0.975,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"proj.acc.readiness","value":0.92,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"proj.acc.review","value":0.6125,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"proj.acc.risk","value":0.9,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"proj.acc.severity","value":0.96,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"proj.exact","value":0.56,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"proj.violate","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"trade.acc","value":0.848,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"trade.exact","value":0.44,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"trade.proj.acc","value":0.87,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"trade.proj.exact","value":0.575,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"trade.violate","value":0.205,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"violate","value":0.2475,"lo":null,"hi":null,"n":null,"of":null}],"notes":"","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"cap.joint-cases-routed-laya","benchmark":"cap.joint","split":"cases","system":"laya","version":"convaiinnovations/laya@55cf4c4ebb4ebe31b2550e8bdf3bd21b99753851+convaiinnovations/laya-multilingual@e4e9ddf21a7b1903b7acffd8814ad4307bf63a67","baseline":true,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"","store_digest":"","facts_digest":"","commit":"a1a4a73d524cecf8038e30c408ad30b66e71d490","questions":null,"answered":null,"when":"","ended":"2026-09-27T10:08:23Z","by":"ra","completion":"unknown","measures":[{"category":"all","metric":"acc","value":0.336,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"acc.action","value":0.28,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"acc.evidence","value":0.3,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"acc.fault","value":0.325,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"acc.option","value":0.315,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"acc.readiness","value":0.3275,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"acc.review","value":0.4825,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"acc.risk","value":0.26,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"acc.severity","value":0.26,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"en.acc","value":0.32,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"en.acc.action","value":0.1,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"en.acc.evidence","value":0.29,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"en.acc.fault","value":0.3,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"en.acc.option","value":0.275,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"en.acc.readiness","value":0.3525,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"en.acc.review","value":0.4825,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"en.acc.risk","value":0.28,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"en.acc.severity","value":0.285,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"en.exact","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"en.maintenance.acc","value":0.305,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"en.maintenance.exact","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"en.maintenance.proj.acc","value":0.354,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"en.maintenance.proj.exact","value":0.005,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"en.maintenance.violate","value":0.91,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"en.proj.acc","value":0.3445,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"en.proj.acc.action","value":0.32,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"en.proj.acc.evidence","value":0.29,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"en.proj.acc.fault","value":0.275,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"en.proj.acc.option","value":0.275,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"en.proj.acc.readiness","value":0.36,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"en.proj.acc.review","value":0.4825,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"en.proj.acc.risk","value":0.28,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"en.proj.acc.severity","value":0.32,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"en.proj.exact","value":0.0025,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"en.proj.violate","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"en.trade.acc","value":0.335,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"en.trade.exact","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"en.trade.proj.acc","value":0.335,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"en.trade.proj.exact","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"en.trade.violate","value":0.03,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"en.unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"en.violate","value":0.47,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"exact","value":0.005,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"maintenance.acc","value":0.329,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"maintenance.exact","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"maintenance.proj.acc","value":0.302,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"maintenance.proj.exact","value":0.07,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"maintenance.violate","value":0.96,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"proj.acc","value":0.3275,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"proj.acc.action","value":0.22,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"proj.acc.evidence","value":0.3,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"proj.acc.fault","value":0.3,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"proj.acc.option","value":0.315,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"proj.acc.readiness","value":0.335,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"proj.acc.review","value":0.4825,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"proj.acc.risk","value":0.285,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"proj.acc.severity","value":0.22,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"proj.exact","value":0.04,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"proj.violate","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"trade.acc","value":0.343,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"trade.exact","value":0.01,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"trade.proj.acc","value":0.353,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"trade.proj.exact","value":0.01,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"trade.violate","value":0.315,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"violate","value":0.6375,"lo":null,"hi":null,"n":null,"of":null}],"notes":"","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"cap.media-cases-jev-1.13-jev","benchmark":"cap.media","split":"cases","system":"jev","version":"typesafe/jev-1.13","baseline":true,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"","store_digest":"","facts_digest":"","commit":"735e240323cd7c9f0c880b7d8c62d89f40be2164","questions":null,"answered":null,"when":"","ended":"2026-09-27T09:42:32Z","by":"ra","completion":"unknown","measures":[{"category":"all","metric":"audio.accuracy","value":0.84,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"cmapss.accuracy","value":0.3,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"cmapss.brier","value":0.7616,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"cmapss.ece","value":0.1039,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"cmapss.errors","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"cmapss.macrof1","value":0.2482,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"cmapss.unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"cmapss.usd","value":0.0081,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"cmapss.zeroprob","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"har.accuracy","value":0.1533,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"har.brier","value":1.0982,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"har.ece","value":0.4456,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"har.errors","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"har.macrof1","value":0.0509,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"har.unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"har.usd","value":0.0252,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"har.zeroprob","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"mimii.accuracy","value":0.84,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"mimii.auc","value":0.8756,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"mimii.brier","value":0.3455,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"mimii.ece","value":0.1971,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"mimii.errors","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"mimii.macrof1","value":0.8394,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"mimii.unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"mimii.usd","value":0.0057,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"mimii.zeroprob","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"sensor.accuracy","value":0.19,"lo":null,"hi":null,"n":null,"of":null}],"notes":"image.input: text only; video.input: text only","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"cap.media-cases-routed-laya","benchmark":"cap.media","split":"cases","system":"laya","version":"convaiinnovations/laya@55cf4c4ebb4ebe31b2550e8bdf3bd21b99753851+convaiinnovations/laya-multilingual@e4e9ddf21a7b1903b7acffd8814ad4307bf63a67","baseline":true,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"","store_digest":"","facts_digest":"","commit":"735e240323cd7c9f0c880b7d8c62d89f40be2164","questions":null,"answered":null,"when":"","ended":"2026-09-27T09:31:39Z","by":"ra","completion":"unknown","measures":[{"category":"all","metric":"audio.accuracy","value":0.5,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"cmapss.accuracy","value":0.17,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"cmapss.brier","value":0.7851,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"cmapss.ece","value":0.1298,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"cmapss.macrof1","value":0.1144,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"cmapss.rejected","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"cmapss.statecut","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"cmapss.unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"cmapss.zeroprob","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"har.accuracy","value":0.2033,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"har.brier","value":0.8797,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"har.ece","value":0.1434,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"har.macrof1","value":0.0563,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"har.rejected","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"har.statecut","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"har.unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"har.zeroprob","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"long.cmapss.accuracy","value":0.14,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"long.cmapss.brier","value":0.9598,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"long.cmapss.ece","value":0.4087,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"long.cmapss.macrof1","value":0.0614,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"long.cmapss.rejected","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"long.cmapss.statecut","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"long.cmapss.unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"long.cmapss.zeroprob","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"long.har.accuracy","value":0.1633,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"long.har.brier","value":1.0574,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"long.har.ece","value":0.4227,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"long.har.macrof1","value":0.0517,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"long.har.rejected","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"long.har.statecut","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"long.har.unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"long.har.zeroprob","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"long.mimii.accuracy","value":0.5,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"long.mimii.auc","value":0.4856,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"long.mimii.brier","value":0.8478,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"long.mimii.ece","value":0.4171,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"long.mimii.macrof1","value":0.3333,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"long.mimii.rejected","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"long.mimii.statecut","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"long.mimii.unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"long.mimii.zeroprob","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"mimii.accuracy","value":0.5,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"mimii.auc","value":0.5368,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"mimii.brier","value":0.5124,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"mimii.ece","value":0.1154,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"mimii.macrof1","value":0.3333,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"mimii.rejected","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"mimii.statecut","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"mimii.unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"mimii.zeroprob","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"sensor.accuracy","value":0.195,"lo":null,"hi":null,"n":null,"of":null}],"notes":"image.input: text only; video.input: text only","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"cap.orig-cases-a6.a211cc70-kai","benchmark":"cap.orig","split":"cases","system":"kai","version":"a6@a211cc70103825bfbc9bfb0fbe970f8bf08da2efe0953cd8045a3cd07397f4bf","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"","store_digest":"","facts_digest":"","commit":"a1a4a73d524cecf8038e30c408ad30b66e71d490","questions":null,"answered":null,"when":"","ended":"2026-09-28T03:19:56Z","by":"dgx","completion":"unknown","measures":[{"category":"all","metric":"agnews.accuracy","value":0.9425,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"agnews.aurc","value":0.0095,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"agnews.brier","value":0.0924,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"agnews.ece","value":0.0266,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"agnews.macrof1","value":0.9362,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"agnews.nll","value":0.1833,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"agnews.unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"agnews.zeroprob","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"banking77full.accuracy","value":0.9225,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"banking77full.aurc","value":0.0213,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"banking77full.brier","value":0.1428,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"banking77full.ece","value":0.0669,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"banking77full.macrof1","value":0.3952,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"banking77full.nll","value":0.6511,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"banking77full.unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"banking77full.zeroprob","value":0.0125,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"beats.jev","value":40,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"beats.laya","value":58,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"emailspam.accuracy","value":0.9975,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"emailspam.aurc","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"emailspam.brier","value":0.0039,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"emailspam.ece","value":0.0022,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"emailspam.macrof1","value":0.9975,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"emailspam.nll","value":0.0059,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"emailspam.unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"emailspam.zeroprob","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"emotion.accuracy","value":0.94,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"emotion.aurc","value":0.0047,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"emotion.brier","value":0.0803,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"emotion.ece","value":0.0225,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"emotion.macrof1","value":0.8798,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"emotion.nll","value":0.1508,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"emotion.unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"emotion.zeroprob","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"guardrailsjailbreak.accuracy","value":0.92,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"guardrailsjailbreak.aurc","value":0.0148,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"guardrailsjailbreak.brier","value":0.1419,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"guardrailsjailbreak.ece","value":0.0665,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"guardrailsjailbreak.macrof1","value":0.8805,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"guardrailsjailbreak.nll","value":0.2892,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"guardrailsjailbreak.unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"guardrailsjailbreak.zeroprob","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"massive.above3xrandom","value":51,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"massive.languages","value":51,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"massive.macroaccuracy","value":0.9104,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"massive.macroece","value":0.0578,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"massive.nonenglishmacro","value":0.9098,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"modelroutingdomain.accuracy","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"modelroutingdomain.aurc","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"modelroutingdomain.brier","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"modelroutingdomain.ece","value":0.0004,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"modelroutingdomain.macrof1","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"modelroutingdomain.nll","value":0.0004,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"modelroutingdomain.unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"modelroutingdomain.zeroprob","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"moderationtoxicity.accuracy","value":0.8075,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"moderationtoxicity.aurc","value":0.1136,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"moderationtoxicity.brier","value":0.37,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"moderationtoxicity.ece","value":0.1854,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"moderationtoxicity.macrof1","value":0.8043,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"moderationtoxicity.nll","value":1.3171,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"moderationtoxicity.unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"moderationtoxicity.zeroprob","value":0.0025,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"phishing.accuracy","value":0.9925,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"phishing.aurc","value":0.0002,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"phishing.brier","value":0.0149,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"phishing.ece","value":0.0073,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"phishing.macrof1","value":0.9921,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"phishing.nll","value":0.0435,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"phishing.unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"phishing.zeroprob","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"ragrelevance.accuracy","value":0.65,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"ragrelevance.aurc","value":0.2171,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"ragrelevance.brier","value":0.4126,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"ragrelevance.ece","value":0.0551,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"ragrelevance.macrof1","value":0.6489,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"ragrelevance.nll","value":0.5992,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"ragrelevance.unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"ragrelevance.zeroprob","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"supporttriage.accuracy","value":0.4275,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"supporttriage.aurc","value":0.3918,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"supporttriage.brier","value":0.9623,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"supporttriage.ece","value":0.4414,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"supporttriage.macrof1","value":0.3975,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"supporttriage.nll","value":3.5796,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"supporttriage.unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"supporttriage.zeroprob","value":0.03,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"typeddecisions.accuracy","value":0.751,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"typeddecisions.aurc","value":0.1298,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"typeddecisions.brier","value":0.4268,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"typeddecisions.ece","value":0.2093,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"typeddecisions.macrof1","value":0.7337,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"typeddecisions.nll","value":0.7641,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"typeddecisions.unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"typeddecisions.zeroprob","value":0,"lo":null,"hi":null,"n":null,"of":null}],"notes":"","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"cap.orig-cases-jev-1.13-jev","benchmark":"cap.orig","split":"cases","system":"jev","version":"typesafe/jev-1.13","baseline":true,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"","store_digest":"","facts_digest":"","commit":"a1a4a73d524cecf8038e30c408ad30b66e71d490","questions":null,"answered":null,"when":"","ended":"2026-09-28T03:19:56Z","by":"dgx","completion":"unknown","measures":[{"category":"all","metric":"agnews.accuracy","value":0.86,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"agnews.aurc","value":0.0832,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"agnews.brier","value":0.2323,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"agnews.ece","value":0.1023,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"agnews.macrof1","value":0.8485,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"agnews.nll","value":1.805,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"agnews.unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"agnews.zeroprob","value":0.0575,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"banking77full.accuracy","value":0.835,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"banking77full.aurc","value":0.0579,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"banking77full.brier","value":0.257,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"banking77full.ece","value":0.0729,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"banking77full.macrof1","value":0.3137,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"banking77full.nll","value":1.3471,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"banking77full.unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"banking77full.zeroprob","value":0.035,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"beats.kai","value":16,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"beats.laya","value":56,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"emailspam.accuracy","value":0.9775,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"emailspam.aurc","value":0.0007,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"emailspam.brier","value":0.0379,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"emailspam.ece","value":0.0604,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"emailspam.macrof1","value":0.9774,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"emailspam.nll","value":0.0967,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"emailspam.unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"emailspam.zeroprob","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"emotion.accuracy","value":0.6025,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"emotion.aurc","value":0.2311,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"emotion.brier","value":0.6464,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"emotion.ece","value":0.2795,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"emotion.macrof1","value":0.5155,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"emotion.nll","value":4.6832,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"emotion.unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"emotion.zeroprob","value":0.1425,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"guardrailsjailbreak.accuracy","value":0.94,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"guardrailsjailbreak.aurc","value":0.012,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"guardrailsjailbreak.brier","value":0.1044,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"guardrailsjailbreak.ece","value":0.0469,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"guardrailsjailbreak.macrof1","value":0.9183,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"guardrailsjailbreak.nll","value":0.1891,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"guardrailsjailbreak.unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"guardrailsjailbreak.zeroprob","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"massive.above3xrandom","value":51,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"massive.languages","value":51,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"massive.macroaccuracy","value":0.8902,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"massive.macroece","value":0.0638,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"massive.nonenglishmacro","value":0.8896,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"modelroutingdomain.accuracy","value":0.9774,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"modelroutingdomain.aurc","value":0.0028,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"modelroutingdomain.brier","value":0.04,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"modelroutingdomain.ece","value":0.0138,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"modelroutingdomain.macrof1","value":0.5915,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"modelroutingdomain.nll","value":0.0847,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"modelroutingdomain.unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"modelroutingdomain.zeroprob","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"moderationtoxicity.accuracy","value":0.6625,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"moderationtoxicity.aurc","value":0.1382,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"moderationtoxicity.brier","value":0.4316,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"moderationtoxicity.ece","value":0.1768,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"moderationtoxicity.macrof1","value":0.6285,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"moderationtoxicity.nll","value":0.6192,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"moderationtoxicity.unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"moderationtoxicity.zeroprob","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"phishing.accuracy","value":0.9,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"phishing.aurc","value":0.0208,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"phishing.brier","value":0.1523,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"phishing.ece","value":0.0388,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"phishing.macrof1","value":0.8874,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"phishing.nll","value":0.2495,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"phishing.unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"phishing.zeroprob","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"ragrelevance.accuracy","value":0.62,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"ragrelevance.aurc","value":0.3309,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"ragrelevance.brier","value":0.6282,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"ragrelevance.ece","value":0.2788,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"ragrelevance.macrof1","value":0.5924,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"ragrelevance.nll","value":1.1129,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"ragrelevance.unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"ragrelevance.zeroprob","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"supporttriage.accuracy","value":0.365,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"supporttriage.aurc","value":0.4868,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"supporttriage.brier","value":1.0557,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"supporttriage.ece","value":0.4827,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"supporttriage.macrof1","value":0.3682,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"supporttriage.nll","value":10.1202,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"supporttriage.unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"supporttriage.zeroprob","value":0.335,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"typeddecisions.accuracy","value":0.7355,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"typeddecisions.aurc","value":0.1277,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"typeddecisions.brier","value":0.3658,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"typeddecisions.ece","value":0.0472,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"typeddecisions.macrof1","value":0.7381,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"typeddecisions.nll","value":0.7067,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"typeddecisions.unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"typeddecisions.zeroprob","value":0.003,"lo":null,"hi":null,"n":null,"of":null}],"notes":"","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"cap.orig-cases-routed-laya","benchmark":"cap.orig","split":"cases","system":"laya","version":"convaiinnovations/laya@55cf4c4ebb4ebe31b2550e8bdf3bd21b99753851+convaiinnovations/laya-multilingual@e4e9ddf21a7b1903b7acffd8814ad4307bf63a67","baseline":true,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"","store_digest":"","facts_digest":"","commit":"a1a4a73d524cecf8038e30c408ad30b66e71d490","questions":null,"answered":null,"when":"","ended":"2026-09-28T03:19:56Z","by":"dgx","completion":"unknown","measures":[{"category":"all","metric":"agnews.accuracy","value":0.95,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"agnews.aurc","value":0.0067,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"agnews.brier","value":0.0808,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"agnews.ece","value":0.0315,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"agnews.macrof1","value":0.9439,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"agnews.nll","value":0.1636,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"agnews.unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"agnews.zeroprob","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"banking77full.accuracy","value":0.425,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"banking77full.aurc","value":0.3559,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"banking77full.brier","value":0.9246,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"banking77full.ece","value":0.3809,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"banking77full.macrof1","value":0.1121,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"banking77full.nll","value":4.7967,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"banking77full.unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"banking77full.zeroprob","value":0.0975,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"beats.jev","value":6,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"beats.kai","value":3,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"emailspam.accuracy","value":0.9975,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"emailspam.aurc","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"emailspam.brier","value":0.0052,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"emailspam.ece","value":0.0108,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"emailspam.macrof1","value":0.9975,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"emailspam.nll","value":0.0126,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"emailspam.unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"emailspam.zeroprob","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"emotion.accuracy","value":0.595,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"emotion.aurc","value":0.2655,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"emotion.brier","value":0.6963,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"emotion.ece","value":0.3063,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"emotion.macrof1","value":0.4752,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"emotion.nll","value":2.0194,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"emotion.unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"emotion.zeroprob","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"guardrailsjailbreak.accuracy","value":0.705,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"guardrailsjailbreak.aurc","value":0.2304,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"guardrailsjailbreak.brier","value":0.5427,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"guardrailsjailbreak.ece","value":0.2617,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"guardrailsjailbreak.macrof1","value":0.6818,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"guardrailsjailbreak.nll","value":1.9121,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"guardrailsjailbreak.unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"guardrailsjailbreak.zeroprob","value":0.0025,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"massive.above3xrandom","value":48,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"massive.languages","value":51,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"massive.macroaccuracy","value":0.3822,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"massive.macroece","value":0.4184,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"massive.nonenglishmacro","value":0.3734,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"modelroutingdomain.accuracy","value":0.6391,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"modelroutingdomain.aurc","value":0.1758,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"modelroutingdomain.brier","value":0.4938,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"modelroutingdomain.ece","value":0.0889,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"modelroutingdomain.macrof1","value":0.3199,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"modelroutingdomain.nll","value":1.089,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"modelroutingdomain.unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"modelroutingdomain.zeroprob","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"moderationtoxicity.accuracy","value":0.53,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"moderationtoxicity.aurc","value":0.2811,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"moderationtoxicity.brier","value":0.6015,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"moderationtoxicity.ece","value":0.2967,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"moderationtoxicity.macrof1","value":0.4003,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"moderationtoxicity.nll","value":0.8856,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"moderationtoxicity.unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"moderationtoxicity.zeroprob","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"phishing.accuracy","value":0.9825,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"phishing.aurc","value":0.0005,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"phishing.brier","value":0.026,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"phishing.ece","value":0.0101,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"phishing.macrof1","value":0.9815,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"phishing.nll","value":0.0439,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"phishing.unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"phishing.zeroprob","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"ragrelevance.accuracy","value":0.625,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"ragrelevance.aurc","value":0.2795,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"ragrelevance.brier","value":0.4656,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"ragrelevance.ece","value":0.1175,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"ragrelevance.macrof1","value":0.6234,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"ragrelevance.nll","value":0.671,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"ragrelevance.unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"ragrelevance.zeroprob","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"supporttriage.accuracy","value":0.5025,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"supporttriage.aurc","value":0.3208,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"supporttriage.brier","value":0.6438,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"supporttriage.ece","value":0.0907,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"supporttriage.macrof1","value":0.4802,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"supporttriage.nll","value":1.4237,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"supporttriage.unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"supporttriage.zeroprob","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"typeddecisions.accuracy","value":0.766,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"typeddecisions.aurc","value":0.0988,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"typeddecisions.brier","value":0.3996,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"typeddecisions.ece","value":0.2132,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"typeddecisions.macrof1","value":0.7503,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"typeddecisions.nll","value":0.7069,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"typeddecisions.unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"typeddecisions.zeroprob","value":0,"lo":null,"hi":null,"n":null,"of":null}],"notes":"","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"cap.refresh-cases-a6.a211cc70-kai","benchmark":"cap.refresh","split":"cases","system":"kai","version":"a6@a211cc70103825bfbc9bfb0fbe970f8bf08da2efe0953cd8045a3cd07397f4bf","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"","store_digest":"","facts_digest":"","commit":"a1a4a73d524cecf8038e30c408ad30b66e71d490","questions":null,"answered":null,"when":"","ended":"2026-09-28T03:24:02Z","by":"dgx","completion":"unknown","measures":[{"category":"all","metric":"asked","value":6,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"full","value":101,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"same","value":10,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"support","value":1,"lo":null,"hi":null,"n":null,"of":null}],"notes":"","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"cap.refresh-cases-jev-1.13-jev","benchmark":"cap.refresh","split":"cases","system":"jev","version":"typesafe/jev-1.13","baseline":true,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"","store_digest":"","facts_digest":"","commit":"a1a4a73d524cecf8038e30c408ad30b66e71d490","questions":null,"answered":null,"when":"","ended":"2026-09-27T09:53:54Z","by":"ra","completion":"unknown","measures":[{"category":"all","metric":"asked","value":100,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"full","value":100,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"reask.usd","value":0.0009,"lo":null,"hi":null,"n":null,"of":null}],"notes":"support: none","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"cap.refresh-cases-routed-laya","benchmark":"cap.refresh","split":"cases","system":"laya","version":"convaiinnovations/laya@55cf4c4ebb4ebe31b2550e8bdf3bd21b99753851+convaiinnovations/laya-multilingual@e4e9ddf21a7b1903b7acffd8814ad4307bf63a67","baseline":true,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"","store_digest":"","facts_digest":"","commit":"a1a4a73d524cecf8038e30c408ad30b66e71d490","questions":null,"answered":null,"when":"","ended":"2026-09-27T09:29:39Z","by":"ra","completion":"unknown","measures":[{"category":"all","metric":"asked","value":100,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"full","value":100,"lo":null,"hi":null,"n":null,"of":null}],"notes":"support: none","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"gate.jev-drawn-a5.df1c16a2-kai","benchmark":"gate.jev","split":"drawn","system":"kai","version":"a5@df1c16a2fef7ec93f2b34a6a194f6dc3c2eee70e936473b802151543c5d46e83","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"11474208ce49e6834bad21db4af8bb7ddf33b0ada6138b3661938522cafc815f","store_digest":"","facts_digest":"","commit":"611059b60f64ab686d6475cd76778b3fac52cd98","questions":null,"answered":null,"when":"","ended":"2026-09-27T21:21:10Z","by":"","completion":"unknown","measures":[{"category":"all","metric":"pass","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"suites","value":62,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"candidate","value":0.8786321277289022,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"baseline","value":0.8669257086999022,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"delta","value":0.011706419029000048,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"regressions","value":16,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"cases","value":100,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/jev.ag_news","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/jev.ag_news","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/jev.ag_news","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/jev.ag_news","metric":"candidate_accuracy","value":0.97,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/jev.ag_news","metric":"candidate_ece","value":0.09678747147321702,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/jev.ag_news","metric":"baseline_accuracy","value":0.83,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/jev.ag_news","metric":"baseline_ece","value":0.12050000071525568,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/jev.emotion","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/jev.emotion","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/jev.emotion","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/jev.emotion","metric":"candidate_accuracy","value":0.9,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/jev.emotion","metric":"candidate_ece","value":0.07829443186521529,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/jev.emotion","metric":"baseline_accuracy","value":0.64,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/jev.emotion","metric":"baseline_ece","value":0.24150000125169754,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/jev.banking77_full","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/jev.banking77_full","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/jev.banking77_full","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/jev.banking77_full","metric":"candidate_accuracy","value":0.89,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/jev.banking77_full","metric":"candidate_ece","value":0.1010316482186318,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/jev.banking77_full","metric":"baseline_accuracy","value":0.85,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/jev.banking77_full","metric":"baseline_ece","value":0.08564848810434346,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.support_triage","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.support_triage","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.support_triage","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.support_triage","metric":"candidate_accuracy","value":0.39,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.support_triage","metric":"candidate_ece","value":0.21937582075595857,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.support_triage","metric":"baseline_accuracy","value":0.29,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.support_triage","metric":"baseline_ece","value":0.561000002026558,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.email_spam","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.email_spam","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.email_spam","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.email_spam","metric":"candidate_accuracy","value":1,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.email_spam","metric":"candidate_ece","value":0.00008834540843960514,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.email_spam","metric":"baseline_accuracy","value":0.96,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.email_spam","metric":"baseline_ece","value":0.07149999797344209,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.phishing","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.phishing","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.phishing","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.phishing","metric":"candidate_accuracy","value":1,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.phishing","metric":"candidate_ece","value":0.0000836509466171087,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.phishing","metric":"baseline_accuracy","value":0.89,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.phishing","metric":"baseline_ece","value":0.08049999296665189,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.guardrails_jailbreak","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.guardrails_jailbreak","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.guardrails_jailbreak","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.guardrails_jailbreak","metric":"candidate_accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.guardrails_jailbreak","metric":"candidate_ece","value":0.055477619767189054,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.guardrails_jailbreak","metric":"baseline_accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.guardrails_jailbreak","metric":"baseline_ece","value":0.0655999952554703,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.moderation_toxicity","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.moderation_toxicity","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.moderation_toxicity","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.moderation_toxicity","metric":"candidate_accuracy","value":0.83,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.moderation_toxicity","metric":"candidate_ece","value":0.1615606594085694,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.moderation_toxicity","metric":"baseline_accuracy","value":0.73,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.moderation_toxicity","metric":"baseline_ece","value":0.1372000062465668,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.rag_relevance","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.rag_relevance","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.rag_relevance","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.rag_relevance","metric":"candidate_accuracy","value":0.58,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.rag_relevance","metric":"candidate_ece","value":0.12734931707382205,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.rag_relevance","metric":"baseline_accuracy","value":0.51,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.rag_relevance","metric":"baseline_ece","value":0.39620000362396246,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.model_routing_domain","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.model_routing_domain","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.model_routing_domain","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.model_routing_domain","metric":"candidate_accuracy","value":1,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.model_routing_domain","metric":"candidate_ece","value":0.0018489766120910467,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.model_routing_domain","metric":"baseline_accuracy","value":0.98,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.model_routing_domain","metric":"baseline_ece","value":0.026600003242492693,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/typed_decisions","metric":"regressed","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/typed_decisions","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/typed_decisions","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/typed_decisions","metric":"candidate_accuracy","value":0.696,"lo":null,"hi":null,"n":500,"of":null},{"category":"harness/typed_decisions","metric":"candidate_ece","value":0.1625631979107857,"lo":null,"hi":null,"n":500,"of":null},{"category":"harness/typed_decisions","metric":"baseline_accuracy","value":0.73,"lo":null,"hi":null,"n":500,"of":null},{"category":"harness/typed_decisions","metric":"baseline_ece","value":0.07618141478300096,"lo":null,"hi":null,"n":500,"of":null},{"category":"harness/massive.af","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.af","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.af","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.af","metric":"candidate_accuracy","value":0.89,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.af","metric":"candidate_ece","value":0.05952447265386586,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.af","metric":"baseline_accuracy","value":0.87,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.af","metric":"baseline_ece","value":0.08726868644356728,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.am","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.am","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.am","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.am","metric":"candidate_accuracy","value":0.84,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.am","metric":"candidate_ece","value":0.06833967849612237,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.am","metric":"baseline_accuracy","value":0.85,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.am","metric":"baseline_ece","value":0.09598384097218512,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ar","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ar","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ar","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ar","metric":"candidate_accuracy","value":0.88,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ar","metric":"candidate_ece","value":0.08501704618334774,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ar","metric":"baseline_accuracy","value":0.87,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ar","metric":"baseline_ece","value":0.08222424164414406,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.az","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.az","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.az","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.az","metric":"candidate_accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.az","metric":"candidate_ece","value":0.06093618080019949,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.az","metric":"baseline_accuracy","value":0.88,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.az","metric":"baseline_ece","value":0.04820606261491777,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.bn","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.bn","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.bn","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.bn","metric":"candidate_accuracy","value":0.87,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.bn","metric":"candidate_ece","value":0.0521162086725235,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.bn","metric":"baseline_accuracy","value":0.86,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.bn","metric":"baseline_ece","value":0.0710626265406609,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.cy","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.cy","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.cy","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.cy","metric":"candidate_accuracy","value":0.82,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.cy","metric":"candidate_ece","value":0.06791832819581037,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.cy","metric":"baseline_accuracy","value":0.76,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.cy","metric":"baseline_ece","value":0.09916767939925195,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.da","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.da","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.da","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.da","metric":"candidate_accuracy","value":0.91,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.da","metric":"candidate_ece","value":0.04912485331296921,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.da","metric":"baseline_accuracy","value":0.88,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.da","metric":"baseline_ece","value":0.038734343349933635,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.de","metric":"regressed","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.de","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.de","metric":"unanswered","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.de","metric":"candidate_accuracy","value":0.9191919191919192,"lo":null,"hi":null,"n":99,"of":null},{"category":"harness/massive.de","metric":"candidate_ece","value":0.06942960951063372,"lo":null,"hi":null,"n":99,"of":null},{"category":"harness/massive.de","metric":"baseline_accuracy","value":0.9393939393939394,"lo":null,"hi":null,"n":99,"of":null},{"category":"harness/massive.de","metric":"baseline_ece","value":0.057024792589322484,"lo":null,"hi":null,"n":99,"of":null},{"category":"harness/massive.el","metric":"regressed","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.el","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.el","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.el","metric":"candidate_accuracy","value":0.9,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.el","metric":"candidate_ece","value":0.05367249518632891,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.el","metric":"baseline_accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.el","metric":"baseline_ece","value":0.057093940079212174,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.en","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.en","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.en","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.en","metric":"candidate_accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.en","metric":"candidate_ece","value":0.05163752526044845,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.en","metric":"baseline_accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.en","metric":"baseline_ece","value":0.02530000150203709,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.es","metric":"regressed","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.es","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.es","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.es","metric":"candidate_accuracy","value":0.9,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.es","metric":"candidate_ece","value":0.04325293913483622,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.es","metric":"baseline_accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.es","metric":"baseline_ece","value":0.06282828330993653,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.fa","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.fa","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.fa","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.fa","metric":"candidate_accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.fa","metric":"candidate_ece","value":0.05692772075533863,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.fa","metric":"baseline_accuracy","value":0.95,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.fa","metric":"baseline_ece","value":0.04546464592218401,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.fi","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.fi","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.fi","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.fi","metric":"candidate_accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.fi","metric":"candidate_ece","value":0.060005676448345244,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.fi","metric":"baseline_accuracy","value":0.89,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.fi","metric":"baseline_ece","value":0.0383111131191253,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.fr","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.fr","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.fr","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.fr","metric":"candidate_accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.fr","metric":"candidate_ece","value":0.05017257452011109,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.fr","metric":"baseline_accuracy","value":0.9,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.fr","metric":"baseline_ece","value":0.0651949521899224,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.he","metric":"regressed","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.he","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.he","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.he","metric":"candidate_accuracy","value":0.87,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.he","metric":"candidate_ece","value":0.07292630210518837,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.he","metric":"baseline_accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.he","metric":"baseline_ece","value":0.042724243700504254,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.hi","metric":"regressed","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.hi","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.hi","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.hi","metric":"candidate_accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.hi","metric":"candidate_ece","value":0.07353801846504208,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.hi","metric":"baseline_accuracy","value":0.95,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.hi","metric":"baseline_ece","value":0.029023234546184544,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.hu","metric":"regressed","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.hu","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.hu","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.hu","metric":"candidate_accuracy","value":0.87,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.hu","metric":"candidate_ece","value":0.050389538556337356,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.hu","metric":"baseline_accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.hu","metric":"baseline_ece","value":0.03867272794246674,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.hy","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.hy","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.hy","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.hy","metric":"candidate_accuracy","value":0.84,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.hy","metric":"candidate_ece","value":0.057336805462837225,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.hy","metric":"baseline_accuracy","value":0.84,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.hy","metric":"baseline_ece","value":0.07812121435999876,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.id","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.id","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.id","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.id","metric":"candidate_accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.id","metric":"candidate_ece","value":0.05904934585094447,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.id","metric":"baseline_accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.id","metric":"baseline_ece","value":0.05200909078121181,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.is","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.is","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.is","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.is","metric":"candidate_accuracy","value":0.86,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.is","metric":"candidate_ece","value":0.05912843495607373,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.is","metric":"baseline_accuracy","value":0.84,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.is","metric":"baseline_ece","value":0.07296262651681899,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.it","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.it","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.it","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.it","metric":"candidate_accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.it","metric":"candidate_ece","value":0.050549613535404186,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.it","metric":"baseline_accuracy","value":0.96,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.it","metric":"baseline_ece","value":0.05021818310022354,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ja","metric":"regressed","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ja","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ja","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ja","metric":"candidate_accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ja","metric":"candidate_ece","value":0.06279848605394359,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ja","metric":"baseline_accuracy","value":0.97,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ja","metric":"baseline_ece","value":0.05874444544315334,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.jv","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.jv","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.jv","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.jv","metric":"candidate_accuracy","value":0.83,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.jv","metric":"candidate_ece","value":0.10250622883439062,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.jv","metric":"baseline_accuracy","value":0.75,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.jv","metric":"baseline_ece","value":0.07757171720266345,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ka","metric":"regressed","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ka","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ka","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ka","metric":"candidate_accuracy","value":0.79,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ka","metric":"candidate_ece","value":0.10097189977765086,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ka","metric":"baseline_accuracy","value":0.88,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ka","metric":"baseline_ece","value":0.09805454522371292,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.km","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.km","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.km","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.km","metric":"candidate_accuracy","value":0.8,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.km","metric":"candidate_ece","value":0.09994862869381903,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.km","metric":"baseline_accuracy","value":0.81,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.km","metric":"baseline_ece","value":0.07332727491855619,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.kn","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.kn","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.kn","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.kn","metric":"candidate_accuracy","value":0.87,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.kn","metric":"candidate_ece","value":0.05703154116868977,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.kn","metric":"baseline_accuracy","value":0.87,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.kn","metric":"baseline_ece","value":0.062040405273437446,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ko","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ko","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ko","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ko","metric":"candidate_accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ko","metric":"candidate_ece","value":0.06531809881329537,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ko","metric":"baseline_accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ko","metric":"baseline_ece","value":0.04226868838071823,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.lv","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.lv","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.lv","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.lv","metric":"candidate_accuracy","value":0.85,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.lv","metric":"candidate_ece","value":0.07480210557579993,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.lv","metric":"baseline_accuracy","value":0.85,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.lv","metric":"baseline_ece","value":0.06903636515140534,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ml","metric":"regressed","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ml","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ml","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ml","metric":"candidate_accuracy","value":0.82,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ml","metric":"candidate_ece","value":0.06367021948099139,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ml","metric":"baseline_accuracy","value":0.91,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ml","metric":"baseline_ece","value":0.049515153467655115,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.mn","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.mn","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.mn","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.mn","metric":"candidate_accuracy","value":0.88,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.mn","metric":"candidate_ece","value":0.10917697399854666,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.mn","metric":"baseline_accuracy","value":0.84,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.mn","metric":"baseline_ece","value":0.08651313126087185,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ms","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ms","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ms","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ms","metric":"candidate_accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ms","metric":"candidate_ece","value":0.05931672349572179,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ms","metric":"baseline_accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ms","metric":"baseline_ece","value":0.06841818481683726,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.my","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.my","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.my","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.my","metric":"candidate_accuracy","value":0.9,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.my","metric":"candidate_ece","value":0.06698188766837115,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.my","metric":"baseline_accuracy","value":0.89,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.my","metric":"baseline_ece","value":0.07051919519901269,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.nb","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.nb","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.nb","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.nb","metric":"candidate_accuracy","value":0.91,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.nb","metric":"candidate_ece","value":0.06739467456936835,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.nb","metric":"baseline_accuracy","value":0.86,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.nb","metric":"baseline_ece","value":0.06732828453183172,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.nl","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.nl","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.nl","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.nl","metric":"candidate_accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.nl","metric":"candidate_ece","value":0.06967766851186752,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.nl","metric":"baseline_accuracy","value":0.9,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.nl","metric":"baseline_ece","value":0.055070708692073866,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.pl","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.pl","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.pl","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.pl","metric":"candidate_accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.pl","metric":"candidate_ece","value":0.06593496680259701,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.pl","metric":"baseline_accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.pl","metric":"baseline_ece","value":0.04303939431905744,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.pt","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.pt","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.pt","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.pt","metric":"candidate_accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.pt","metric":"candidate_ece","value":0.048627837598323856,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.pt","metric":"baseline_accuracy","value":0.9,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.pt","metric":"baseline_ece","value":0.09853636592626569,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ro","metric":"regressed","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ro","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ro","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ro","metric":"candidate_accuracy","value":0.9,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ro","metric":"candidate_ece","value":0.05783455386757851,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ro","metric":"baseline_accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ro","metric":"baseline_ece","value":0.08220606148242947,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ru","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ru","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ru","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ru","metric":"candidate_accuracy","value":0.96,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ru","metric":"candidate_ece","value":0.06510985448956491,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ru","metric":"baseline_accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ru","metric":"baseline_ece","value":0.0405292943120003,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sl","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.sl","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.sl","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.sl","metric":"candidate_accuracy","value":0.9,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sl","metric":"candidate_ece","value":0.05202076986432077,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sl","metric":"baseline_accuracy","value":0.89,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sl","metric":"baseline_ece","value":0.057861616015434277,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sq","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.sq","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.sq","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.sq","metric":"candidate_accuracy","value":0.88,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sq","metric":"candidate_ece","value":0.07386779703199861,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sq","metric":"baseline_accuracy","value":0.76,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sq","metric":"baseline_ece","value":0.0900494961440563,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sv","metric":"regressed","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.sv","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.sv","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.sv","metric":"candidate_accuracy","value":0.88,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sv","metric":"candidate_ece","value":0.06018931411206718,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sv","metric":"baseline_accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sv","metric":"baseline_ece","value":0.07373636573553077,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sw","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.sw","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.sw","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.sw","metric":"candidate_accuracy","value":0.81,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sw","metric":"candidate_ece","value":0.06739190667867664,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sw","metric":"baseline_accuracy","value":0.7,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sw","metric":"baseline_ece","value":0.11177171409130097,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ta","metric":"regressed","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ta","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ta","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ta","metric":"candidate_accuracy","value":0.84,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ta","metric":"candidate_ece","value":0.08809512436389924,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ta","metric":"baseline_accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ta","metric":"baseline_ece","value":0.09244343683123586,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.te","metric":"regressed","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.te","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.te","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.te","metric":"candidate_accuracy","value":0.83,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.te","metric":"candidate_ece","value":0.0734219098091125,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.te","metric":"baseline_accuracy","value":0.91,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.te","metric":"baseline_ece","value":0.04149899095296861,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.th","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.th","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.th","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.th","metric":"candidate_accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.th","metric":"candidate_ece","value":0.049760746955871585,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.th","metric":"baseline_accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.th","metric":"baseline_ece","value":0.04299999922513963,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.tl","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.tl","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.tl","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.tl","metric":"candidate_accuracy","value":0.86,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.tl","metric":"candidate_ece","value":0.050127259045839354,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.tl","metric":"baseline_accuracy","value":0.85,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.tl","metric":"baseline_ece","value":0.08445252478122714,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.tr","metric":"regressed","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.tr","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.tr","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.tr","metric":"candidate_accuracy","value":0.88,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.tr","metric":"candidate_ece","value":0.07631486147642136,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.tr","metric":"baseline_accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.tr","metric":"baseline_ece","value":0.0609899020195007,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ur","metric":"regressed","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ur","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ur","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ur","metric":"candidate_accuracy","value":0.86,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ur","metric":"candidate_ece","value":0.07368002861738204,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ur","metric":"baseline_accuracy","value":0.9,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ur","metric":"baseline_ece","value":0.06538687020540238,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.vi","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.vi","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.vi","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.vi","metric":"candidate_accuracy","value":0.89,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.vi","metric":"candidate_ece","value":0.05955071091651916,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.vi","metric":"baseline_accuracy","value":0.89,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.vi","metric":"baseline_ece","value":0.04620000123977666,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.zh-CN","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.zh-CN","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.zh-CN","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.zh-CN","metric":"candidate_accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.zh-CN","metric":"candidate_ece","value":0.050360209196805984,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.zh-CN","metric":"baseline_accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.zh-CN","metric":"baseline_ece","value":0.03629999935626985,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.zh-TW","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.zh-TW","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.zh-TW","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.zh-TW","metric":"candidate_accuracy","value":0.97,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.zh-TW","metric":"candidate_ece","value":0.07615835040807721,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.zh-TW","metric":"baseline_accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.zh-TW","metric":"baseline_ece","value":0.06207878708839416,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/clinc150","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/clinc150","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/clinc150","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/clinc150","metric":"candidate_accuracy","value":0.76,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/clinc150","metric":"candidate_ece","value":0.14612227261066435,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/sst5","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/sst5","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/sst5","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/sst5","metric":"candidate_accuracy","value":0.39,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/sst5","metric":"candidate_ece","value":0.18840801835060123,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.ar","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.ar","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.ar","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.ar","metric":"candidate_accuracy","value":0.66,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.ar","metric":"candidate_ece","value":0.14292311161756513,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.bg","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.bg","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.bg","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.bg","metric":"candidate_accuracy","value":0.77,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.bg","metric":"candidate_ece","value":0.1984384161233902,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.de","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.de","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.de","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.de","metric":"candidate_accuracy","value":0.8,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.de","metric":"candidate_ece","value":0.21225321918725967,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.el","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.el","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.el","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.el","metric":"candidate_accuracy","value":0.76,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.el","metric":"candidate_ece","value":0.24297191977500918,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.en","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.en","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.en","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.en","metric":"candidate_accuracy","value":0.77,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.en","metric":"candidate_ece","value":0.157812180519104,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.es","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.es","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.es","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.es","metric":"candidate_accuracy","value":0.79,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.es","metric":"candidate_ece","value":0.22035292655229569,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.fr","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.fr","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.fr","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.fr","metric":"candidate_accuracy","value":0.77,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.fr","metric":"candidate_ece","value":0.18113152056932452,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.hi","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.hi","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.hi","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.hi","metric":"candidate_accuracy","value":0.64,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.hi","metric":"candidate_ece","value":0.13377510339021684,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.ru","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.ru","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.ru","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.ru","metric":"candidate_accuracy","value":0.72,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.ru","metric":"candidate_ece","value":0.19517139703035355,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.sw","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.sw","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.sw","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.sw","metric":"candidate_accuracy","value":0.59,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.sw","metric":"candidate_ece","value":0.1174077433347702,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.th","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.th","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.th","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.th","metric":"candidate_accuracy","value":0.66,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.th","metric":"candidate_ece","value":0.1255523270368576,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.tr","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.tr","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.tr","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.tr","metric":"candidate_accuracy","value":0.76,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.tr","metric":"candidate_ece","value":0.24283051997423175,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.ur","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.ur","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.ur","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.ur","metric":"candidate_accuracy","value":0.59,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.ur","metric":"candidate_ece","value":0.14045946687459945,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.vi","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.vi","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.vi","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.vi","metric":"candidate_accuracy","value":0.73,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.vi","metric":"candidate_ece","value":0.16299013167619708,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.zh","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.zh","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.zh","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.zh","metric":"candidate_accuracy","value":0.68,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.zh","metric":"candidate_ece","value":0.15210912466049198,"lo":null,"hi":null,"n":100,"of":null},{"category":"robustness/shuffle","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/shuffle","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/shuffle","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/shuffle","metric":"candidate_accuracy","value":0.878845819572397,"lo":null,"hi":null,"n":5753,"of":null},{"category":"robustness/shuffle","metric":"candidate_ece","value":0.009889678792260997,"lo":null,"hi":null,"n":5753,"of":null},{"category":"robustness/alias","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/alias","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/alias","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/alias","metric":"candidate_accuracy","value":0.8578052550231839,"lo":null,"hi":null,"n":647,"of":null},{"category":"robustness/alias","metric":"candidate_ece","value":0.04941958235072942,"lo":null,"hi":null,"n":647,"of":null},{"category":"robustness/swap","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/swap","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/swap","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/swap","metric":"candidate_accuracy","value":0.8562596599690881,"lo":null,"hi":null,"n":647,"of":null},{"category":"robustness/swap","metric":"candidate_ece","value":0.04407818801988954,"lo":null,"hi":null,"n":647,"of":null},{"category":"robustness/subset","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/subset","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/subset","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/subset","metric":"candidate_accuracy","value":0.940188679245283,"lo":null,"hi":null,"n":5300,"of":null},{"category":"robustness/subset","metric":"candidate_ece","value":0.011920550423410763,"lo":null,"hi":null,"n":5300,"of":null},{"category":"robustness/near","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/near","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/near","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/near","metric":"candidate_accuracy","value":0.9060377358490566,"lo":null,"hi":null,"n":5300,"of":null},{"category":"robustness/near","metric":"candidate_ece","value":0.030989159809530865,"lo":null,"hi":null,"n":5300,"of":null}],"notes":"tolerance accuracy 0.02, ece 0.03; gate sha256:15b20237530cf8f6b9251b13cabbe7fdc60d950d1d01029f57c9ad9d1f5aaa2c","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"gate.jev-drawn-a6.a211cc70-kai","benchmark":"gate.jev","split":"drawn","system":"kai","version":"a6@a211cc70103825bfbc9bfb0fbe970f8bf08da2efe0953cd8045a3cd07397f4bf","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"11474208ce49e6834bad21db4af8bb7ddf33b0ada6138b3661938522cafc815f","store_digest":"","facts_digest":"","commit":"a1a4a73d524cecf8038e30c408ad30b66e71d490","questions":null,"answered":null,"when":"","ended":"2026-09-28T03:18:42Z","by":"","completion":"unknown","measures":[{"category":"all","metric":"pass","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"suites","value":62,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"candidate","value":0.8966950146627568,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"baseline","value":0.8669257086999022,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"delta","value":0.029769305962854653,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"regressions","value":11,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"cases","value":100,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/jev.ag_news","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/jev.ag_news","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/jev.ag_news","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/jev.ag_news","metric":"candidate_accuracy","value":0.95,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/jev.ag_news","metric":"candidate_ece","value":0.04369553148746491,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/jev.ag_news","metric":"baseline_accuracy","value":0.83,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/jev.ag_news","metric":"baseline_ece","value":0.12050000071525568,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/jev.emotion","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/jev.emotion","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/jev.emotion","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/jev.emotion","metric":"candidate_accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/jev.emotion","metric":"candidate_ece","value":0.04338422119617459,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/jev.emotion","metric":"baseline_accuracy","value":0.64,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/jev.emotion","metric":"baseline_ece","value":0.24150000125169754,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/jev.banking77_full","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/jev.banking77_full","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/jev.banking77_full","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/jev.banking77_full","metric":"candidate_accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/jev.banking77_full","metric":"candidate_ece","value":0.06478451311588287,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/jev.banking77_full","metric":"baseline_accuracy","value":0.85,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/jev.banking77_full","metric":"baseline_ece","value":0.08564848810434346,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.support_triage","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.support_triage","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.support_triage","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.support_triage","metric":"candidate_accuracy","value":0.37,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.support_triage","metric":"candidate_ece","value":0.4822892910242081,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.support_triage","metric":"baseline_accuracy","value":0.29,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.support_triage","metric":"baseline_ece","value":0.561000002026558,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.email_spam","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.email_spam","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.email_spam","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.email_spam","metric":"candidate_accuracy","value":0.99,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.email_spam","metric":"candidate_ece","value":0.00896555125713343,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.email_spam","metric":"baseline_accuracy","value":0.96,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.email_spam","metric":"baseline_ece","value":0.07149999797344209,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.phishing","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.phishing","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.phishing","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.phishing","metric":"candidate_accuracy","value":1,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.phishing","metric":"candidate_ece","value":0.0000335377454757646,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.phishing","metric":"baseline_accuracy","value":0.89,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.phishing","metric":"baseline_ece","value":0.08049999296665189,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.guardrails_jailbreak","metric":"regressed","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.guardrails_jailbreak","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.guardrails_jailbreak","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.guardrails_jailbreak","metric":"candidate_accuracy","value":0.91,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.guardrails_jailbreak","metric":"candidate_ece","value":0.060433058142662074,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.guardrails_jailbreak","metric":"baseline_accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.guardrails_jailbreak","metric":"baseline_ece","value":0.0655999952554703,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.moderation_toxicity","metric":"regressed","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.moderation_toxicity","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.moderation_toxicity","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.moderation_toxicity","metric":"candidate_accuracy","value":0.81,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.moderation_toxicity","metric":"candidate_ece","value":0.1910340225696563,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.moderation_toxicity","metric":"baseline_accuracy","value":0.73,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.moderation_toxicity","metric":"baseline_ece","value":0.1372000062465668,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.rag_relevance","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.rag_relevance","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.rag_relevance","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.rag_relevance","metric":"candidate_accuracy","value":0.57,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.rag_relevance","metric":"candidate_ece","value":0.11459835350513459,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.rag_relevance","metric":"baseline_accuracy","value":0.51,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.rag_relevance","metric":"baseline_ece","value":0.39620000362396246,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.model_routing_domain","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.model_routing_domain","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.model_routing_domain","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.model_routing_domain","metric":"candidate_accuracy","value":1,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.model_routing_domain","metric":"candidate_ece","value":0.0003541868925094338,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.model_routing_domain","metric":"baseline_accuracy","value":0.98,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.model_routing_domain","metric":"baseline_ece","value":0.026600003242492693,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/typed_decisions","metric":"regressed","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/typed_decisions","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/typed_decisions","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/typed_decisions","metric":"candidate_accuracy","value":0.746,"lo":null,"hi":null,"n":500,"of":null},{"category":"harness/typed_decisions","metric":"candidate_ece","value":0.20499304324388506,"lo":null,"hi":null,"n":500,"of":null},{"category":"harness/typed_decisions","metric":"baseline_accuracy","value":0.73,"lo":null,"hi":null,"n":500,"of":null},{"category":"harness/typed_decisions","metric":"baseline_ece","value":0.07618141478300096,"lo":null,"hi":null,"n":500,"of":null},{"category":"harness/massive.af","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.af","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.af","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.af","metric":"candidate_accuracy","value":0.9,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.af","metric":"candidate_ece","value":0.06145614624023434,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.af","metric":"baseline_accuracy","value":0.87,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.af","metric":"baseline_ece","value":0.08726868644356728,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.am","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.am","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.am","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.am","metric":"candidate_accuracy","value":0.85,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.am","metric":"candidate_ece","value":0.0714535458385944,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.am","metric":"baseline_accuracy","value":0.85,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.am","metric":"baseline_ece","value":0.09598384097218512,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ar","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ar","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ar","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ar","metric":"candidate_accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ar","metric":"candidate_ece","value":0.06882738918066027,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ar","metric":"baseline_accuracy","value":0.87,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ar","metric":"baseline_ece","value":0.08222424164414406,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.az","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.az","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.az","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.az","metric":"candidate_accuracy","value":0.95,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.az","metric":"candidate_ece","value":0.048022165298461886,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.az","metric":"baseline_accuracy","value":0.88,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.az","metric":"baseline_ece","value":0.04820606261491777,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.bn","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.bn","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.bn","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.bn","metric":"candidate_accuracy","value":0.9,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.bn","metric":"candidate_ece","value":0.08672305166721346,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.bn","metric":"baseline_accuracy","value":0.86,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.bn","metric":"baseline_ece","value":0.0710626265406609,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.cy","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.cy","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.cy","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.cy","metric":"candidate_accuracy","value":0.86,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.cy","metric":"candidate_ece","value":0.049075031131505915,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.cy","metric":"baseline_accuracy","value":0.76,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.cy","metric":"baseline_ece","value":0.09916767939925195,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.da","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.da","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.da","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.da","metric":"candidate_accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.da","metric":"candidate_ece","value":0.03354248374700548,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.da","metric":"baseline_accuracy","value":0.88,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.da","metric":"baseline_ece","value":0.038734343349933635,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.de","metric":"regressed","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.de","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.de","metric":"unanswered","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.de","metric":"candidate_accuracy","value":0.9090909090909091,"lo":null,"hi":null,"n":99,"of":null},{"category":"harness/massive.de","metric":"candidate_ece","value":0.06348552186079703,"lo":null,"hi":null,"n":99,"of":null},{"category":"harness/massive.de","metric":"baseline_accuracy","value":0.9393939393939394,"lo":null,"hi":null,"n":99,"of":null},{"category":"harness/massive.de","metric":"baseline_ece","value":0.057024792589322484,"lo":null,"hi":null,"n":99,"of":null},{"category":"harness/massive.el","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.el","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.el","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.el","metric":"candidate_accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.el","metric":"candidate_ece","value":0.04968505382537845,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.el","metric":"baseline_accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.el","metric":"baseline_ece","value":0.057093940079212174,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.en","metric":"regressed","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.en","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.en","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.en","metric":"candidate_accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.en","metric":"candidate_ece","value":0.05991215109825135,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.en","metric":"baseline_accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.en","metric":"baseline_ece","value":0.02530000150203709,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.es","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.es","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.es","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.es","metric":"candidate_accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.es","metric":"candidate_ece","value":0.060675814151763846,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.es","metric":"baseline_accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.es","metric":"baseline_ece","value":0.06282828330993653,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.fa","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.fa","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.fa","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.fa","metric":"candidate_accuracy","value":0.95,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.fa","metric":"candidate_ece","value":0.03944258913397788,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.fa","metric":"baseline_accuracy","value":0.95,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.fa","metric":"baseline_ece","value":0.04546464592218401,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.fi","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.fi","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.fi","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.fi","metric":"candidate_accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.fi","metric":"candidate_ece","value":0.04163172006607054,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.fi","metric":"baseline_accuracy","value":0.89,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.fi","metric":"baseline_ece","value":0.0383111131191253,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.fr","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.fr","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.fr","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.fr","metric":"candidate_accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.fr","metric":"candidate_ece","value":0.056455891728401285,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.fr","metric":"baseline_accuracy","value":0.9,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.fr","metric":"baseline_ece","value":0.0651949521899224,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.he","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.he","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.he","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.he","metric":"candidate_accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.he","metric":"candidate_ece","value":0.06197642922401433,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.he","metric":"baseline_accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.he","metric":"baseline_ece","value":0.042724243700504254,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.hi","metric":"regressed","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.hi","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.hi","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.hi","metric":"candidate_accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.hi","metric":"candidate_ece","value":0.05225563615560533,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.hi","metric":"baseline_accuracy","value":0.95,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.hi","metric":"baseline_ece","value":0.029023234546184544,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.hu","metric":"regressed","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.hu","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.hu","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.hu","metric":"candidate_accuracy","value":0.86,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.hu","metric":"candidate_ece","value":0.0787442502379418,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.hu","metric":"baseline_accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.hu","metric":"baseline_ece","value":0.03867272794246674,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.hy","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.hy","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.hy","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.hy","metric":"candidate_accuracy","value":0.87,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.hy","metric":"candidate_ece","value":0.06506374388933181,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.hy","metric":"baseline_accuracy","value":0.84,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.hy","metric":"baseline_ece","value":0.07812121435999876,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.id","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.id","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.id","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.id","metric":"candidate_accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.id","metric":"candidate_ece","value":0.03650752246379858,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.id","metric":"baseline_accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.id","metric":"baseline_ece","value":0.05200909078121181,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.is","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.is","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.is","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.is","metric":"candidate_accuracy","value":0.87,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.is","metric":"candidate_ece","value":0.0637242865562439,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.is","metric":"baseline_accuracy","value":0.84,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.is","metric":"baseline_ece","value":0.07296262651681899,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.it","metric":"regressed","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.it","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.it","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.it","metric":"candidate_accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.it","metric":"candidate_ece","value":0.056774215698242245,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.it","metric":"baseline_accuracy","value":0.96,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.it","metric":"baseline_ece","value":0.05021818310022354,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ja","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ja","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ja","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ja","metric":"candidate_accuracy","value":0.97,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ja","metric":"candidate_ece","value":0.03286569952964781,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ja","metric":"baseline_accuracy","value":0.97,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ja","metric":"baseline_ece","value":0.05874444544315334,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.jv","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.jv","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.jv","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.jv","metric":"candidate_accuracy","value":0.85,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.jv","metric":"candidate_ece","value":0.07020064458251002,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.jv","metric":"baseline_accuracy","value":0.75,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.jv","metric":"baseline_ece","value":0.07757171720266345,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ka","metric":"regressed","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ka","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ka","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ka","metric":"candidate_accuracy","value":0.83,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ka","metric":"candidate_ece","value":0.08449125021696088,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ka","metric":"baseline_accuracy","value":0.88,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ka","metric":"baseline_ece","value":0.09805454522371292,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.km","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.km","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.km","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.km","metric":"candidate_accuracy","value":0.85,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.km","metric":"candidate_ece","value":0.08929385468363762,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.km","metric":"baseline_accuracy","value":0.81,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.km","metric":"baseline_ece","value":0.07332727491855619,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.kn","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.kn","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.kn","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.kn","metric":"candidate_accuracy","value":0.9,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.kn","metric":"candidate_ece","value":0.08473139792680738,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.kn","metric":"baseline_accuracy","value":0.87,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.kn","metric":"baseline_ece","value":0.062040405273437446,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ko","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ko","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ko","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ko","metric":"candidate_accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ko","metric":"candidate_ece","value":0.03255369871854782,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ko","metric":"baseline_accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ko","metric":"baseline_ece","value":0.04226868838071823,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.lv","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.lv","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.lv","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.lv","metric":"candidate_accuracy","value":0.89,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.lv","metric":"candidate_ece","value":0.07157214641571044,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.lv","metric":"baseline_accuracy","value":0.85,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.lv","metric":"baseline_ece","value":0.06903636515140534,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ml","metric":"regressed","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ml","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ml","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ml","metric":"candidate_accuracy","value":0.87,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ml","metric":"candidate_ece","value":0.08362111099064345,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ml","metric":"baseline_accuracy","value":0.91,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ml","metric":"baseline_ece","value":0.049515153467655115,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.mn","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.mn","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.mn","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.mn","metric":"candidate_accuracy","value":0.88,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.mn","metric":"candidate_ece","value":0.08079831957817081,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.mn","metric":"baseline_accuracy","value":0.84,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.mn","metric":"baseline_ece","value":0.08651313126087185,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ms","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ms","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ms","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ms","metric":"candidate_accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ms","metric":"candidate_ece","value":0.06407836556434633,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ms","metric":"baseline_accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ms","metric":"baseline_ece","value":0.06841818481683726,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.my","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.my","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.my","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.my","metric":"candidate_accuracy","value":0.95,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.my","metric":"candidate_ece","value":0.04778237879276278,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.my","metric":"baseline_accuracy","value":0.89,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.my","metric":"baseline_ece","value":0.07051919519901269,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.nb","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.nb","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.nb","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.nb","metric":"candidate_accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.nb","metric":"candidate_ece","value":0.04958597868680945,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.nb","metric":"baseline_accuracy","value":0.86,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.nb","metric":"baseline_ece","value":0.06732828453183172,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.nl","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.nl","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.nl","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.nl","metric":"candidate_accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.nl","metric":"candidate_ece","value":0.05387416154146192,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.nl","metric":"baseline_accuracy","value":0.9,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.nl","metric":"baseline_ece","value":0.055070708692073866,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.pl","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.pl","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.pl","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.pl","metric":"candidate_accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.pl","metric":"candidate_ece","value":0.029607748985290512,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.pl","metric":"baseline_accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.pl","metric":"baseline_ece","value":0.04303939431905744,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.pt","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.pt","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.pt","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.pt","metric":"candidate_accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.pt","metric":"candidate_ece","value":0.03121975854039198,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.pt","metric":"baseline_accuracy","value":0.9,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.pt","metric":"baseline_ece","value":0.09853636592626569,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ro","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ro","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ro","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ro","metric":"candidate_accuracy","value":0.95,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ro","metric":"candidate_ece","value":0.04741007596254352,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ro","metric":"baseline_accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ro","metric":"baseline_ece","value":0.08220606148242947,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ru","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ru","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ru","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ru","metric":"candidate_accuracy","value":0.98,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ru","metric":"candidate_ece","value":0.04467343077063564,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ru","metric":"baseline_accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ru","metric":"baseline_ece","value":0.0405292943120003,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sl","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.sl","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.sl","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.sl","metric":"candidate_accuracy","value":0.89,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sl","metric":"candidate_ece","value":0.048560316711664275,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sl","metric":"baseline_accuracy","value":0.89,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sl","metric":"baseline_ece","value":0.057861616015434277,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sq","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.sq","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.sq","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.sq","metric":"candidate_accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sq","metric":"candidate_ece","value":0.047673491090536056,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sq","metric":"baseline_accuracy","value":0.76,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sq","metric":"baseline_ece","value":0.0900494961440563,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sv","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.sv","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.sv","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.sv","metric":"candidate_accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sv","metric":"candidate_ece","value":0.05960643857717518,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sv","metric":"baseline_accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sv","metric":"baseline_ece","value":0.07373636573553077,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sw","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.sw","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.sw","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.sw","metric":"candidate_accuracy","value":0.83,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sw","metric":"candidate_ece","value":0.07877558827400208,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sw","metric":"baseline_accuracy","value":0.7,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sw","metric":"baseline_ece","value":0.11177171409130097,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ta","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ta","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ta","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ta","metric":"candidate_accuracy","value":0.91,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ta","metric":"candidate_ece","value":0.04905078291893006,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ta","metric":"baseline_accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ta","metric":"baseline_ece","value":0.09244343683123586,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.te","metric":"regressed","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.te","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.te","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.te","metric":"candidate_accuracy","value":0.85,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.te","metric":"candidate_ece","value":0.11967309281229979,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.te","metric":"baseline_accuracy","value":0.91,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.te","metric":"baseline_ece","value":0.04149899095296861,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.th","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.th","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.th","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.th","metric":"candidate_accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.th","metric":"candidate_ece","value":0.03415450304746626,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.th","metric":"baseline_accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.th","metric":"baseline_ece","value":0.04299999922513963,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.tl","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.tl","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.tl","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.tl","metric":"candidate_accuracy","value":0.91,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.tl","metric":"candidate_ece","value":0.07738257154822352,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.tl","metric":"baseline_accuracy","value":0.85,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.tl","metric":"baseline_ece","value":0.08445252478122714,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.tr","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.tr","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.tr","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.tr","metric":"candidate_accuracy","value":0.9,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.tr","metric":"candidate_ece","value":0.07372443810105328,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.tr","metric":"baseline_accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.tr","metric":"baseline_ece","value":0.0609899020195007,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ur","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ur","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ur","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ur","metric":"candidate_accuracy","value":0.91,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ur","metric":"candidate_ece","value":0.06974980145692826,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ur","metric":"baseline_accuracy","value":0.9,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ur","metric":"baseline_ece","value":0.06538687020540238,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.vi","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.vi","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.vi","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.vi","metric":"candidate_accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.vi","metric":"candidate_ece","value":0.06033400148153302,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.vi","metric":"baseline_accuracy","value":0.89,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.vi","metric":"baseline_ece","value":0.04620000123977666,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.zh-CN","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.zh-CN","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.zh-CN","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.zh-CN","metric":"candidate_accuracy","value":0.95,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.zh-CN","metric":"candidate_ece","value":0.039882711470127084,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.zh-CN","metric":"baseline_accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.zh-CN","metric":"baseline_ece","value":0.03629999935626985,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.zh-TW","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.zh-TW","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.zh-TW","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.zh-TW","metric":"candidate_accuracy","value":0.96,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.zh-TW","metric":"candidate_ece","value":0.04104021757841109,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.zh-TW","metric":"baseline_accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.zh-TW","metric":"baseline_ece","value":0.06207878708839416,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/clinc150","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/clinc150","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/clinc150","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/clinc150","metric":"candidate_accuracy","value":0.74,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/clinc150","metric":"candidate_ece","value":0.16210970818996423,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/sst5","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/sst5","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/sst5","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/sst5","metric":"candidate_accuracy","value":0.52,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/sst5","metric":"candidate_ece","value":0.1463029870390892,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.ar","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.ar","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.ar","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.ar","metric":"candidate_accuracy","value":0.69,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.ar","metric":"candidate_ece","value":0.0982473087310791,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.bg","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.bg","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.bg","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.bg","metric":"candidate_accuracy","value":0.79,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.bg","metric":"candidate_ece","value":0.14978477597236634,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.de","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.de","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.de","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.de","metric":"candidate_accuracy","value":0.8,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.de","metric":"candidate_ece","value":0.1645866242051125,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.el","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.el","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.el","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.el","metric":"candidate_accuracy","value":0.81,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.el","metric":"candidate_ece","value":0.18419245570898057,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.en","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.en","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.en","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.en","metric":"candidate_accuracy","value":0.77,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.en","metric":"candidate_ece","value":0.10587243705987928,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.es","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.es","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.es","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.es","metric":"candidate_accuracy","value":0.72,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.es","metric":"candidate_ece","value":0.13122842520475386,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.fr","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.fr","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.fr","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.fr","metric":"candidate_accuracy","value":0.78,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.fr","metric":"candidate_ece","value":0.1400255334377289,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.hi","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.hi","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.hi","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.hi","metric":"candidate_accuracy","value":0.65,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.hi","metric":"candidate_ece","value":0.1458137741684914,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.ru","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.ru","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.ru","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.ru","metric":"candidate_accuracy","value":0.71,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.ru","metric":"candidate_ece","value":0.09388305157423021,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.sw","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.sw","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.sw","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.sw","metric":"candidate_accuracy","value":0.61,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.sw","metric":"candidate_ece","value":0.09246866583824158,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.th","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.th","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.th","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.th","metric":"candidate_accuracy","value":0.7,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.th","metric":"candidate_ece","value":0.13970610022544858,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.tr","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.tr","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.tr","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.tr","metric":"candidate_accuracy","value":0.73,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.tr","metric":"candidate_ece","value":0.1332623443007469,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.ur","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.ur","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.ur","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.ur","metric":"candidate_accuracy","value":0.61,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.ur","metric":"candidate_ece","value":0.08311391204595568,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.vi","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.vi","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.vi","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.vi","metric":"candidate_accuracy","value":0.73,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.vi","metric":"candidate_ece","value":0.14557734400033948,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.zh","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.zh","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.zh","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.zh","metric":"candidate_accuracy","value":0.7,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.zh","metric":"candidate_ece","value":0.06273882865905764,"lo":null,"hi":null,"n":100,"of":null},{"category":"robustness/shuffle","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/shuffle","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/shuffle","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/shuffle","metric":"candidate_accuracy","value":0.898140100816965,"lo":null,"hi":null,"n":5753,"of":null},{"category":"robustness/shuffle","metric":"candidate_ece","value":0.025156406533214614,"lo":null,"hi":null,"n":5753,"of":null},{"category":"robustness/alias","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/alias","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/alias","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/alias","metric":"candidate_accuracy","value":0.8438948995363215,"lo":null,"hi":null,"n":647,"of":null},{"category":"robustness/alias","metric":"candidate_ece","value":0.05747238497092885,"lo":null,"hi":null,"n":647,"of":null},{"category":"robustness/swap","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/swap","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/swap","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/swap","metric":"candidate_accuracy","value":0.8469860896445132,"lo":null,"hi":null,"n":647,"of":null},{"category":"robustness/swap","metric":"candidate_ece","value":0.046368170374143654,"lo":null,"hi":null,"n":647,"of":null},{"category":"robustness/subset","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/subset","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/subset","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/subset","metric":"candidate_accuracy","value":0.9509433962264151,"lo":null,"hi":null,"n":5300,"of":null},{"category":"robustness/subset","metric":"candidate_ece","value":0.01296593981250278,"lo":null,"hi":null,"n":5300,"of":null},{"category":"robustness/near","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/near","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/near","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/near","metric":"candidate_accuracy","value":0.9235849056603773,"lo":null,"hi":null,"n":5300,"of":null},{"category":"robustness/near","metric":"candidate_ece","value":0.01731854601851056,"lo":null,"hi":null,"n":5300,"of":null}],"notes":"tolerance accuracy 0.02, ece 0.03; gate sha256:70a351500cfd32436bc08930bf9ecb07d8af92e674949b0fd509b9d1f88bd760","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"gate.jev-drawn-a7.0834a74f-kai","benchmark":"gate.jev","split":"drawn","system":"kai","version":"a7@0834a74f2d140642a453373da09e5a128e3d2c8d9e8bb1dcaf86d7210a4ecdfc","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"11474208ce49e6834bad21db4af8bb7ddf33b0ada6138b3661938522cafc815f","store_digest":"","facts_digest":"","commit":"a1a4a73d524cecf8038e30c408ad30b66e71d490","questions":null,"answered":null,"when":"","ended":"2026-09-28T12:24:45Z","by":"","completion":"unknown","measures":[{"category":"all","metric":"pass","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"suites","value":62,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"candidate","value":0.8962756598240467,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"baseline","value":0.8669257086999022,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"delta","value":0.029349951124144513,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"regressions","value":13,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"cases","value":100,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/jev.ag_news","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/jev.ag_news","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/jev.ag_news","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/jev.ag_news","metric":"candidate_accuracy","value":0.96,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/jev.ag_news","metric":"candidate_ece","value":0.055983276665210704,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/jev.ag_news","metric":"baseline_accuracy","value":0.83,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/jev.ag_news","metric":"baseline_ece","value":0.12050000071525568,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/jev.emotion","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/jev.emotion","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/jev.emotion","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/jev.emotion","metric":"candidate_accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/jev.emotion","metric":"candidate_ece","value":0.037905553877353705,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/jev.emotion","metric":"baseline_accuracy","value":0.64,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/jev.emotion","metric":"baseline_ece","value":0.24150000125169754,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/jev.banking77_full","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/jev.banking77_full","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/jev.banking77_full","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/jev.banking77_full","metric":"candidate_accuracy","value":0.91,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/jev.banking77_full","metric":"candidate_ece","value":0.09422180116176605,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/jev.banking77_full","metric":"baseline_accuracy","value":0.85,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/jev.banking77_full","metric":"baseline_ece","value":0.08564848810434346,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.support_triage","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.support_triage","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.support_triage","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.support_triage","metric":"candidate_accuracy","value":0.35,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.support_triage","metric":"candidate_ece","value":0.512402692437172,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.support_triage","metric":"baseline_accuracy","value":0.29,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.support_triage","metric":"baseline_ece","value":0.561000002026558,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.email_spam","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.email_spam","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.email_spam","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.email_spam","metric":"candidate_accuracy","value":0.99,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.email_spam","metric":"candidate_ece","value":0.00944998085498816,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.email_spam","metric":"baseline_accuracy","value":0.96,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.email_spam","metric":"baseline_ece","value":0.07149999797344209,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.phishing","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.phishing","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.phishing","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.phishing","metric":"candidate_accuracy","value":1,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.phishing","metric":"candidate_ece","value":0.0000173753499984608,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.phishing","metric":"baseline_accuracy","value":0.89,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.phishing","metric":"baseline_ece","value":0.08049999296665189,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.guardrails_jailbreak","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.guardrails_jailbreak","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.guardrails_jailbreak","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.guardrails_jailbreak","metric":"candidate_accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.guardrails_jailbreak","metric":"candidate_ece","value":0.04400036811828614,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.guardrails_jailbreak","metric":"baseline_accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.guardrails_jailbreak","metric":"baseline_ece","value":0.0655999952554703,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.moderation_toxicity","metric":"regressed","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.moderation_toxicity","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.moderation_toxicity","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.moderation_toxicity","metric":"candidate_accuracy","value":0.81,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.moderation_toxicity","metric":"candidate_ece","value":0.1898956561088562,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.moderation_toxicity","metric":"baseline_accuracy","value":0.73,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.moderation_toxicity","metric":"baseline_ece","value":0.1372000062465668,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.rag_relevance","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.rag_relevance","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.rag_relevance","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.rag_relevance","metric":"candidate_accuracy","value":0.55,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.rag_relevance","metric":"candidate_ece","value":0.12484459042549137,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.rag_relevance","metric":"baseline_accuracy","value":0.51,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.rag_relevance","metric":"baseline_ece","value":0.39620000362396246,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.model_routing_domain","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.model_routing_domain","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.model_routing_domain","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.model_routing_domain","metric":"candidate_accuracy","value":1,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.model_routing_domain","metric":"candidate_ece","value":0.0002811008691787631,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.model_routing_domain","metric":"baseline_accuracy","value":0.98,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.model_routing_domain","metric":"baseline_ece","value":0.026600003242492693,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/typed_decisions","metric":"regressed","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/typed_decisions","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/typed_decisions","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/typed_decisions","metric":"candidate_accuracy","value":0.75,"lo":null,"hi":null,"n":500,"of":null},{"category":"harness/typed_decisions","metric":"candidate_ece","value":0.1635413258075714,"lo":null,"hi":null,"n":500,"of":null},{"category":"harness/typed_decisions","metric":"baseline_accuracy","value":0.73,"lo":null,"hi":null,"n":500,"of":null},{"category":"harness/typed_decisions","metric":"baseline_ece","value":0.07618141478300096,"lo":null,"hi":null,"n":500,"of":null},{"category":"harness/massive.af","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.af","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.af","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.af","metric":"candidate_accuracy","value":0.91,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.af","metric":"candidate_ece","value":0.04738804578781133,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.af","metric":"baseline_accuracy","value":0.87,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.af","metric":"baseline_ece","value":0.08726868644356728,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.am","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.am","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.am","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.am","metric":"candidate_accuracy","value":0.85,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.am","metric":"candidate_ece","value":0.08526809126138692,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.am","metric":"baseline_accuracy","value":0.85,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.am","metric":"baseline_ece","value":0.09598384097218512,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ar","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ar","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ar","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ar","metric":"candidate_accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ar","metric":"candidate_ece","value":0.06399317130446432,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ar","metric":"baseline_accuracy","value":0.87,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ar","metric":"baseline_ece","value":0.08222424164414406,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.az","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.az","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.az","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.az","metric":"candidate_accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.az","metric":"candidate_ece","value":0.0359751570224762,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.az","metric":"baseline_accuracy","value":0.88,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.az","metric":"baseline_ece","value":0.04820606261491777,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.bn","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.bn","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.bn","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.bn","metric":"candidate_accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.bn","metric":"candidate_ece","value":0.06581725120544439,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.bn","metric":"baseline_accuracy","value":0.86,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.bn","metric":"baseline_ece","value":0.0710626265406609,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.cy","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.cy","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.cy","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.cy","metric":"candidate_accuracy","value":0.86,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.cy","metric":"candidate_ece","value":0.08170279085636137,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.cy","metric":"baseline_accuracy","value":0.76,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.cy","metric":"baseline_ece","value":0.09916767939925195,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.da","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.da","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.da","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.da","metric":"candidate_accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.da","metric":"candidate_ece","value":0.050635895729064906,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.da","metric":"baseline_accuracy","value":0.88,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.da","metric":"baseline_ece","value":0.038734343349933635,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.de","metric":"regressed","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.de","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.de","metric":"unanswered","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.de","metric":"candidate_accuracy","value":0.9090909090909091,"lo":null,"hi":null,"n":99,"of":null},{"category":"harness/massive.de","metric":"candidate_ece","value":0.04236009400902375,"lo":null,"hi":null,"n":99,"of":null},{"category":"harness/massive.de","metric":"baseline_accuracy","value":0.9393939393939394,"lo":null,"hi":null,"n":99,"of":null},{"category":"harness/massive.de","metric":"baseline_ece","value":0.057024792589322484,"lo":null,"hi":null,"n":99,"of":null},{"category":"harness/massive.el","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.el","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.el","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.el","metric":"candidate_accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.el","metric":"candidate_ece","value":0.03144277065992359,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.el","metric":"baseline_accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.el","metric":"baseline_ece","value":0.057093940079212174,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.en","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.en","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.en","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.en","metric":"candidate_accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.en","metric":"candidate_ece","value":0.04790517151355743,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.en","metric":"baseline_accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.en","metric":"baseline_ece","value":0.02530000150203709,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.es","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.es","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.es","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.es","metric":"candidate_accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.es","metric":"candidate_ece","value":0.050300997197628004,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.es","metric":"baseline_accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.es","metric":"baseline_ece","value":0.06282828330993653,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.fa","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.fa","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.fa","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.fa","metric":"candidate_accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.fa","metric":"candidate_ece","value":0.03188615605235109,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.fa","metric":"baseline_accuracy","value":0.95,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.fa","metric":"baseline_ece","value":0.04546464592218401,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.fi","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.fi","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.fi","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.fi","metric":"candidate_accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.fi","metric":"candidate_ece","value":0.055109691023826686,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.fi","metric":"baseline_accuracy","value":0.89,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.fi","metric":"baseline_ece","value":0.0383111131191253,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.fr","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.fr","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.fr","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.fr","metric":"candidate_accuracy","value":0.91,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.fr","metric":"candidate_ece","value":0.07386496245861054,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.fr","metric":"baseline_accuracy","value":0.9,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.fr","metric":"baseline_ece","value":0.0651949521899224,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.he","metric":"regressed","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.he","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.he","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.he","metric":"candidate_accuracy","value":0.91,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.he","metric":"candidate_ece","value":0.055365550518035864,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.he","metric":"baseline_accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.he","metric":"baseline_ece","value":0.042724243700504254,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.hi","metric":"regressed","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.hi","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.hi","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.hi","metric":"candidate_accuracy","value":0.91,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.hi","metric":"candidate_ece","value":0.04933870881795882,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.hi","metric":"baseline_accuracy","value":0.95,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.hi","metric":"baseline_ece","value":0.029023234546184544,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.hu","metric":"regressed","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.hu","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.hu","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.hu","metric":"candidate_accuracy","value":0.9,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.hu","metric":"candidate_ece","value":0.07938083112239844,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.hu","metric":"baseline_accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.hu","metric":"baseline_ece","value":0.03867272794246674,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.hy","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.hy","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.hy","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.hy","metric":"candidate_accuracy","value":0.89,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.hy","metric":"candidate_ece","value":0.0601737295091152,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.hy","metric":"baseline_accuracy","value":0.84,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.hy","metric":"baseline_ece","value":0.07812121435999876,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.id","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.id","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.id","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.id","metric":"candidate_accuracy","value":0.96,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.id","metric":"candidate_ece","value":0.05166170924901959,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.id","metric":"baseline_accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.id","metric":"baseline_ece","value":0.05200909078121181,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.is","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.is","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.is","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.is","metric":"candidate_accuracy","value":0.85,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.is","metric":"candidate_ece","value":0.07128985196352003,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.is","metric":"baseline_accuracy","value":0.84,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.is","metric":"baseline_ece","value":0.07296262651681899,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.it","metric":"regressed","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.it","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.it","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.it","metric":"candidate_accuracy","value":0.91,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.it","metric":"candidate_ece","value":0.05676230788230893,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.it","metric":"baseline_accuracy","value":0.96,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.it","metric":"baseline_ece","value":0.05021818310022354,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ja","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ja","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ja","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ja","metric":"candidate_accuracy","value":0.97,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ja","metric":"candidate_ece","value":0.03706384837627408,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ja","metric":"baseline_accuracy","value":0.97,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ja","metric":"baseline_ece","value":0.05874444544315334,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.jv","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.jv","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.jv","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.jv","metric":"candidate_accuracy","value":0.89,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.jv","metric":"candidate_ece","value":0.07372990816831584,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.jv","metric":"baseline_accuracy","value":0.75,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.jv","metric":"baseline_ece","value":0.07757171720266345,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ka","metric":"regressed","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ka","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ka","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ka","metric":"candidate_accuracy","value":0.83,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ka","metric":"candidate_ece","value":0.07632239848375316,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ka","metric":"baseline_accuracy","value":0.88,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ka","metric":"baseline_ece","value":0.09805454522371292,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.km","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.km","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.km","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.km","metric":"candidate_accuracy","value":0.84,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.km","metric":"candidate_ece","value":0.07897142395377156,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.km","metric":"baseline_accuracy","value":0.81,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.km","metric":"baseline_ece","value":0.07332727491855619,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.kn","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.kn","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.kn","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.kn","metric":"candidate_accuracy","value":0.91,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.kn","metric":"candidate_ece","value":0.08909560441970826,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.kn","metric":"baseline_accuracy","value":0.87,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.kn","metric":"baseline_ece","value":0.062040405273437446,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ko","metric":"regressed","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ko","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ko","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ko","metric":"candidate_accuracy","value":0.91,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ko","metric":"candidate_ece","value":0.0583547832071781,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ko","metric":"baseline_accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ko","metric":"baseline_ece","value":0.04226868838071823,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.lv","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.lv","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.lv","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.lv","metric":"candidate_accuracy","value":0.89,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.lv","metric":"candidate_ece","value":0.08676462039351468,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.lv","metric":"baseline_accuracy","value":0.85,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.lv","metric":"baseline_ece","value":0.06903636515140534,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ml","metric":"regressed","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ml","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ml","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ml","metric":"candidate_accuracy","value":0.87,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ml","metric":"candidate_ece","value":0.08541164651513103,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ml","metric":"baseline_accuracy","value":0.91,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ml","metric":"baseline_ece","value":0.049515153467655115,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.mn","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.mn","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.mn","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.mn","metric":"candidate_accuracy","value":0.88,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.mn","metric":"candidate_ece","value":0.11159766197204596,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.mn","metric":"baseline_accuracy","value":0.84,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.mn","metric":"baseline_ece","value":0.08651313126087185,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ms","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ms","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ms","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ms","metric":"candidate_accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ms","metric":"candidate_ece","value":0.03943374574184415,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ms","metric":"baseline_accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ms","metric":"baseline_ece","value":0.06841818481683726,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.my","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.my","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.my","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.my","metric":"candidate_accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.my","metric":"candidate_ece","value":0.07324161291122437,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.my","metric":"baseline_accuracy","value":0.89,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.my","metric":"baseline_ece","value":0.07051919519901269,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.nb","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.nb","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.nb","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.nb","metric":"candidate_accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.nb","metric":"candidate_ece","value":0.037134091854095545,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.nb","metric":"baseline_accuracy","value":0.86,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.nb","metric":"baseline_ece","value":0.06732828453183172,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.nl","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.nl","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.nl","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.nl","metric":"candidate_accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.nl","metric":"candidate_ece","value":0.0496180704236031,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.nl","metric":"baseline_accuracy","value":0.9,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.nl","metric":"baseline_ece","value":0.055070708692073866,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.pl","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.pl","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.pl","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.pl","metric":"candidate_accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.pl","metric":"candidate_ece","value":0.052072535604238544,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.pl","metric":"baseline_accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.pl","metric":"baseline_ece","value":0.04303939431905744,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.pt","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.pt","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.pt","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.pt","metric":"candidate_accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.pt","metric":"candidate_ece","value":0.037603752911090804,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.pt","metric":"baseline_accuracy","value":0.9,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.pt","metric":"baseline_ece","value":0.09853636592626569,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ro","metric":"regressed","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ro","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ro","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ro","metric":"candidate_accuracy","value":0.91,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ro","metric":"candidate_ece","value":0.05261404126882551,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ro","metric":"baseline_accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ro","metric":"baseline_ece","value":0.08220606148242947,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ru","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ru","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ru","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ru","metric":"candidate_accuracy","value":0.97,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ru","metric":"candidate_ece","value":0.035205752253532396,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ru","metric":"baseline_accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ru","metric":"baseline_ece","value":0.0405292943120003,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sl","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.sl","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.sl","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.sl","metric":"candidate_accuracy","value":0.91,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sl","metric":"candidate_ece","value":0.05365944206714634,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sl","metric":"baseline_accuracy","value":0.89,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sl","metric":"baseline_ece","value":0.057861616015434277,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sq","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.sq","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.sq","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.sq","metric":"candidate_accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sq","metric":"candidate_ece","value":0.04722087457776066,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sq","metric":"baseline_accuracy","value":0.76,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sq","metric":"baseline_ece","value":0.0900494961440563,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sv","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.sv","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.sv","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.sv","metric":"candidate_accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sv","metric":"candidate_ece","value":0.04295731350779536,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sv","metric":"baseline_accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sv","metric":"baseline_ece","value":0.07373636573553077,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sw","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.sw","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.sw","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.sw","metric":"candidate_accuracy","value":0.84,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sw","metric":"candidate_ece","value":0.08623769655823708,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sw","metric":"baseline_accuracy","value":0.7,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sw","metric":"baseline_ece","value":0.11177171409130097,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ta","metric":"regressed","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ta","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ta","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ta","metric":"candidate_accuracy","value":0.88,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ta","metric":"candidate_ece","value":0.06233964949846268,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ta","metric":"baseline_accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ta","metric":"baseline_ece","value":0.09244343683123586,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.te","metric":"regressed","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.te","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.te","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.te","metric":"candidate_accuracy","value":0.87,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.te","metric":"candidate_ece","value":0.0855315700173378,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.te","metric":"baseline_accuracy","value":0.91,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.te","metric":"baseline_ece","value":0.04149899095296861,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.th","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.th","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.th","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.th","metric":"candidate_accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.th","metric":"candidate_ece","value":0.03923365086317059,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.th","metric":"baseline_accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.th","metric":"baseline_ece","value":0.04299999922513963,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.tl","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.tl","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.tl","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.tl","metric":"candidate_accuracy","value":0.91,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.tl","metric":"candidate_ece","value":0.06683538392186163,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.tl","metric":"baseline_accuracy","value":0.85,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.tl","metric":"baseline_ece","value":0.08445252478122714,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.tr","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.tr","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.tr","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.tr","metric":"candidate_accuracy","value":0.9,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.tr","metric":"candidate_ece","value":0.07519040256738663,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.tr","metric":"baseline_accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.tr","metric":"baseline_ece","value":0.0609899020195007,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ur","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ur","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ur","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ur","metric":"candidate_accuracy","value":0.9,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ur","metric":"candidate_ece","value":0.06569901391863815,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ur","metric":"baseline_accuracy","value":0.9,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ur","metric":"baseline_ece","value":0.06538687020540238,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.vi","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.vi","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.vi","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.vi","metric":"candidate_accuracy","value":0.91,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.vi","metric":"candidate_ece","value":0.0680993591248989,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.vi","metric":"baseline_accuracy","value":0.89,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.vi","metric":"baseline_ece","value":0.04620000123977666,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.zh-CN","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.zh-CN","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.zh-CN","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.zh-CN","metric":"candidate_accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.zh-CN","metric":"candidate_ece","value":0.05872932702302928,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.zh-CN","metric":"baseline_accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.zh-CN","metric":"baseline_ece","value":0.03629999935626985,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.zh-TW","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.zh-TW","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.zh-TW","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.zh-TW","metric":"candidate_accuracy","value":0.95,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.zh-TW","metric":"candidate_ece","value":0.032052750289440116,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.zh-TW","metric":"baseline_accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.zh-TW","metric":"baseline_ece","value":0.06207878708839416,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/clinc150","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/clinc150","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/clinc150","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/clinc150","metric":"candidate_accuracy","value":0.79,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/clinc150","metric":"candidate_ece","value":0.14865223526954646,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/sst5","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/sst5","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/sst5","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/sst5","metric":"candidate_accuracy","value":0.55,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/sst5","metric":"candidate_ece","value":0.06311321079730987,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.ar","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.ar","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.ar","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.ar","metric":"candidate_accuracy","value":0.67,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.ar","metric":"candidate_ece","value":0.15268935799598696,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.bg","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.bg","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.bg","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.bg","metric":"candidate_accuracy","value":0.78,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.bg","metric":"candidate_ece","value":0.16177826255559924,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.de","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.de","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.de","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.de","metric":"candidate_accuracy","value":0.82,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.de","metric":"candidate_ece","value":0.17352425545454023,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.el","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.el","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.el","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.el","metric":"candidate_accuracy","value":0.8,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.el","metric":"candidate_ece","value":0.1832862091064453,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.en","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.en","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.en","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.en","metric":"candidate_accuracy","value":0.74,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.en","metric":"candidate_ece","value":0.12187356531620024,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.es","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.es","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.es","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.es","metric":"candidate_accuracy","value":0.74,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.es","metric":"candidate_ece","value":0.1250601354241371,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.fr","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.fr","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.fr","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.fr","metric":"candidate_accuracy","value":0.81,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.fr","metric":"candidate_ece","value":0.15819115191698074,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.hi","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.hi","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.hi","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.hi","metric":"candidate_accuracy","value":0.66,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.hi","metric":"candidate_ece","value":0.09690238654613496,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.ru","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.ru","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.ru","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.ru","metric":"candidate_accuracy","value":0.73,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.ru","metric":"candidate_ece","value":0.16613810688257216,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.sw","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.sw","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.sw","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.sw","metric":"candidate_accuracy","value":0.6,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.sw","metric":"candidate_ece","value":0.0703927117586136,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.th","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.th","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.th","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.th","metric":"candidate_accuracy","value":0.67,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.th","metric":"candidate_ece","value":0.0998651620745659,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.tr","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.tr","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.tr","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.tr","metric":"candidate_accuracy","value":0.7,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.tr","metric":"candidate_ece","value":0.1450413915514946,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.ur","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.ur","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.ur","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.ur","metric":"candidate_accuracy","value":0.62,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.ur","metric":"candidate_ece","value":0.0880414754152298,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.vi","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.vi","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.vi","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.vi","metric":"candidate_accuracy","value":0.7,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.vi","metric":"candidate_ece","value":0.1799289581179619,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.zh","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.zh","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.zh","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.zh","metric":"candidate_accuracy","value":0.63,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.zh","metric":"candidate_ece","value":0.12515602439641954,"lo":null,"hi":null,"n":100,"of":null},{"category":"robustness/shuffle","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/shuffle","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/shuffle","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/shuffle","metric":"candidate_accuracy","value":0.8972709890491918,"lo":null,"hi":null,"n":5753,"of":null},{"category":"robustness/shuffle","metric":"candidate_ece","value":0.039063116139574534,"lo":null,"hi":null,"n":5753,"of":null},{"category":"robustness/alias","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/alias","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/alias","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/alias","metric":"candidate_accuracy","value":0.848531684698609,"lo":null,"hi":null,"n":647,"of":null},{"category":"robustness/alias","metric":"candidate_ece","value":0.059081201299080746,"lo":null,"hi":null,"n":647,"of":null},{"category":"robustness/swap","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/swap","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/swap","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/swap","metric":"candidate_accuracy","value":0.8469860896445132,"lo":null,"hi":null,"n":647,"of":null},{"category":"robustness/swap","metric":"candidate_ece","value":0.05197275265291265,"lo":null,"hi":null,"n":647,"of":null},{"category":"robustness/subset","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/subset","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/subset","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/subset","metric":"candidate_accuracy","value":0.9490566037735849,"lo":null,"hi":null,"n":5300,"of":null},{"category":"robustness/subset","metric":"candidate_ece","value":0.018249458639689256,"lo":null,"hi":null,"n":5300,"of":null},{"category":"robustness/near","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/near","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/near","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/near","metric":"candidate_accuracy","value":0.9233962264150943,"lo":null,"hi":null,"n":5300,"of":null},{"category":"robustness/near","metric":"candidate_ece","value":0.023286296534088497,"lo":null,"hi":null,"n":5300,"of":null}],"notes":"tolerance accuracy 0.02, ece 0.03; gate sha256:0f264b487e08bec3891c84e7cf934532a595c624ca8074a1cc132ac410252ba2","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"gate.jev-drawn-a8.0cfeef05-kai","benchmark":"gate.jev","split":"drawn","system":"kai","version":"a8@0cfeef05c879d7f24bbde275a6c23e85e80b6baf06cf9311d529010688d4a697","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"11474208ce49e6834bad21db4af8bb7ddf33b0ada6138b3661938522cafc815f","store_digest":"","facts_digest":"","commit":"a1a4a73d524cecf8038e30c408ad30b66e71d490","questions":null,"answered":null,"when":"","ended":"2026-09-28T16:33:11Z","by":"","completion":"unknown","measures":[{"category":"all","metric":"pass","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"suites","value":62,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"candidate","value":0.89321440208537,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"baseline","value":0.8669257086999022,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"delta","value":0.0262886933854678,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"regressions","value":14,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"cases","value":100,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/jev.ag_news","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/jev.ag_news","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/jev.ag_news","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/jev.ag_news","metric":"candidate_accuracy","value":0.96,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/jev.ag_news","metric":"candidate_ece","value":0.03911538153886794,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/jev.ag_news","metric":"baseline_accuracy","value":0.83,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/jev.ag_news","metric":"baseline_ece","value":0.12050000071525568,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/jev.emotion","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/jev.emotion","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/jev.emotion","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/jev.emotion","metric":"candidate_accuracy","value":0.95,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/jev.emotion","metric":"candidate_ece","value":0.042020905315876,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/jev.emotion","metric":"baseline_accuracy","value":0.64,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/jev.emotion","metric":"baseline_ece","value":0.24150000125169754,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/jev.banking77_full","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/jev.banking77_full","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/jev.banking77_full","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/jev.banking77_full","metric":"candidate_accuracy","value":0.89,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/jev.banking77_full","metric":"candidate_ece","value":0.09766138911247257,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/jev.banking77_full","metric":"baseline_accuracy","value":0.85,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/jev.banking77_full","metric":"baseline_ece","value":0.08564848810434346,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.support_triage","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.support_triage","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.support_triage","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.support_triage","metric":"candidate_accuracy","value":0.33,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.support_triage","metric":"candidate_ece","value":0.4952091199159622,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.support_triage","metric":"baseline_accuracy","value":0.29,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.support_triage","metric":"baseline_ece","value":0.561000002026558,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.email_spam","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.email_spam","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.email_spam","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.email_spam","metric":"candidate_accuracy","value":0.99,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.email_spam","metric":"candidate_ece","value":0.009769555330276458,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.email_spam","metric":"baseline_accuracy","value":0.96,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.email_spam","metric":"baseline_ece","value":0.07149999797344209,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.phishing","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.phishing","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.phishing","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.phishing","metric":"candidate_accuracy","value":1,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.phishing","metric":"candidate_ece","value":0.0000613886117935003,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.phishing","metric":"baseline_accuracy","value":0.89,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.phishing","metric":"baseline_ece","value":0.08049999296665189,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.guardrails_jailbreak","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.guardrails_jailbreak","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.guardrails_jailbreak","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.guardrails_jailbreak","metric":"candidate_accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.guardrails_jailbreak","metric":"candidate_ece","value":0.05118291318416589,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.guardrails_jailbreak","metric":"baseline_accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.guardrails_jailbreak","metric":"baseline_ece","value":0.0655999952554703,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.moderation_toxicity","metric":"regressed","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.moderation_toxicity","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.moderation_toxicity","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.moderation_toxicity","metric":"candidate_accuracy","value":0.81,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.moderation_toxicity","metric":"candidate_ece","value":0.16926764965057373,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.moderation_toxicity","metric":"baseline_accuracy","value":0.73,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.moderation_toxicity","metric":"baseline_ece","value":0.1372000062465668,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.rag_relevance","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.rag_relevance","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.rag_relevance","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.rag_relevance","metric":"candidate_accuracy","value":0.57,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.rag_relevance","metric":"candidate_ece","value":0.12333745658397677,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.rag_relevance","metric":"baseline_accuracy","value":0.51,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.rag_relevance","metric":"baseline_ece","value":0.39620000362396246,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.model_routing_domain","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.model_routing_domain","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.model_routing_domain","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.model_routing_domain","metric":"candidate_accuracy","value":1,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.model_routing_domain","metric":"candidate_ece","value":0.003844951391220098,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.model_routing_domain","metric":"baseline_accuracy","value":0.98,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.model_routing_domain","metric":"baseline_ece","value":0.026600003242492693,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/typed_decisions","metric":"regressed","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/typed_decisions","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/typed_decisions","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/typed_decisions","metric":"candidate_accuracy","value":0.75,"lo":null,"hi":null,"n":500,"of":null},{"category":"harness/typed_decisions","metric":"candidate_ece","value":0.1543838354349136,"lo":null,"hi":null,"n":500,"of":null},{"category":"harness/typed_decisions","metric":"baseline_accuracy","value":0.73,"lo":null,"hi":null,"n":500,"of":null},{"category":"harness/typed_decisions","metric":"baseline_ece","value":0.07618141478300096,"lo":null,"hi":null,"n":500,"of":null},{"category":"harness/massive.af","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.af","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.af","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.af","metric":"candidate_accuracy","value":0.89,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.af","metric":"candidate_ece","value":0.05268498986959454,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.af","metric":"baseline_accuracy","value":0.87,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.af","metric":"baseline_ece","value":0.08726868644356728,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.am","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.am","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.am","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.am","metric":"candidate_accuracy","value":0.85,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.am","metric":"candidate_ece","value":0.11598066598176962,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.am","metric":"baseline_accuracy","value":0.85,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.am","metric":"baseline_ece","value":0.09598384097218512,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ar","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ar","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ar","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ar","metric":"candidate_accuracy","value":0.89,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ar","metric":"candidate_ece","value":0.07288791090250017,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ar","metric":"baseline_accuracy","value":0.87,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ar","metric":"baseline_ece","value":0.08222424164414406,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.az","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.az","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.az","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.az","metric":"candidate_accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.az","metric":"candidate_ece","value":0.06024763107299812,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.az","metric":"baseline_accuracy","value":0.88,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.az","metric":"baseline_ece","value":0.04820606261491777,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.bn","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.bn","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.bn","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.bn","metric":"candidate_accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.bn","metric":"candidate_ece","value":0.05731592342257498,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.bn","metric":"baseline_accuracy","value":0.86,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.bn","metric":"baseline_ece","value":0.0710626265406609,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.cy","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.cy","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.cy","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.cy","metric":"candidate_accuracy","value":0.87,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.cy","metric":"candidate_ece","value":0.08664740681648261,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.cy","metric":"baseline_accuracy","value":0.76,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.cy","metric":"baseline_ece","value":0.09916767939925195,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.da","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.da","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.da","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.da","metric":"candidate_accuracy","value":0.91,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.da","metric":"candidate_ece","value":0.053155619204044345,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.da","metric":"baseline_accuracy","value":0.88,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.da","metric":"baseline_ece","value":0.038734343349933635,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.de","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.de","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.de","metric":"unanswered","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.de","metric":"candidate_accuracy","value":0.9292929292929293,"lo":null,"hi":null,"n":99,"of":null},{"category":"harness/massive.de","metric":"candidate_ece","value":0.05187850498189824,"lo":null,"hi":null,"n":99,"of":null},{"category":"harness/massive.de","metric":"baseline_accuracy","value":0.9393939393939394,"lo":null,"hi":null,"n":99,"of":null},{"category":"harness/massive.de","metric":"baseline_ece","value":0.057024792589322484,"lo":null,"hi":null,"n":99,"of":null},{"category":"harness/massive.el","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.el","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.el","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.el","metric":"candidate_accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.el","metric":"candidate_ece","value":0.0822948729991913,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.el","metric":"baseline_accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.el","metric":"baseline_ece","value":0.057093940079212174,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.en","metric":"regressed","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.en","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.en","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.en","metric":"candidate_accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.en","metric":"candidate_ece","value":0.05915291845798494,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.en","metric":"baseline_accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.en","metric":"baseline_ece","value":0.02530000150203709,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.es","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.es","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.es","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.es","metric":"candidate_accuracy","value":0.91,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.es","metric":"candidate_ece","value":0.06837773680686952,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.es","metric":"baseline_accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.es","metric":"baseline_ece","value":0.06282828330993653,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.fa","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.fa","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.fa","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.fa","metric":"candidate_accuracy","value":0.95,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.fa","metric":"candidate_ece","value":0.04708155661821374,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.fa","metric":"baseline_accuracy","value":0.95,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.fa","metric":"baseline_ece","value":0.04546464592218401,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.fi","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.fi","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.fi","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.fi","metric":"candidate_accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.fi","metric":"candidate_ece","value":0.056733679771423406,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.fi","metric":"baseline_accuracy","value":0.89,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.fi","metric":"baseline_ece","value":0.0383111131191253,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.fr","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.fr","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.fr","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.fr","metric":"candidate_accuracy","value":0.9,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.fr","metric":"candidate_ece","value":0.07309922486543659,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.fr","metric":"baseline_accuracy","value":0.9,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.fr","metric":"baseline_ece","value":0.0651949521899224,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.he","metric":"regressed","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.he","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.he","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.he","metric":"candidate_accuracy","value":0.91,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.he","metric":"candidate_ece","value":0.07781283855438227,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.he","metric":"baseline_accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.he","metric":"baseline_ece","value":0.042724243700504254,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.hi","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.hi","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.hi","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.hi","metric":"candidate_accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.hi","metric":"candidate_ece","value":0.043011419475078645,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.hi","metric":"baseline_accuracy","value":0.95,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.hi","metric":"baseline_ece","value":0.029023234546184544,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.hu","metric":"regressed","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.hu","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.hu","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.hu","metric":"candidate_accuracy","value":0.91,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.hu","metric":"candidate_ece","value":0.08298780947923662,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.hu","metric":"baseline_accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.hu","metric":"baseline_ece","value":0.03867272794246674,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.hy","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.hy","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.hy","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.hy","metric":"candidate_accuracy","value":0.89,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.hy","metric":"candidate_ece","value":0.05867982119321828,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.hy","metric":"baseline_accuracy","value":0.84,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.hy","metric":"baseline_ece","value":0.07812121435999876,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.id","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.id","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.id","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.id","metric":"candidate_accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.id","metric":"candidate_ece","value":0.04446434766054152,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.id","metric":"baseline_accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.id","metric":"baseline_ece","value":0.05200909078121181,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.is","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.is","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.is","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.is","metric":"candidate_accuracy","value":0.88,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.is","metric":"candidate_ece","value":0.08488803684711461,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.is","metric":"baseline_accuracy","value":0.84,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.is","metric":"baseline_ece","value":0.07296262651681899,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.it","metric":"regressed","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.it","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.it","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.it","metric":"candidate_accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.it","metric":"candidate_ece","value":0.05702402949333189,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.it","metric":"baseline_accuracy","value":0.96,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.it","metric":"baseline_ece","value":0.05021818310022354,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ja","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ja","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ja","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ja","metric":"candidate_accuracy","value":0.96,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ja","metric":"candidate_ece","value":0.034759365916252095,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ja","metric":"baseline_accuracy","value":0.97,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ja","metric":"baseline_ece","value":0.05874444544315334,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.jv","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.jv","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.jv","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.jv","metric":"candidate_accuracy","value":0.88,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.jv","metric":"candidate_ece","value":0.06856640487909312,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.jv","metric":"baseline_accuracy","value":0.75,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.jv","metric":"baseline_ece","value":0.07757171720266345,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ka","metric":"regressed","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ka","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ka","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ka","metric":"candidate_accuracy","value":0.8,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ka","metric":"candidate_ece","value":0.1308203849196434,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ka","metric":"baseline_accuracy","value":0.88,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ka","metric":"baseline_ece","value":0.09805454522371292,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.km","metric":"regressed","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.km","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.km","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.km","metric":"candidate_accuracy","value":0.82,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.km","metric":"candidate_ece","value":0.14559533908963204,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.km","metric":"baseline_accuracy","value":0.81,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.km","metric":"baseline_ece","value":0.07332727491855619,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.kn","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.kn","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.kn","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.kn","metric":"candidate_accuracy","value":0.9,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.kn","metric":"candidate_ece","value":0.07564685180783273,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.kn","metric":"baseline_accuracy","value":0.87,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.kn","metric":"baseline_ece","value":0.062040405273437446,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ko","metric":"regressed","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ko","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ko","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ko","metric":"candidate_accuracy","value":0.9,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ko","metric":"candidate_ece","value":0.07660935476422309,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ko","metric":"baseline_accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ko","metric":"baseline_ece","value":0.04226868838071823,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.lv","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.lv","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.lv","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.lv","metric":"candidate_accuracy","value":0.89,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.lv","metric":"candidate_ece","value":0.09463837064802645,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.lv","metric":"baseline_accuracy","value":0.85,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.lv","metric":"baseline_ece","value":0.06903636515140534,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ml","metric":"regressed","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ml","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ml","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ml","metric":"candidate_accuracy","value":0.88,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ml","metric":"candidate_ece","value":0.07211435198783879,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ml","metric":"baseline_accuracy","value":0.91,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ml","metric":"baseline_ece","value":0.049515153467655115,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.mn","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.mn","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.mn","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.mn","metric":"candidate_accuracy","value":0.88,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.mn","metric":"candidate_ece","value":0.09503076672554014,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.mn","metric":"baseline_accuracy","value":0.84,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.mn","metric":"baseline_ece","value":0.08651313126087185,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ms","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ms","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ms","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ms","metric":"candidate_accuracy","value":0.9,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ms","metric":"candidate_ece","value":0.07233261793851858,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ms","metric":"baseline_accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ms","metric":"baseline_ece","value":0.06841818481683726,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.my","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.my","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.my","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.my","metric":"candidate_accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.my","metric":"candidate_ece","value":0.059497409313917074,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.my","metric":"baseline_accuracy","value":0.89,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.my","metric":"baseline_ece","value":0.07051919519901269,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.nb","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.nb","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.nb","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.nb","metric":"candidate_accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.nb","metric":"candidate_ece","value":0.05309729158878326,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.nb","metric":"baseline_accuracy","value":0.86,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.nb","metric":"baseline_ece","value":0.06732828453183172,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.nl","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.nl","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.nl","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.nl","metric":"candidate_accuracy","value":0.88,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.nl","metric":"candidate_ece","value":0.08047885030508038,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.nl","metric":"baseline_accuracy","value":0.9,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.nl","metric":"baseline_ece","value":0.055070708692073866,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.pl","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.pl","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.pl","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.pl","metric":"candidate_accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.pl","metric":"candidate_ece","value":0.04532576993107795,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.pl","metric":"baseline_accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.pl","metric":"baseline_ece","value":0.04303939431905744,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.pt","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.pt","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.pt","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.pt","metric":"candidate_accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.pt","metric":"candidate_ece","value":0.04678081497550015,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.pt","metric":"baseline_accuracy","value":0.9,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.pt","metric":"baseline_ece","value":0.09853636592626569,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ro","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ro","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ro","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ro","metric":"candidate_accuracy","value":0.96,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ro","metric":"candidate_ece","value":0.03957485139369964,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ro","metric":"baseline_accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ro","metric":"baseline_ece","value":0.08220606148242947,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ru","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ru","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ru","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ru","metric":"candidate_accuracy","value":0.95,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ru","metric":"candidate_ece","value":0.04580723613500593,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ru","metric":"baseline_accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ru","metric":"baseline_ece","value":0.0405292943120003,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sl","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.sl","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.sl","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.sl","metric":"candidate_accuracy","value":0.88,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sl","metric":"candidate_ece","value":0.06133252531290058,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sl","metric":"baseline_accuracy","value":0.89,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sl","metric":"baseline_ece","value":0.057861616015434277,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sq","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.sq","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.sq","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.sq","metric":"candidate_accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sq","metric":"candidate_ece","value":0.054145977795124055,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sq","metric":"baseline_accuracy","value":0.76,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sq","metric":"baseline_ece","value":0.0900494961440563,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sv","metric":"regressed","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.sv","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.sv","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.sv","metric":"candidate_accuracy","value":0.91,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sv","metric":"candidate_ece","value":0.06203487917780875,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sv","metric":"baseline_accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sv","metric":"baseline_ece","value":0.07373636573553077,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sw","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.sw","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.sw","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.sw","metric":"candidate_accuracy","value":0.85,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sw","metric":"candidate_ece","value":0.0911237014085054,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sw","metric":"baseline_accuracy","value":0.7,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sw","metric":"baseline_ece","value":0.11177171409130097,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ta","metric":"regressed","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ta","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ta","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ta","metric":"candidate_accuracy","value":0.89,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ta","metric":"candidate_ece","value":0.06441951379179953,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ta","metric":"baseline_accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ta","metric":"baseline_ece","value":0.09244343683123586,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.te","metric":"regressed","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.te","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.te","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.te","metric":"candidate_accuracy","value":0.87,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.te","metric":"candidate_ece","value":0.08435814142227176,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.te","metric":"baseline_accuracy","value":0.91,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.te","metric":"baseline_ece","value":0.04149899095296861,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.th","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.th","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.th","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.th","metric":"candidate_accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.th","metric":"candidate_ece","value":0.044177895784378036,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.th","metric":"baseline_accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.th","metric":"baseline_ece","value":0.04299999922513963,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.tl","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.tl","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.tl","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.tl","metric":"candidate_accuracy","value":0.88,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.tl","metric":"candidate_ece","value":0.08951716601848603,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.tl","metric":"baseline_accuracy","value":0.85,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.tl","metric":"baseline_ece","value":0.08445252478122714,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.tr","metric":"regressed","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.tr","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.tr","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.tr","metric":"candidate_accuracy","value":0.87,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.tr","metric":"candidate_ece","value":0.10064284861087798,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.tr","metric":"baseline_accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.tr","metric":"baseline_ece","value":0.0609899020195007,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ur","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ur","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ur","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ur","metric":"candidate_accuracy","value":0.91,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ur","metric":"candidate_ece","value":0.06861960574984545,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ur","metric":"baseline_accuracy","value":0.9,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ur","metric":"baseline_ece","value":0.06538687020540238,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.vi","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.vi","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.vi","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.vi","metric":"candidate_accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.vi","metric":"candidate_ece","value":0.04812568008899687,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.vi","metric":"baseline_accuracy","value":0.89,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.vi","metric":"baseline_ece","value":0.04620000123977666,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.zh-CN","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.zh-CN","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.zh-CN","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.zh-CN","metric":"candidate_accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.zh-CN","metric":"candidate_ece","value":0.05050742074847214,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.zh-CN","metric":"baseline_accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.zh-CN","metric":"baseline_ece","value":0.03629999935626985,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.zh-TW","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.zh-TW","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.zh-TW","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.zh-TW","metric":"candidate_accuracy","value":0.96,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.zh-TW","metric":"candidate_ece","value":0.03419531375169753,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.zh-TW","metric":"baseline_accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.zh-TW","metric":"baseline_ece","value":0.06207878708839416,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/clinc150","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/clinc150","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/clinc150","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/clinc150","metric":"candidate_accuracy","value":0.78,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/clinc150","metric":"candidate_ece","value":0.15651181817054743,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/sst5","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/sst5","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/sst5","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/sst5","metric":"candidate_accuracy","value":0.5,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/sst5","metric":"candidate_ece","value":0.1442073956131935,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.ar","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.ar","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.ar","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.ar","metric":"candidate_accuracy","value":0.62,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.ar","metric":"candidate_ece","value":0.13826530635356904,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.bg","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.bg","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.bg","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.bg","metric":"candidate_accuracy","value":0.76,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.bg","metric":"candidate_ece","value":0.12432977795600889,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.de","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.de","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.de","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.de","metric":"candidate_accuracy","value":0.76,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.de","metric":"candidate_ece","value":0.12723101794719702,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.el","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.el","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.el","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.el","metric":"candidate_accuracy","value":0.73,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.el","metric":"candidate_ece","value":0.11346075385808947,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.en","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.en","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.en","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.en","metric":"candidate_accuracy","value":0.76,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.en","metric":"candidate_ece","value":0.12963819324970244,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.es","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.es","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.es","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.es","metric":"candidate_accuracy","value":0.77,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.es","metric":"candidate_ece","value":0.15921563208103184,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.fr","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.fr","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.fr","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.fr","metric":"candidate_accuracy","value":0.79,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.fr","metric":"candidate_ece","value":0.14418751090764997,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.hi","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.hi","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.hi","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.hi","metric":"candidate_accuracy","value":0.59,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.hi","metric":"candidate_ece","value":0.05843946486711503,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.ru","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.ru","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.ru","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.ru","metric":"candidate_accuracy","value":0.72,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.ru","metric":"candidate_ece","value":0.11137971103191374,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.sw","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.sw","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.sw","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.sw","metric":"candidate_accuracy","value":0.58,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.sw","metric":"candidate_ece","value":0.12170465260744094,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.th","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.th","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.th","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.th","metric":"candidate_accuracy","value":0.61,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.th","metric":"candidate_ece","value":0.1322586926817894,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.tr","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.tr","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.tr","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.tr","metric":"candidate_accuracy","value":0.72,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.tr","metric":"candidate_ece","value":0.12287913411855698,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.ur","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.ur","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.ur","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.ur","metric":"candidate_accuracy","value":0.5,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.ur","metric":"candidate_ece","value":0.1103538781404495,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.vi","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.vi","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.vi","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.vi","metric":"candidate_accuracy","value":0.71,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.vi","metric":"candidate_ece","value":0.11963065892457966,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.zh","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.zh","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.zh","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.zh","metric":"candidate_accuracy","value":0.67,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.zh","metric":"candidate_ece","value":0.10528100371360781,"lo":null,"hi":null,"n":100,"of":null},{"category":"robustness/shuffle","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/shuffle","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/shuffle","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/shuffle","metric":"candidate_accuracy","value":0.8932730749174344,"lo":null,"hi":null,"n":5753,"of":null},{"category":"robustness/shuffle","metric":"candidate_ece","value":0.04835260770055868,"lo":null,"hi":null,"n":5753,"of":null},{"category":"robustness/alias","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/alias","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/alias","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/alias","metric":"candidate_accuracy","value":0.848531684698609,"lo":null,"hi":null,"n":647,"of":null},{"category":"robustness/alias","metric":"candidate_ece","value":0.04920363361721247,"lo":null,"hi":null,"n":647,"of":null},{"category":"robustness/swap","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/swap","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/swap","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/swap","metric":"candidate_accuracy","value":0.8500772797527048,"lo":null,"hi":null,"n":647,"of":null},{"category":"robustness/swap","metric":"candidate_ece","value":0.05356464936891703,"lo":null,"hi":null,"n":647,"of":null},{"category":"robustness/subset","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/subset","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/subset","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/subset","metric":"candidate_accuracy","value":0.9507547169811321,"lo":null,"hi":null,"n":5300,"of":null},{"category":"robustness/subset","metric":"candidate_ece","value":0.018969453463576848,"lo":null,"hi":null,"n":5300,"of":null},{"category":"robustness/near","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/near","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/near","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/near","metric":"candidate_accuracy","value":0.9222641509433962,"lo":null,"hi":null,"n":5300,"of":null},{"category":"robustness/near","metric":"candidate_ece","value":0.029289239268257973,"lo":null,"hi":null,"n":5300,"of":null}],"notes":"tolerance accuracy 0.02, ece 0.03; gate sha256:866b219e0c9e383173d0a82a031b669933c58dc69183ca974987aea7f7be9f3e","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"gate.laya-drawn-a5.df1c16a2-kai","benchmark":"gate.laya","split":"drawn","system":"kai","version":"a5@df1c16a2fef7ec93f2b34a6a194f6dc3c2eee70e936473b802151543c5d46e83","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"11474208ce49e6834bad21db4af8bb7ddf33b0ada6138b3661938522cafc815f","store_digest":"","facts_digest":"","commit":"611059b60f64ab686d6475cd76778b3fac52cd98","questions":null,"answered":null,"when":"","ended":"2026-09-27T21:21:10Z","by":"","completion":"unknown","measures":[{"category":"all","metric":"pass","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"suites","value":62,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"candidate","value":0.8786451612903229,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"baseline","value":0.43848387096774183,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"delta","value":0.44016129032258106,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"regressions","value":3,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"cases","value":100,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/jev.ag_news","metric":"regressed","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/jev.ag_news","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/jev.ag_news","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/jev.ag_news","metric":"candidate_accuracy","value":0.97,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/jev.ag_news","metric":"candidate_ece","value":0.09678747147321702,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/jev.ag_news","metric":"baseline_accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/jev.ag_news","metric":"baseline_ece","value":0.04963111966848367,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/jev.emotion","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/jev.emotion","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/jev.emotion","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/jev.emotion","metric":"candidate_accuracy","value":0.9,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/jev.emotion","metric":"candidate_ece","value":0.07829443186521529,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/jev.emotion","metric":"baseline_accuracy","value":0.63,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/jev.emotion","metric":"baseline_ece","value":0.29464753001928334,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/jev.banking77_full","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/jev.banking77_full","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/jev.banking77_full","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/jev.banking77_full","metric":"candidate_accuracy","value":0.89,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/jev.banking77_full","metric":"candidate_ece","value":0.1010316482186318,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/jev.banking77_full","metric":"baseline_accuracy","value":0.42,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/jev.banking77_full","metric":"baseline_ece","value":0.39392668925225743,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.support_triage","metric":"regressed","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.support_triage","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.support_triage","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.support_triage","metric":"candidate_accuracy","value":0.39,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.support_triage","metric":"candidate_ece","value":0.21937582075595857,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.support_triage","metric":"baseline_accuracy","value":0.48,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.support_triage","metric":"baseline_ece","value":0.11383459806442263,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.email_spam","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.email_spam","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.email_spam","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.email_spam","metric":"candidate_accuracy","value":1,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.email_spam","metric":"candidate_ece","value":0.00008834540843960514,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.email_spam","metric":"baseline_accuracy","value":1,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.email_spam","metric":"baseline_ece","value":0.0038367044925689377,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.phishing","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.phishing","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.phishing","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.phishing","metric":"candidate_accuracy","value":1,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.phishing","metric":"candidate_ece","value":0.0000836509466171087,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.phishing","metric":"baseline_accuracy","value":1,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.phishing","metric":"baseline_ece","value":0.010809252858161901,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.guardrails_jailbreak","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.guardrails_jailbreak","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.guardrails_jailbreak","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.guardrails_jailbreak","metric":"candidate_accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.guardrails_jailbreak","metric":"candidate_ece","value":0.055477619767189054,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.guardrails_jailbreak","metric":"baseline_accuracy","value":0.7,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.guardrails_jailbreak","metric":"baseline_ece","value":0.2747894221544265,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.moderation_toxicity","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.moderation_toxicity","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.moderation_toxicity","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.moderation_toxicity","metric":"candidate_accuracy","value":0.83,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.moderation_toxicity","metric":"candidate_ece","value":0.1615606594085694,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.moderation_toxicity","metric":"baseline_accuracy","value":0.6,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.moderation_toxicity","metric":"baseline_ece","value":0.2243971598148346,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.rag_relevance","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.rag_relevance","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.rag_relevance","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.rag_relevance","metric":"candidate_accuracy","value":0.58,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.rag_relevance","metric":"candidate_ece","value":0.12734931707382205,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.rag_relevance","metric":"baseline_accuracy","value":0.48,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.rag_relevance","metric":"baseline_ece","value":0.26150231361389165,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.model_routing_domain","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.model_routing_domain","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.model_routing_domain","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.model_routing_domain","metric":"candidate_accuracy","value":1,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.model_routing_domain","metric":"candidate_ece","value":0.0018489766120910467,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.model_routing_domain","metric":"baseline_accuracy","value":0.7,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.model_routing_domain","metric":"baseline_ece","value":0.09083123058080676,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/typed_decisions","metric":"regressed","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/typed_decisions","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/typed_decisions","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/typed_decisions","metric":"candidate_accuracy","value":0.696,"lo":null,"hi":null,"n":500,"of":null},{"category":"harness/typed_decisions","metric":"candidate_ece","value":0.1625631979107857,"lo":null,"hi":null,"n":500,"of":null},{"category":"harness/typed_decisions","metric":"baseline_accuracy","value":0.746,"lo":null,"hi":null,"n":500,"of":null},{"category":"harness/typed_decisions","metric":"baseline_ece","value":0.19970310807228087,"lo":null,"hi":null,"n":500,"of":null},{"category":"harness/massive.af","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.af","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.af","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.af","metric":"candidate_accuracy","value":0.89,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.af","metric":"candidate_ece","value":0.05952447265386586,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.af","metric":"baseline_accuracy","value":0.34,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.af","metric":"baseline_ece","value":0.48447977453470237,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.am","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.am","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.am","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.am","metric":"candidate_accuracy","value":0.84,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.am","metric":"candidate_ece","value":0.06833967849612237,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.am","metric":"baseline_accuracy","value":0.15,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.am","metric":"baseline_ece","value":0.48387789905071266,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ar","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ar","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ar","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ar","metric":"candidate_accuracy","value":0.88,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ar","metric":"candidate_ece","value":0.08501704618334774,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ar","metric":"baseline_accuracy","value":0.46,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ar","metric":"baseline_ece","value":0.3150437718629837,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.az","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.az","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.az","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.az","metric":"candidate_accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.az","metric":"candidate_ece","value":0.06093618080019949,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.az","metric":"baseline_accuracy","value":0.36,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.az","metric":"baseline_ece","value":0.39286278426647187,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.bn","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.bn","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.bn","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.bn","metric":"candidate_accuracy","value":0.87,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.bn","metric":"candidate_ece","value":0.0521162086725235,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.bn","metric":"baseline_accuracy","value":0.45,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.bn","metric":"baseline_ece","value":0.3539693360030651,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.cy","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.cy","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.cy","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.cy","metric":"candidate_accuracy","value":0.82,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.cy","metric":"candidate_ece","value":0.06791832819581037,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.cy","metric":"baseline_accuracy","value":0.16,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.cy","metric":"baseline_ece","value":0.6107046373188496,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.da","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.da","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.da","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.da","metric":"candidate_accuracy","value":0.91,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.da","metric":"candidate_ece","value":0.04912485331296921,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.da","metric":"baseline_accuracy","value":0.46,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.da","metric":"baseline_ece","value":0.45181164443492894,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.de","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.de","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.de","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.de","metric":"candidate_accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.de","metric":"candidate_ece","value":0.0687420403957367,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.de","metric":"baseline_accuracy","value":0.53,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.de","metric":"baseline_ece","value":0.30368388086557385,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.el","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.el","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.el","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.el","metric":"candidate_accuracy","value":0.9,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.el","metric":"candidate_ece","value":0.05367249518632891,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.el","metric":"baseline_accuracy","value":0.44,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.el","metric":"baseline_ece","value":0.4296881093084812,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.en","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.en","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.en","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.en","metric":"candidate_accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.en","metric":"candidate_ece","value":0.05163752526044845,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.en","metric":"baseline_accuracy","value":0.82,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.en","metric":"baseline_ece","value":0.13824885368347167,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.es","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.es","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.es","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.es","metric":"candidate_accuracy","value":0.9,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.es","metric":"candidate_ece","value":0.04325293913483622,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.es","metric":"baseline_accuracy","value":0.53,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.es","metric":"baseline_ece","value":0.34762571096420286,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.fa","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.fa","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.fa","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.fa","metric":"candidate_accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.fa","metric":"candidate_ece","value":0.05692772075533863,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.fa","metric":"baseline_accuracy","value":0.51,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.fa","metric":"baseline_ece","value":0.2985081049799919,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.fi","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.fi","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.fi","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.fi","metric":"candidate_accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.fi","metric":"candidate_ece","value":0.060005676448345244,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.fi","metric":"baseline_accuracy","value":0.3,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.fi","metric":"baseline_ece","value":0.4618027746677399,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.fr","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.fr","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.fr","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.fr","metric":"candidate_accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.fr","metric":"candidate_ece","value":0.05017257452011109,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.fr","metric":"baseline_accuracy","value":0.6,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.fr","metric":"baseline_ece","value":0.26832883805036545,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.he","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.he","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.he","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.he","metric":"candidate_accuracy","value":0.87,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.he","metric":"candidate_ece","value":0.07292630210518837,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.he","metric":"baseline_accuracy","value":0.37,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.he","metric":"baseline_ece","value":0.4185565242916346,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.hi","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.hi","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.hi","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.hi","metric":"candidate_accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.hi","metric":"candidate_ece","value":0.07353801846504208,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.hi","metric":"baseline_accuracy","value":0.46,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.hi","metric":"baseline_ece","value":0.36848925307393077,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.hu","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.hu","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.hu","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.hu","metric":"candidate_accuracy","value":0.87,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.hu","metric":"candidate_ece","value":0.050389538556337356,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.hu","metric":"baseline_accuracy","value":0.34,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.hu","metric":"baseline_ece","value":0.4373521935939789,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.hy","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.hy","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.hy","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.hy","metric":"candidate_accuracy","value":0.84,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.hy","metric":"candidate_ece","value":0.057336805462837225,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.hy","metric":"baseline_accuracy","value":0.25,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.hy","metric":"baseline_ece","value":0.5127132028341294,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.id","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.id","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.id","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.id","metric":"candidate_accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.id","metric":"candidate_ece","value":0.05904934585094447,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.id","metric":"baseline_accuracy","value":0.36,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.id","metric":"baseline_ece","value":0.48560510724782946,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.is","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.is","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.is","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.is","metric":"candidate_accuracy","value":0.86,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.is","metric":"candidate_ece","value":0.05912843495607373,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.is","metric":"baseline_accuracy","value":0.33,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.is","metric":"baseline_ece","value":0.48597560346126556,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.it","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.it","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.it","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.it","metric":"candidate_accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.it","metric":"candidate_ece","value":0.050549613535404186,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.it","metric":"baseline_accuracy","value":0.47,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.it","metric":"baseline_ece","value":0.38000325053930284,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ja","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ja","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ja","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ja","metric":"candidate_accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ja","metric":"candidate_ece","value":0.06279848605394359,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ja","metric":"baseline_accuracy","value":0.64,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ja","metric":"baseline_ece","value":0.23371360868215557,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.jv","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.jv","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.jv","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.jv","metric":"candidate_accuracy","value":0.83,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.jv","metric":"candidate_ece","value":0.10250622883439062,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.jv","metric":"baseline_accuracy","value":0.16,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.jv","metric":"baseline_ece","value":0.6633555415272713,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ka","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ka","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ka","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ka","metric":"candidate_accuracy","value":0.79,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ka","metric":"candidate_ece","value":0.10097189977765086,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ka","metric":"baseline_accuracy","value":0.15,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ka","metric":"baseline_ece","value":0.5422389303892852,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.km","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.km","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.km","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.km","metric":"candidate_accuracy","value":0.8,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.km","metric":"candidate_ece","value":0.09994862869381903,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.km","metric":"baseline_accuracy","value":0.2,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.km","metric":"baseline_ece","value":0.44535648457705973,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.kn","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.kn","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.kn","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.kn","metric":"candidate_accuracy","value":0.87,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.kn","metric":"candidate_ece","value":0.05703154116868977,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.kn","metric":"baseline_accuracy","value":0.3,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.kn","metric":"baseline_ece","value":0.3880462409555912,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ko","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ko","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ko","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ko","metric":"candidate_accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ko","metric":"candidate_ece","value":0.06531809881329537,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ko","metric":"baseline_accuracy","value":0.47,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ko","metric":"baseline_ece","value":0.31081631831824785,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.lv","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.lv","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.lv","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.lv","metric":"candidate_accuracy","value":0.85,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.lv","metric":"candidate_ece","value":0.07480210557579993,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.lv","metric":"baseline_accuracy","value":0.3,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.lv","metric":"baseline_ece","value":0.4859770181775093,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ml","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ml","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ml","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ml","metric":"candidate_accuracy","value":0.82,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ml","metric":"candidate_ece","value":0.06367021948099139,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ml","metric":"baseline_accuracy","value":0.28,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ml","metric":"baseline_ece","value":0.4433153660595417,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.mn","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.mn","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.mn","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.mn","metric":"candidate_accuracy","value":0.88,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.mn","metric":"candidate_ece","value":0.10917697399854666,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.mn","metric":"baseline_accuracy","value":0.16,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.mn","metric":"baseline_ece","value":0.5769782765209674,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ms","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ms","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ms","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ms","metric":"candidate_accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ms","metric":"candidate_ece","value":0.05931672349572179,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ms","metric":"baseline_accuracy","value":0.27,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ms","metric":"baseline_ece","value":0.5063951462507248,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.my","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.my","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.my","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.my","metric":"candidate_accuracy","value":0.9,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.my","metric":"candidate_ece","value":0.06698188766837115,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.my","metric":"baseline_accuracy","value":0.16,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.my","metric":"baseline_ece","value":0.4766131702065468,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.nb","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.nb","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.nb","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.nb","metric":"candidate_accuracy","value":0.91,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.nb","metric":"candidate_ece","value":0.06739467456936835,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.nb","metric":"baseline_accuracy","value":0.5,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.nb","metric":"baseline_ece","value":0.36785355895757677,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.nl","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.nl","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.nl","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.nl","metric":"candidate_accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.nl","metric":"candidate_ece","value":0.06967766851186752,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.nl","metric":"baseline_accuracy","value":0.4,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.nl","metric":"baseline_ece","value":0.479215635061264,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.pl","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.pl","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.pl","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.pl","metric":"candidate_accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.pl","metric":"candidate_ece","value":0.06593496680259701,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.pl","metric":"baseline_accuracy","value":0.47,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.pl","metric":"baseline_ece","value":0.37366658061742786,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.pt","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.pt","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.pt","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.pt","metric":"candidate_accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.pt","metric":"candidate_ece","value":0.048627837598323856,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.pt","metric":"baseline_accuracy","value":0.44,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.pt","metric":"baseline_ece","value":0.3956287080049515,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ro","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ro","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ro","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ro","metric":"candidate_accuracy","value":0.9,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ro","metric":"candidate_ece","value":0.05783455386757851,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ro","metric":"baseline_accuracy","value":0.37,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ro","metric":"baseline_ece","value":0.472951978892088,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ru","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ru","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ru","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ru","metric":"candidate_accuracy","value":0.96,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ru","metric":"candidate_ece","value":0.06510985448956491,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ru","metric":"baseline_accuracy","value":0.57,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ru","metric":"baseline_ece","value":0.2959438425302505,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sl","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.sl","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.sl","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.sl","metric":"candidate_accuracy","value":0.9,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sl","metric":"candidate_ece","value":0.05202076986432077,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sl","metric":"baseline_accuracy","value":0.33,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sl","metric":"baseline_ece","value":0.48708411037921906,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sq","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.sq","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.sq","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.sq","metric":"candidate_accuracy","value":0.88,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sq","metric":"candidate_ece","value":0.07386779703199861,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sq","metric":"baseline_accuracy","value":0.29,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sq","metric":"baseline_ece","value":0.4996085746586323,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sv","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.sv","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.sv","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.sv","metric":"candidate_accuracy","value":0.88,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sv","metric":"candidate_ece","value":0.06018931411206718,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sv","metric":"baseline_accuracy","value":0.48,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sv","metric":"baseline_ece","value":0.3698970249295235,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sw","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.sw","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.sw","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.sw","metric":"candidate_accuracy","value":0.81,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sw","metric":"candidate_ece","value":0.06739190667867664,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sw","metric":"baseline_accuracy","value":0.13,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sw","metric":"baseline_ece","value":0.6278604088723658,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ta","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ta","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ta","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ta","metric":"candidate_accuracy","value":0.84,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ta","metric":"candidate_ece","value":0.08809512436389924,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ta","metric":"baseline_accuracy","value":0.31,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ta","metric":"baseline_ece","value":0.4479472939670086,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.te","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.te","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.te","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.te","metric":"candidate_accuracy","value":0.83,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.te","metric":"candidate_ece","value":0.0734219098091125,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.te","metric":"baseline_accuracy","value":0.22,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.te","metric":"baseline_ece","value":0.4968701080977917,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.th","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.th","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.th","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.th","metric":"candidate_accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.th","metric":"candidate_ece","value":0.049760746955871585,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.th","metric":"baseline_accuracy","value":0.48,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.th","metric":"baseline_ece","value":0.356314766779542,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.tl","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.tl","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.tl","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.tl","metric":"candidate_accuracy","value":0.86,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.tl","metric":"candidate_ece","value":0.050127259045839354,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.tl","metric":"baseline_accuracy","value":0.3,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.tl","metric":"baseline_ece","value":0.49216231718659403,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.tr","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.tr","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.tr","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.tr","metric":"candidate_accuracy","value":0.88,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.tr","metric":"candidate_ece","value":0.07631486147642136,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.tr","metric":"baseline_accuracy","value":0.41,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.tr","metric":"baseline_ece","value":0.3854488131403923,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ur","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ur","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ur","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ur","metric":"candidate_accuracy","value":0.86,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ur","metric":"candidate_ece","value":0.07368002861738204,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ur","metric":"baseline_accuracy","value":0.42,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ur","metric":"baseline_ece","value":0.33551987260580063,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.vi","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.vi","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.vi","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.vi","metric":"candidate_accuracy","value":0.89,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.vi","metric":"candidate_ece","value":0.05955071091651916,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.vi","metric":"baseline_accuracy","value":0.33,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.vi","metric":"baseline_ece","value":0.4669697824120522,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.zh-CN","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.zh-CN","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.zh-CN","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.zh-CN","metric":"candidate_accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.zh-CN","metric":"candidate_ece","value":0.050360209196805984,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.zh-CN","metric":"baseline_accuracy","value":0.65,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.zh-CN","metric":"baseline_ece","value":0.2191513399779796,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.zh-TW","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.zh-TW","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.zh-TW","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.zh-TW","metric":"candidate_accuracy","value":0.97,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.zh-TW","metric":"candidate_ece","value":0.07615835040807721,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.zh-TW","metric":"baseline_accuracy","value":0.61,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.zh-TW","metric":"baseline_ece","value":0.2661561845242977,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/clinc150","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/clinc150","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/clinc150","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/clinc150","metric":"candidate_accuracy","value":0.76,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/clinc150","metric":"candidate_ece","value":0.14612227261066435,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/sst5","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/sst5","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/sst5","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/sst5","metric":"candidate_accuracy","value":0.39,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/sst5","metric":"candidate_ece","value":0.18840801835060123,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.ar","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.ar","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.ar","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.ar","metric":"candidate_accuracy","value":0.66,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.ar","metric":"candidate_ece","value":0.14292311161756513,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.bg","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.bg","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.bg","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.bg","metric":"candidate_accuracy","value":0.77,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.bg","metric":"candidate_ece","value":0.1984384161233902,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.de","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.de","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.de","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.de","metric":"candidate_accuracy","value":0.8,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.de","metric":"candidate_ece","value":0.21225321918725967,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.el","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.el","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.el","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.el","metric":"candidate_accuracy","value":0.76,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.el","metric":"candidate_ece","value":0.24297191977500918,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.en","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.en","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.en","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.en","metric":"candidate_accuracy","value":0.77,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.en","metric":"candidate_ece","value":0.157812180519104,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.es","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.es","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.es","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.es","metric":"candidate_accuracy","value":0.79,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.es","metric":"candidate_ece","value":0.22035292655229569,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.fr","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.fr","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.fr","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.fr","metric":"candidate_accuracy","value":0.77,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.fr","metric":"candidate_ece","value":0.18113152056932452,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.hi","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.hi","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.hi","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.hi","metric":"candidate_accuracy","value":0.64,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.hi","metric":"candidate_ece","value":0.13377510339021684,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.ru","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.ru","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.ru","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.ru","metric":"candidate_accuracy","value":0.72,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.ru","metric":"candidate_ece","value":0.19517139703035355,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.sw","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.sw","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.sw","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.sw","metric":"candidate_accuracy","value":0.59,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.sw","metric":"candidate_ece","value":0.1174077433347702,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.th","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.th","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.th","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.th","metric":"candidate_accuracy","value":0.66,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.th","metric":"candidate_ece","value":0.1255523270368576,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.tr","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.tr","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.tr","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.tr","metric":"candidate_accuracy","value":0.76,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.tr","metric":"candidate_ece","value":0.24283051997423175,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.ur","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.ur","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.ur","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.ur","metric":"candidate_accuracy","value":0.59,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.ur","metric":"candidate_ece","value":0.14045946687459945,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.vi","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.vi","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.vi","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.vi","metric":"candidate_accuracy","value":0.73,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.vi","metric":"candidate_ece","value":0.16299013167619708,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.zh","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.zh","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.zh","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.zh","metric":"candidate_accuracy","value":0.68,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.zh","metric":"candidate_ece","value":0.15210912466049198,"lo":null,"hi":null,"n":100,"of":null},{"category":"robustness/shuffle","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/shuffle","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/shuffle","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/shuffle","metric":"candidate_accuracy","value":0.878845819572397,"lo":null,"hi":null,"n":5753,"of":null},{"category":"robustness/shuffle","metric":"candidate_ece","value":0.009889678792260997,"lo":null,"hi":null,"n":5753,"of":null},{"category":"robustness/alias","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/alias","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/alias","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/alias","metric":"candidate_accuracy","value":0.8578052550231839,"lo":null,"hi":null,"n":647,"of":null},{"category":"robustness/alias","metric":"candidate_ece","value":0.04941958235072942,"lo":null,"hi":null,"n":647,"of":null},{"category":"robustness/swap","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/swap","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/swap","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/swap","metric":"candidate_accuracy","value":0.8562596599690881,"lo":null,"hi":null,"n":647,"of":null},{"category":"robustness/swap","metric":"candidate_ece","value":0.04407818801988954,"lo":null,"hi":null,"n":647,"of":null},{"category":"robustness/subset","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/subset","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/subset","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/subset","metric":"candidate_accuracy","value":0.940188679245283,"lo":null,"hi":null,"n":5300,"of":null},{"category":"robustness/subset","metric":"candidate_ece","value":0.011920550423410763,"lo":null,"hi":null,"n":5300,"of":null},{"category":"robustness/near","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/near","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/near","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/near","metric":"candidate_accuracy","value":0.9060377358490566,"lo":null,"hi":null,"n":5300,"of":null},{"category":"robustness/near","metric":"candidate_ece","value":0.030989159809530865,"lo":null,"hi":null,"n":5300,"of":null},{"category":"parity","metric":"pass","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"parity","metric":"labels","value":108,"lo":null,"hi":null,"n":null,"of":null},{"category":"parity","metric":"equal","value":108,"lo":null,"hi":null,"n":null,"of":null},{"category":"parity","metric":"logit","value":0.0000286102294921875,"lo":null,"hi":null,"n":null,"of":null},{"category":"parity","metric":"bound","value":0.0001,"lo":null,"hi":null,"n":null,"of":null}],"notes":"tolerance accuracy 0.02, ece 0.03; gate sha256:af0823ceeb14fe3a544e93fffa0bd2f9100ab76ecb664bb130ee8ecbbde6a273","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"gate.laya-drawn-a6.a211cc70-kai","benchmark":"gate.laya","split":"drawn","system":"kai","version":"a6@a211cc70103825bfbc9bfb0fbe970f8bf08da2efe0953cd8045a3cd07397f4bf","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"11474208ce49e6834bad21db4af8bb7ddf33b0ada6138b3661938522cafc815f","store_digest":"","facts_digest":"","commit":"a1a4a73d524cecf8038e30c408ad30b66e71d490","questions":null,"answered":null,"when":"","ended":"2026-09-28T03:18:42Z","by":"","completion":"unknown","measures":[{"category":"all","metric":"pass","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"suites","value":62,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"candidate","value":0.896709677419355,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"baseline","value":0.43848387096774183,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"delta","value":0.4582258064516132,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"regressions","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"cases","value":100,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/jev.ag_news","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/jev.ag_news","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/jev.ag_news","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/jev.ag_news","metric":"candidate_accuracy","value":0.95,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/jev.ag_news","metric":"candidate_ece","value":0.04369553148746491,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/jev.ag_news","metric":"baseline_accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/jev.ag_news","metric":"baseline_ece","value":0.04963111966848367,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/jev.emotion","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/jev.emotion","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/jev.emotion","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/jev.emotion","metric":"candidate_accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/jev.emotion","metric":"candidate_ece","value":0.04338422119617459,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/jev.emotion","metric":"baseline_accuracy","value":0.63,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/jev.emotion","metric":"baseline_ece","value":0.29464753001928334,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/jev.banking77_full","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/jev.banking77_full","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/jev.banking77_full","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/jev.banking77_full","metric":"candidate_accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/jev.banking77_full","metric":"candidate_ece","value":0.06478451311588287,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/jev.banking77_full","metric":"baseline_accuracy","value":0.42,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/jev.banking77_full","metric":"baseline_ece","value":0.39392668925225743,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.support_triage","metric":"regressed","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.support_triage","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.support_triage","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.support_triage","metric":"candidate_accuracy","value":0.37,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.support_triage","metric":"candidate_ece","value":0.4822892910242081,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.support_triage","metric":"baseline_accuracy","value":0.48,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.support_triage","metric":"baseline_ece","value":0.11383459806442263,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.email_spam","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.email_spam","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.email_spam","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.email_spam","metric":"candidate_accuracy","value":0.99,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.email_spam","metric":"candidate_ece","value":0.00896555125713343,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.email_spam","metric":"baseline_accuracy","value":1,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.email_spam","metric":"baseline_ece","value":0.0038367044925689377,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.phishing","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.phishing","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.phishing","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.phishing","metric":"candidate_accuracy","value":1,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.phishing","metric":"candidate_ece","value":0.0000335377454757646,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.phishing","metric":"baseline_accuracy","value":1,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.phishing","metric":"baseline_ece","value":0.010809252858161901,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.guardrails_jailbreak","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.guardrails_jailbreak","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.guardrails_jailbreak","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.guardrails_jailbreak","metric":"candidate_accuracy","value":0.91,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.guardrails_jailbreak","metric":"candidate_ece","value":0.060433058142662074,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.guardrails_jailbreak","metric":"baseline_accuracy","value":0.7,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.guardrails_jailbreak","metric":"baseline_ece","value":0.2747894221544265,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.moderation_toxicity","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.moderation_toxicity","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.moderation_toxicity","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.moderation_toxicity","metric":"candidate_accuracy","value":0.81,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.moderation_toxicity","metric":"candidate_ece","value":0.1910340225696563,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.moderation_toxicity","metric":"baseline_accuracy","value":0.6,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.moderation_toxicity","metric":"baseline_ece","value":0.2243971598148346,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.rag_relevance","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.rag_relevance","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.rag_relevance","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.rag_relevance","metric":"candidate_accuracy","value":0.57,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.rag_relevance","metric":"candidate_ece","value":0.11459835350513459,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.rag_relevance","metric":"baseline_accuracy","value":0.48,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.rag_relevance","metric":"baseline_ece","value":0.26150231361389165,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.model_routing_domain","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.model_routing_domain","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.model_routing_domain","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.model_routing_domain","metric":"candidate_accuracy","value":1,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.model_routing_domain","metric":"candidate_ece","value":0.0003541868925094338,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.model_routing_domain","metric":"baseline_accuracy","value":0.7,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.model_routing_domain","metric":"baseline_ece","value":0.09083123058080676,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/typed_decisions","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/typed_decisions","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/typed_decisions","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/typed_decisions","metric":"candidate_accuracy","value":0.746,"lo":null,"hi":null,"n":500,"of":null},{"category":"harness/typed_decisions","metric":"candidate_ece","value":0.20499304324388506,"lo":null,"hi":null,"n":500,"of":null},{"category":"harness/typed_decisions","metric":"baseline_accuracy","value":0.746,"lo":null,"hi":null,"n":500,"of":null},{"category":"harness/typed_decisions","metric":"baseline_ece","value":0.19970310807228087,"lo":null,"hi":null,"n":500,"of":null},{"category":"harness/massive.af","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.af","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.af","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.af","metric":"candidate_accuracy","value":0.9,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.af","metric":"candidate_ece","value":0.06145614624023434,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.af","metric":"baseline_accuracy","value":0.34,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.af","metric":"baseline_ece","value":0.48447977453470237,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.am","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.am","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.am","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.am","metric":"candidate_accuracy","value":0.85,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.am","metric":"candidate_ece","value":0.0714535458385944,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.am","metric":"baseline_accuracy","value":0.15,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.am","metric":"baseline_ece","value":0.48387789905071266,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ar","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ar","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ar","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ar","metric":"candidate_accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ar","metric":"candidate_ece","value":0.06882738918066027,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ar","metric":"baseline_accuracy","value":0.46,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ar","metric":"baseline_ece","value":0.3150437718629837,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.az","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.az","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.az","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.az","metric":"candidate_accuracy","value":0.95,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.az","metric":"candidate_ece","value":0.048022165298461886,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.az","metric":"baseline_accuracy","value":0.36,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.az","metric":"baseline_ece","value":0.39286278426647187,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.bn","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.bn","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.bn","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.bn","metric":"candidate_accuracy","value":0.9,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.bn","metric":"candidate_ece","value":0.08672305166721346,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.bn","metric":"baseline_accuracy","value":0.45,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.bn","metric":"baseline_ece","value":0.3539693360030651,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.cy","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.cy","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.cy","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.cy","metric":"candidate_accuracy","value":0.86,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.cy","metric":"candidate_ece","value":0.049075031131505915,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.cy","metric":"baseline_accuracy","value":0.16,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.cy","metric":"baseline_ece","value":0.6107046373188496,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.da","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.da","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.da","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.da","metric":"candidate_accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.da","metric":"candidate_ece","value":0.03354248374700548,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.da","metric":"baseline_accuracy","value":0.46,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.da","metric":"baseline_ece","value":0.45181164443492894,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.de","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.de","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.de","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.de","metric":"candidate_accuracy","value":0.91,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.de","metric":"candidate_ece","value":0.06285556972026828,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.de","metric":"baseline_accuracy","value":0.53,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.de","metric":"baseline_ece","value":0.30368388086557385,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.el","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.el","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.el","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.el","metric":"candidate_accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.el","metric":"candidate_ece","value":0.04968505382537845,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.el","metric":"baseline_accuracy","value":0.44,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.el","metric":"baseline_ece","value":0.4296881093084812,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.en","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.en","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.en","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.en","metric":"candidate_accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.en","metric":"candidate_ece","value":0.05991215109825135,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.en","metric":"baseline_accuracy","value":0.82,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.en","metric":"baseline_ece","value":0.13824885368347167,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.es","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.es","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.es","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.es","metric":"candidate_accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.es","metric":"candidate_ece","value":0.060675814151763846,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.es","metric":"baseline_accuracy","value":0.53,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.es","metric":"baseline_ece","value":0.34762571096420286,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.fa","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.fa","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.fa","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.fa","metric":"candidate_accuracy","value":0.95,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.fa","metric":"candidate_ece","value":0.03944258913397788,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.fa","metric":"baseline_accuracy","value":0.51,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.fa","metric":"baseline_ece","value":0.2985081049799919,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.fi","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.fi","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.fi","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.fi","metric":"candidate_accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.fi","metric":"candidate_ece","value":0.04163172006607054,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.fi","metric":"baseline_accuracy","value":0.3,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.fi","metric":"baseline_ece","value":0.4618027746677399,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.fr","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.fr","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.fr","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.fr","metric":"candidate_accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.fr","metric":"candidate_ece","value":0.056455891728401285,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.fr","metric":"baseline_accuracy","value":0.6,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.fr","metric":"baseline_ece","value":0.26832883805036545,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.he","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.he","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.he","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.he","metric":"candidate_accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.he","metric":"candidate_ece","value":0.06197642922401433,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.he","metric":"baseline_accuracy","value":0.37,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.he","metric":"baseline_ece","value":0.4185565242916346,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.hi","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.hi","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.hi","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.hi","metric":"candidate_accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.hi","metric":"candidate_ece","value":0.05225563615560533,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.hi","metric":"baseline_accuracy","value":0.46,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.hi","metric":"baseline_ece","value":0.36848925307393077,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.hu","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.hu","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.hu","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.hu","metric":"candidate_accuracy","value":0.86,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.hu","metric":"candidate_ece","value":0.0787442502379418,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.hu","metric":"baseline_accuracy","value":0.34,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.hu","metric":"baseline_ece","value":0.4373521935939789,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.hy","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.hy","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.hy","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.hy","metric":"candidate_accuracy","value":0.87,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.hy","metric":"candidate_ece","value":0.06506374388933181,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.hy","metric":"baseline_accuracy","value":0.25,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.hy","metric":"baseline_ece","value":0.5127132028341294,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.id","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.id","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.id","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.id","metric":"candidate_accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.id","metric":"candidate_ece","value":0.03650752246379858,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.id","metric":"baseline_accuracy","value":0.36,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.id","metric":"baseline_ece","value":0.48560510724782946,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.is","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.is","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.is","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.is","metric":"candidate_accuracy","value":0.87,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.is","metric":"candidate_ece","value":0.0637242865562439,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.is","metric":"baseline_accuracy","value":0.33,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.is","metric":"baseline_ece","value":0.48597560346126556,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.it","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.it","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.it","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.it","metric":"candidate_accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.it","metric":"candidate_ece","value":0.056774215698242245,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.it","metric":"baseline_accuracy","value":0.47,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.it","metric":"baseline_ece","value":0.38000325053930284,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ja","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ja","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ja","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ja","metric":"candidate_accuracy","value":0.97,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ja","metric":"candidate_ece","value":0.03286569952964781,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ja","metric":"baseline_accuracy","value":0.64,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ja","metric":"baseline_ece","value":0.23371360868215557,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.jv","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.jv","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.jv","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.jv","metric":"candidate_accuracy","value":0.85,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.jv","metric":"candidate_ece","value":0.07020064458251002,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.jv","metric":"baseline_accuracy","value":0.16,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.jv","metric":"baseline_ece","value":0.6633555415272713,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ka","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ka","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ka","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ka","metric":"candidate_accuracy","value":0.83,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ka","metric":"candidate_ece","value":0.08449125021696088,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ka","metric":"baseline_accuracy","value":0.15,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ka","metric":"baseline_ece","value":0.5422389303892852,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.km","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.km","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.km","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.km","metric":"candidate_accuracy","value":0.85,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.km","metric":"candidate_ece","value":0.08929385468363762,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.km","metric":"baseline_accuracy","value":0.2,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.km","metric":"baseline_ece","value":0.44535648457705973,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.kn","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.kn","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.kn","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.kn","metric":"candidate_accuracy","value":0.9,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.kn","metric":"candidate_ece","value":0.08473139792680738,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.kn","metric":"baseline_accuracy","value":0.3,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.kn","metric":"baseline_ece","value":0.3880462409555912,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ko","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ko","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ko","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ko","metric":"candidate_accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ko","metric":"candidate_ece","value":0.03255369871854782,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ko","metric":"baseline_accuracy","value":0.47,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ko","metric":"baseline_ece","value":0.31081631831824785,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.lv","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.lv","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.lv","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.lv","metric":"candidate_accuracy","value":0.89,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.lv","metric":"candidate_ece","value":0.07157214641571044,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.lv","metric":"baseline_accuracy","value":0.3,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.lv","metric":"baseline_ece","value":0.4859770181775093,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ml","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ml","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ml","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ml","metric":"candidate_accuracy","value":0.87,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ml","metric":"candidate_ece","value":0.08362111099064345,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ml","metric":"baseline_accuracy","value":0.28,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ml","metric":"baseline_ece","value":0.4433153660595417,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.mn","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.mn","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.mn","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.mn","metric":"candidate_accuracy","value":0.88,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.mn","metric":"candidate_ece","value":0.08079831957817081,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.mn","metric":"baseline_accuracy","value":0.16,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.mn","metric":"baseline_ece","value":0.5769782765209674,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ms","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ms","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ms","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ms","metric":"candidate_accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ms","metric":"candidate_ece","value":0.06407836556434633,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ms","metric":"baseline_accuracy","value":0.27,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ms","metric":"baseline_ece","value":0.5063951462507248,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.my","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.my","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.my","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.my","metric":"candidate_accuracy","value":0.95,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.my","metric":"candidate_ece","value":0.04778237879276278,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.my","metric":"baseline_accuracy","value":0.16,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.my","metric":"baseline_ece","value":0.4766131702065468,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.nb","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.nb","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.nb","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.nb","metric":"candidate_accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.nb","metric":"candidate_ece","value":0.04958597868680945,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.nb","metric":"baseline_accuracy","value":0.5,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.nb","metric":"baseline_ece","value":0.36785355895757677,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.nl","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.nl","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.nl","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.nl","metric":"candidate_accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.nl","metric":"candidate_ece","value":0.05387416154146192,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.nl","metric":"baseline_accuracy","value":0.4,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.nl","metric":"baseline_ece","value":0.479215635061264,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.pl","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.pl","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.pl","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.pl","metric":"candidate_accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.pl","metric":"candidate_ece","value":0.029607748985290512,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.pl","metric":"baseline_accuracy","value":0.47,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.pl","metric":"baseline_ece","value":0.37366658061742786,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.pt","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.pt","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.pt","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.pt","metric":"candidate_accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.pt","metric":"candidate_ece","value":0.03121975854039198,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.pt","metric":"baseline_accuracy","value":0.44,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.pt","metric":"baseline_ece","value":0.3956287080049515,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ro","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ro","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ro","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ro","metric":"candidate_accuracy","value":0.95,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ro","metric":"candidate_ece","value":0.04741007596254352,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ro","metric":"baseline_accuracy","value":0.37,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ro","metric":"baseline_ece","value":0.472951978892088,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ru","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ru","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ru","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ru","metric":"candidate_accuracy","value":0.98,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ru","metric":"candidate_ece","value":0.04467343077063564,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ru","metric":"baseline_accuracy","value":0.57,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ru","metric":"baseline_ece","value":0.2959438425302505,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sl","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.sl","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.sl","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.sl","metric":"candidate_accuracy","value":0.89,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sl","metric":"candidate_ece","value":0.048560316711664275,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sl","metric":"baseline_accuracy","value":0.33,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sl","metric":"baseline_ece","value":0.48708411037921906,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sq","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.sq","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.sq","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.sq","metric":"candidate_accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sq","metric":"candidate_ece","value":0.047673491090536056,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sq","metric":"baseline_accuracy","value":0.29,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sq","metric":"baseline_ece","value":0.4996085746586323,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sv","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.sv","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.sv","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.sv","metric":"candidate_accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sv","metric":"candidate_ece","value":0.05960643857717518,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sv","metric":"baseline_accuracy","value":0.48,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sv","metric":"baseline_ece","value":0.3698970249295235,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sw","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.sw","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.sw","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.sw","metric":"candidate_accuracy","value":0.83,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sw","metric":"candidate_ece","value":0.07877558827400208,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sw","metric":"baseline_accuracy","value":0.13,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sw","metric":"baseline_ece","value":0.6278604088723658,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ta","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ta","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ta","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ta","metric":"candidate_accuracy","value":0.91,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ta","metric":"candidate_ece","value":0.04905078291893006,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ta","metric":"baseline_accuracy","value":0.31,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ta","metric":"baseline_ece","value":0.4479472939670086,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.te","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.te","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.te","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.te","metric":"candidate_accuracy","value":0.85,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.te","metric":"candidate_ece","value":0.11967309281229979,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.te","metric":"baseline_accuracy","value":0.22,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.te","metric":"baseline_ece","value":0.4968701080977917,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.th","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.th","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.th","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.th","metric":"candidate_accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.th","metric":"candidate_ece","value":0.03415450304746626,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.th","metric":"baseline_accuracy","value":0.48,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.th","metric":"baseline_ece","value":0.356314766779542,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.tl","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.tl","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.tl","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.tl","metric":"candidate_accuracy","value":0.91,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.tl","metric":"candidate_ece","value":0.07738257154822352,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.tl","metric":"baseline_accuracy","value":0.3,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.tl","metric":"baseline_ece","value":0.49216231718659403,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.tr","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.tr","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.tr","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.tr","metric":"candidate_accuracy","value":0.9,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.tr","metric":"candidate_ece","value":0.07372443810105328,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.tr","metric":"baseline_accuracy","value":0.41,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.tr","metric":"baseline_ece","value":0.3854488131403923,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ur","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ur","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ur","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ur","metric":"candidate_accuracy","value":0.91,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ur","metric":"candidate_ece","value":0.06974980145692826,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ur","metric":"baseline_accuracy","value":0.42,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ur","metric":"baseline_ece","value":0.33551987260580063,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.vi","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.vi","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.vi","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.vi","metric":"candidate_accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.vi","metric":"candidate_ece","value":0.06033400148153302,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.vi","metric":"baseline_accuracy","value":0.33,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.vi","metric":"baseline_ece","value":0.4669697824120522,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.zh-CN","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.zh-CN","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.zh-CN","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.zh-CN","metric":"candidate_accuracy","value":0.95,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.zh-CN","metric":"candidate_ece","value":0.039882711470127084,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.zh-CN","metric":"baseline_accuracy","value":0.65,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.zh-CN","metric":"baseline_ece","value":0.2191513399779796,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.zh-TW","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.zh-TW","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.zh-TW","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.zh-TW","metric":"candidate_accuracy","value":0.96,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.zh-TW","metric":"candidate_ece","value":0.04104021757841109,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.zh-TW","metric":"baseline_accuracy","value":0.61,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.zh-TW","metric":"baseline_ece","value":0.2661561845242977,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/clinc150","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/clinc150","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/clinc150","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/clinc150","metric":"candidate_accuracy","value":0.74,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/clinc150","metric":"candidate_ece","value":0.16210970818996423,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/sst5","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/sst5","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/sst5","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/sst5","metric":"candidate_accuracy","value":0.52,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/sst5","metric":"candidate_ece","value":0.1463029870390892,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.ar","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.ar","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.ar","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.ar","metric":"candidate_accuracy","value":0.69,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.ar","metric":"candidate_ece","value":0.0982473087310791,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.bg","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.bg","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.bg","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.bg","metric":"candidate_accuracy","value":0.79,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.bg","metric":"candidate_ece","value":0.14978477597236634,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.de","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.de","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.de","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.de","metric":"candidate_accuracy","value":0.8,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.de","metric":"candidate_ece","value":0.1645866242051125,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.el","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.el","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.el","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.el","metric":"candidate_accuracy","value":0.81,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.el","metric":"candidate_ece","value":0.18419245570898057,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.en","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.en","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.en","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.en","metric":"candidate_accuracy","value":0.77,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.en","metric":"candidate_ece","value":0.10587243705987928,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.es","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.es","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.es","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.es","metric":"candidate_accuracy","value":0.72,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.es","metric":"candidate_ece","value":0.13122842520475386,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.fr","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.fr","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.fr","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.fr","metric":"candidate_accuracy","value":0.78,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.fr","metric":"candidate_ece","value":0.1400255334377289,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.hi","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.hi","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.hi","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.hi","metric":"candidate_accuracy","value":0.65,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.hi","metric":"candidate_ece","value":0.1458137741684914,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.ru","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.ru","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.ru","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.ru","metric":"candidate_accuracy","value":0.71,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.ru","metric":"candidate_ece","value":0.09388305157423021,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.sw","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.sw","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.sw","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.sw","metric":"candidate_accuracy","value":0.61,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.sw","metric":"candidate_ece","value":0.09246866583824158,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.th","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.th","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.th","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.th","metric":"candidate_accuracy","value":0.7,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.th","metric":"candidate_ece","value":0.13970610022544858,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.tr","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.tr","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.tr","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.tr","metric":"candidate_accuracy","value":0.73,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.tr","metric":"candidate_ece","value":0.1332623443007469,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.ur","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.ur","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.ur","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.ur","metric":"candidate_accuracy","value":0.61,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.ur","metric":"candidate_ece","value":0.08311391204595568,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.vi","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.vi","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.vi","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.vi","metric":"candidate_accuracy","value":0.73,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.vi","metric":"candidate_ece","value":0.14557734400033948,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.zh","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.zh","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.zh","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.zh","metric":"candidate_accuracy","value":0.7,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.zh","metric":"candidate_ece","value":0.06273882865905764,"lo":null,"hi":null,"n":100,"of":null},{"category":"robustness/shuffle","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/shuffle","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/shuffle","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/shuffle","metric":"candidate_accuracy","value":0.898140100816965,"lo":null,"hi":null,"n":5753,"of":null},{"category":"robustness/shuffle","metric":"candidate_ece","value":0.025156406533214614,"lo":null,"hi":null,"n":5753,"of":null},{"category":"robustness/alias","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/alias","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/alias","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/alias","metric":"candidate_accuracy","value":0.8438948995363215,"lo":null,"hi":null,"n":647,"of":null},{"category":"robustness/alias","metric":"candidate_ece","value":0.05747238497092885,"lo":null,"hi":null,"n":647,"of":null},{"category":"robustness/swap","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/swap","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/swap","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/swap","metric":"candidate_accuracy","value":0.8469860896445132,"lo":null,"hi":null,"n":647,"of":null},{"category":"robustness/swap","metric":"candidate_ece","value":0.046368170374143654,"lo":null,"hi":null,"n":647,"of":null},{"category":"robustness/subset","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/subset","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/subset","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/subset","metric":"candidate_accuracy","value":0.9509433962264151,"lo":null,"hi":null,"n":5300,"of":null},{"category":"robustness/subset","metric":"candidate_ece","value":0.01296593981250278,"lo":null,"hi":null,"n":5300,"of":null},{"category":"robustness/near","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/near","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/near","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/near","metric":"candidate_accuracy","value":0.9235849056603773,"lo":null,"hi":null,"n":5300,"of":null},{"category":"robustness/near","metric":"candidate_ece","value":0.01731854601851056,"lo":null,"hi":null,"n":5300,"of":null},{"category":"parity","metric":"pass","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"parity","metric":"labels","value":108,"lo":null,"hi":null,"n":null,"of":null},{"category":"parity","metric":"equal","value":108,"lo":null,"hi":null,"n":null,"of":null},{"category":"parity","metric":"logit","value":0.0000286102294921875,"lo":null,"hi":null,"n":null,"of":null},{"category":"parity","metric":"bound","value":0.0001,"lo":null,"hi":null,"n":null,"of":null}],"notes":"tolerance accuracy 0.02, ece 0.03; gate sha256:2e705afebce648bde63e16ba7f9b0bf65164da89c69ec6bcb97959074d4405cc","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"gate.laya-drawn-a7.0834a74f-kai","benchmark":"gate.laya","split":"drawn","system":"kai","version":"a7@0834a74f2d140642a453373da09e5a128e3d2c8d9e8bb1dcaf86d7210a4ecdfc","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"11474208ce49e6834bad21db4af8bb7ddf33b0ada6138b3661938522cafc815f","store_digest":"","facts_digest":"","commit":"a1a4a73d524cecf8038e30c408ad30b66e71d490","questions":null,"answered":null,"when":"","ended":"2026-09-28T12:24:45Z","by":"","completion":"unknown","measures":[{"category":"all","metric":"pass","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"suites","value":62,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"candidate","value":0.8962903225806449,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"baseline","value":0.43848387096774183,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"delta","value":0.45780645161290306,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"regressions","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"cases","value":100,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/jev.ag_news","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/jev.ag_news","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/jev.ag_news","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/jev.ag_news","metric":"candidate_accuracy","value":0.96,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/jev.ag_news","metric":"candidate_ece","value":0.055983276665210704,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/jev.ag_news","metric":"baseline_accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/jev.ag_news","metric":"baseline_ece","value":0.04963111966848367,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/jev.emotion","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/jev.emotion","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/jev.emotion","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/jev.emotion","metric":"candidate_accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/jev.emotion","metric":"candidate_ece","value":0.037905553877353705,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/jev.emotion","metric":"baseline_accuracy","value":0.63,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/jev.emotion","metric":"baseline_ece","value":0.29464753001928334,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/jev.banking77_full","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/jev.banking77_full","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/jev.banking77_full","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/jev.banking77_full","metric":"candidate_accuracy","value":0.91,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/jev.banking77_full","metric":"candidate_ece","value":0.09422180116176605,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/jev.banking77_full","metric":"baseline_accuracy","value":0.42,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/jev.banking77_full","metric":"baseline_ece","value":0.39392668925225743,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.support_triage","metric":"regressed","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.support_triage","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.support_triage","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.support_triage","metric":"candidate_accuracy","value":0.35,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.support_triage","metric":"candidate_ece","value":0.512402692437172,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.support_triage","metric":"baseline_accuracy","value":0.48,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.support_triage","metric":"baseline_ece","value":0.11383459806442263,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.email_spam","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.email_spam","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.email_spam","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.email_spam","metric":"candidate_accuracy","value":0.99,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.email_spam","metric":"candidate_ece","value":0.00944998085498816,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.email_spam","metric":"baseline_accuracy","value":1,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.email_spam","metric":"baseline_ece","value":0.0038367044925689377,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.phishing","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.phishing","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.phishing","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.phishing","metric":"candidate_accuracy","value":1,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.phishing","metric":"candidate_ece","value":0.0000173753499984608,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.phishing","metric":"baseline_accuracy","value":1,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.phishing","metric":"baseline_ece","value":0.010809252858161901,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.guardrails_jailbreak","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.guardrails_jailbreak","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.guardrails_jailbreak","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.guardrails_jailbreak","metric":"candidate_accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.guardrails_jailbreak","metric":"candidate_ece","value":0.04400036811828614,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.guardrails_jailbreak","metric":"baseline_accuracy","value":0.7,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.guardrails_jailbreak","metric":"baseline_ece","value":0.2747894221544265,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.moderation_toxicity","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.moderation_toxicity","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.moderation_toxicity","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.moderation_toxicity","metric":"candidate_accuracy","value":0.81,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.moderation_toxicity","metric":"candidate_ece","value":0.1898956561088562,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.moderation_toxicity","metric":"baseline_accuracy","value":0.6,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.moderation_toxicity","metric":"baseline_ece","value":0.2243971598148346,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.rag_relevance","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.rag_relevance","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.rag_relevance","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.rag_relevance","metric":"candidate_accuracy","value":0.55,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.rag_relevance","metric":"candidate_ece","value":0.12484459042549137,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.rag_relevance","metric":"baseline_accuracy","value":0.48,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.rag_relevance","metric":"baseline_ece","value":0.26150231361389165,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.model_routing_domain","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.model_routing_domain","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.model_routing_domain","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.model_routing_domain","metric":"candidate_accuracy","value":1,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.model_routing_domain","metric":"candidate_ece","value":0.0002811008691787631,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.model_routing_domain","metric":"baseline_accuracy","value":0.7,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.model_routing_domain","metric":"baseline_ece","value":0.09083123058080676,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/typed_decisions","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/typed_decisions","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/typed_decisions","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/typed_decisions","metric":"candidate_accuracy","value":0.75,"lo":null,"hi":null,"n":500,"of":null},{"category":"harness/typed_decisions","metric":"candidate_ece","value":0.1635413258075714,"lo":null,"hi":null,"n":500,"of":null},{"category":"harness/typed_decisions","metric":"baseline_accuracy","value":0.746,"lo":null,"hi":null,"n":500,"of":null},{"category":"harness/typed_decisions","metric":"baseline_ece","value":0.19970310807228087,"lo":null,"hi":null,"n":500,"of":null},{"category":"harness/massive.af","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.af","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.af","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.af","metric":"candidate_accuracy","value":0.91,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.af","metric":"candidate_ece","value":0.04738804578781133,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.af","metric":"baseline_accuracy","value":0.34,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.af","metric":"baseline_ece","value":0.48447977453470237,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.am","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.am","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.am","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.am","metric":"candidate_accuracy","value":0.85,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.am","metric":"candidate_ece","value":0.08526809126138692,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.am","metric":"baseline_accuracy","value":0.15,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.am","metric":"baseline_ece","value":0.48387789905071266,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ar","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ar","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ar","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ar","metric":"candidate_accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ar","metric":"candidate_ece","value":0.06399317130446432,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ar","metric":"baseline_accuracy","value":0.46,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ar","metric":"baseline_ece","value":0.3150437718629837,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.az","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.az","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.az","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.az","metric":"candidate_accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.az","metric":"candidate_ece","value":0.0359751570224762,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.az","metric":"baseline_accuracy","value":0.36,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.az","metric":"baseline_ece","value":0.39286278426647187,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.bn","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.bn","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.bn","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.bn","metric":"candidate_accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.bn","metric":"candidate_ece","value":0.06581725120544439,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.bn","metric":"baseline_accuracy","value":0.45,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.bn","metric":"baseline_ece","value":0.3539693360030651,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.cy","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.cy","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.cy","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.cy","metric":"candidate_accuracy","value":0.86,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.cy","metric":"candidate_ece","value":0.08170279085636137,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.cy","metric":"baseline_accuracy","value":0.16,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.cy","metric":"baseline_ece","value":0.6107046373188496,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.da","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.da","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.da","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.da","metric":"candidate_accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.da","metric":"candidate_ece","value":0.050635895729064906,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.da","metric":"baseline_accuracy","value":0.46,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.da","metric":"baseline_ece","value":0.45181164443492894,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.de","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.de","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.de","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.de","metric":"candidate_accuracy","value":0.91,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.de","metric":"candidate_ece","value":0.04193521454930304,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.de","metric":"baseline_accuracy","value":0.53,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.de","metric":"baseline_ece","value":0.30368388086557385,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.el","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.el","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.el","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.el","metric":"candidate_accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.el","metric":"candidate_ece","value":0.03144277065992359,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.el","metric":"baseline_accuracy","value":0.44,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.el","metric":"baseline_ece","value":0.4296881093084812,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.en","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.en","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.en","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.en","metric":"candidate_accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.en","metric":"candidate_ece","value":0.04790517151355743,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.en","metric":"baseline_accuracy","value":0.82,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.en","metric":"baseline_ece","value":0.13824885368347167,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.es","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.es","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.es","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.es","metric":"candidate_accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.es","metric":"candidate_ece","value":0.050300997197628004,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.es","metric":"baseline_accuracy","value":0.53,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.es","metric":"baseline_ece","value":0.34762571096420286,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.fa","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.fa","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.fa","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.fa","metric":"candidate_accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.fa","metric":"candidate_ece","value":0.03188615605235109,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.fa","metric":"baseline_accuracy","value":0.51,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.fa","metric":"baseline_ece","value":0.2985081049799919,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.fi","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.fi","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.fi","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.fi","metric":"candidate_accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.fi","metric":"candidate_ece","value":0.055109691023826686,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.fi","metric":"baseline_accuracy","value":0.3,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.fi","metric":"baseline_ece","value":0.4618027746677399,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.fr","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.fr","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.fr","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.fr","metric":"candidate_accuracy","value":0.91,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.fr","metric":"candidate_ece","value":0.07386496245861054,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.fr","metric":"baseline_accuracy","value":0.6,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.fr","metric":"baseline_ece","value":0.26832883805036545,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.he","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.he","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.he","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.he","metric":"candidate_accuracy","value":0.91,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.he","metric":"candidate_ece","value":0.055365550518035864,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.he","metric":"baseline_accuracy","value":0.37,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.he","metric":"baseline_ece","value":0.4185565242916346,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.hi","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.hi","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.hi","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.hi","metric":"candidate_accuracy","value":0.91,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.hi","metric":"candidate_ece","value":0.04933870881795882,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.hi","metric":"baseline_accuracy","value":0.46,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.hi","metric":"baseline_ece","value":0.36848925307393077,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.hu","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.hu","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.hu","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.hu","metric":"candidate_accuracy","value":0.9,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.hu","metric":"candidate_ece","value":0.07938083112239844,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.hu","metric":"baseline_accuracy","value":0.34,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.hu","metric":"baseline_ece","value":0.4373521935939789,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.hy","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.hy","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.hy","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.hy","metric":"candidate_accuracy","value":0.89,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.hy","metric":"candidate_ece","value":0.0601737295091152,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.hy","metric":"baseline_accuracy","value":0.25,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.hy","metric":"baseline_ece","value":0.5127132028341294,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.id","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.id","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.id","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.id","metric":"candidate_accuracy","value":0.96,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.id","metric":"candidate_ece","value":0.05166170924901959,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.id","metric":"baseline_accuracy","value":0.36,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.id","metric":"baseline_ece","value":0.48560510724782946,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.is","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.is","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.is","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.is","metric":"candidate_accuracy","value":0.85,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.is","metric":"candidate_ece","value":0.07128985196352003,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.is","metric":"baseline_accuracy","value":0.33,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.is","metric":"baseline_ece","value":0.48597560346126556,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.it","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.it","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.it","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.it","metric":"candidate_accuracy","value":0.91,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.it","metric":"candidate_ece","value":0.05676230788230893,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.it","metric":"baseline_accuracy","value":0.47,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.it","metric":"baseline_ece","value":0.38000325053930284,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ja","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ja","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ja","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ja","metric":"candidate_accuracy","value":0.97,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ja","metric":"candidate_ece","value":0.03706384837627408,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ja","metric":"baseline_accuracy","value":0.64,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ja","metric":"baseline_ece","value":0.23371360868215557,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.jv","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.jv","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.jv","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.jv","metric":"candidate_accuracy","value":0.89,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.jv","metric":"candidate_ece","value":0.07372990816831584,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.jv","metric":"baseline_accuracy","value":0.16,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.jv","metric":"baseline_ece","value":0.6633555415272713,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ka","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ka","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ka","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ka","metric":"candidate_accuracy","value":0.83,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ka","metric":"candidate_ece","value":0.07632239848375316,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ka","metric":"baseline_accuracy","value":0.15,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ka","metric":"baseline_ece","value":0.5422389303892852,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.km","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.km","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.km","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.km","metric":"candidate_accuracy","value":0.84,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.km","metric":"candidate_ece","value":0.07897142395377156,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.km","metric":"baseline_accuracy","value":0.2,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.km","metric":"baseline_ece","value":0.44535648457705973,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.kn","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.kn","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.kn","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.kn","metric":"candidate_accuracy","value":0.91,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.kn","metric":"candidate_ece","value":0.08909560441970826,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.kn","metric":"baseline_accuracy","value":0.3,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.kn","metric":"baseline_ece","value":0.3880462409555912,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ko","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ko","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ko","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ko","metric":"candidate_accuracy","value":0.91,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ko","metric":"candidate_ece","value":0.0583547832071781,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ko","metric":"baseline_accuracy","value":0.47,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ko","metric":"baseline_ece","value":0.31081631831824785,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.lv","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.lv","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.lv","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.lv","metric":"candidate_accuracy","value":0.89,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.lv","metric":"candidate_ece","value":0.08676462039351468,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.lv","metric":"baseline_accuracy","value":0.3,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.lv","metric":"baseline_ece","value":0.4859770181775093,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ml","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ml","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ml","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ml","metric":"candidate_accuracy","value":0.87,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ml","metric":"candidate_ece","value":0.08541164651513103,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ml","metric":"baseline_accuracy","value":0.28,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ml","metric":"baseline_ece","value":0.4433153660595417,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.mn","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.mn","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.mn","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.mn","metric":"candidate_accuracy","value":0.88,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.mn","metric":"candidate_ece","value":0.11159766197204596,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.mn","metric":"baseline_accuracy","value":0.16,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.mn","metric":"baseline_ece","value":0.5769782765209674,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ms","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ms","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ms","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ms","metric":"candidate_accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ms","metric":"candidate_ece","value":0.03943374574184415,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ms","metric":"baseline_accuracy","value":0.27,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ms","metric":"baseline_ece","value":0.5063951462507248,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.my","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.my","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.my","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.my","metric":"candidate_accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.my","metric":"candidate_ece","value":0.07324161291122437,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.my","metric":"baseline_accuracy","value":0.16,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.my","metric":"baseline_ece","value":0.4766131702065468,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.nb","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.nb","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.nb","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.nb","metric":"candidate_accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.nb","metric":"candidate_ece","value":0.037134091854095545,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.nb","metric":"baseline_accuracy","value":0.5,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.nb","metric":"baseline_ece","value":0.36785355895757677,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.nl","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.nl","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.nl","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.nl","metric":"candidate_accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.nl","metric":"candidate_ece","value":0.0496180704236031,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.nl","metric":"baseline_accuracy","value":0.4,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.nl","metric":"baseline_ece","value":0.479215635061264,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.pl","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.pl","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.pl","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.pl","metric":"candidate_accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.pl","metric":"candidate_ece","value":0.052072535604238544,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.pl","metric":"baseline_accuracy","value":0.47,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.pl","metric":"baseline_ece","value":0.37366658061742786,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.pt","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.pt","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.pt","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.pt","metric":"candidate_accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.pt","metric":"candidate_ece","value":0.037603752911090804,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.pt","metric":"baseline_accuracy","value":0.44,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.pt","metric":"baseline_ece","value":0.3956287080049515,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ro","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ro","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ro","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ro","metric":"candidate_accuracy","value":0.91,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ro","metric":"candidate_ece","value":0.05261404126882551,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ro","metric":"baseline_accuracy","value":0.37,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ro","metric":"baseline_ece","value":0.472951978892088,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ru","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ru","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ru","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ru","metric":"candidate_accuracy","value":0.97,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ru","metric":"candidate_ece","value":0.035205752253532396,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ru","metric":"baseline_accuracy","value":0.57,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ru","metric":"baseline_ece","value":0.2959438425302505,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sl","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.sl","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.sl","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.sl","metric":"candidate_accuracy","value":0.91,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sl","metric":"candidate_ece","value":0.05365944206714634,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sl","metric":"baseline_accuracy","value":0.33,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sl","metric":"baseline_ece","value":0.48708411037921906,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sq","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.sq","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.sq","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.sq","metric":"candidate_accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sq","metric":"candidate_ece","value":0.04722087457776066,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sq","metric":"baseline_accuracy","value":0.29,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sq","metric":"baseline_ece","value":0.4996085746586323,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sv","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.sv","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.sv","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.sv","metric":"candidate_accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sv","metric":"candidate_ece","value":0.04295731350779536,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sv","metric":"baseline_accuracy","value":0.48,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sv","metric":"baseline_ece","value":0.3698970249295235,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sw","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.sw","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.sw","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.sw","metric":"candidate_accuracy","value":0.84,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sw","metric":"candidate_ece","value":0.08623769655823708,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sw","metric":"baseline_accuracy","value":0.13,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sw","metric":"baseline_ece","value":0.6278604088723658,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ta","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ta","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ta","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ta","metric":"candidate_accuracy","value":0.88,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ta","metric":"candidate_ece","value":0.06233964949846268,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ta","metric":"baseline_accuracy","value":0.31,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ta","metric":"baseline_ece","value":0.4479472939670086,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.te","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.te","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.te","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.te","metric":"candidate_accuracy","value":0.87,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.te","metric":"candidate_ece","value":0.0855315700173378,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.te","metric":"baseline_accuracy","value":0.22,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.te","metric":"baseline_ece","value":0.4968701080977917,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.th","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.th","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.th","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.th","metric":"candidate_accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.th","metric":"candidate_ece","value":0.03923365086317059,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.th","metric":"baseline_accuracy","value":0.48,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.th","metric":"baseline_ece","value":0.356314766779542,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.tl","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.tl","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.tl","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.tl","metric":"candidate_accuracy","value":0.91,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.tl","metric":"candidate_ece","value":0.06683538392186163,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.tl","metric":"baseline_accuracy","value":0.3,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.tl","metric":"baseline_ece","value":0.49216231718659403,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.tr","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.tr","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.tr","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.tr","metric":"candidate_accuracy","value":0.9,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.tr","metric":"candidate_ece","value":0.07519040256738663,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.tr","metric":"baseline_accuracy","value":0.41,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.tr","metric":"baseline_ece","value":0.3854488131403923,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ur","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ur","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ur","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ur","metric":"candidate_accuracy","value":0.9,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ur","metric":"candidate_ece","value":0.06569901391863815,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ur","metric":"baseline_accuracy","value":0.42,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ur","metric":"baseline_ece","value":0.33551987260580063,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.vi","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.vi","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.vi","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.vi","metric":"candidate_accuracy","value":0.91,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.vi","metric":"candidate_ece","value":0.0680993591248989,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.vi","metric":"baseline_accuracy","value":0.33,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.vi","metric":"baseline_ece","value":0.4669697824120522,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.zh-CN","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.zh-CN","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.zh-CN","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.zh-CN","metric":"candidate_accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.zh-CN","metric":"candidate_ece","value":0.05872932702302928,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.zh-CN","metric":"baseline_accuracy","value":0.65,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.zh-CN","metric":"baseline_ece","value":0.2191513399779796,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.zh-TW","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.zh-TW","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.zh-TW","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.zh-TW","metric":"candidate_accuracy","value":0.95,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.zh-TW","metric":"candidate_ece","value":0.032052750289440116,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.zh-TW","metric":"baseline_accuracy","value":0.61,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.zh-TW","metric":"baseline_ece","value":0.2661561845242977,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/clinc150","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/clinc150","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/clinc150","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/clinc150","metric":"candidate_accuracy","value":0.79,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/clinc150","metric":"candidate_ece","value":0.14865223526954646,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/sst5","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/sst5","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/sst5","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/sst5","metric":"candidate_accuracy","value":0.55,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/sst5","metric":"candidate_ece","value":0.06311321079730987,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.ar","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.ar","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.ar","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.ar","metric":"candidate_accuracy","value":0.67,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.ar","metric":"candidate_ece","value":0.15268935799598696,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.bg","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.bg","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.bg","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.bg","metric":"candidate_accuracy","value":0.78,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.bg","metric":"candidate_ece","value":0.16177826255559924,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.de","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.de","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.de","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.de","metric":"candidate_accuracy","value":0.82,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.de","metric":"candidate_ece","value":0.17352425545454023,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.el","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.el","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.el","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.el","metric":"candidate_accuracy","value":0.8,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.el","metric":"candidate_ece","value":0.1832862091064453,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.en","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.en","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.en","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.en","metric":"candidate_accuracy","value":0.74,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.en","metric":"candidate_ece","value":0.12187356531620024,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.es","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.es","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.es","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.es","metric":"candidate_accuracy","value":0.74,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.es","metric":"candidate_ece","value":0.1250601354241371,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.fr","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.fr","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.fr","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.fr","metric":"candidate_accuracy","value":0.81,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.fr","metric":"candidate_ece","value":0.15819115191698074,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.hi","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.hi","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.hi","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.hi","metric":"candidate_accuracy","value":0.66,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.hi","metric":"candidate_ece","value":0.09690238654613496,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.ru","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.ru","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.ru","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.ru","metric":"candidate_accuracy","value":0.73,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.ru","metric":"candidate_ece","value":0.16613810688257216,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.sw","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.sw","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.sw","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.sw","metric":"candidate_accuracy","value":0.6,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.sw","metric":"candidate_ece","value":0.0703927117586136,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.th","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.th","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.th","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.th","metric":"candidate_accuracy","value":0.67,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.th","metric":"candidate_ece","value":0.0998651620745659,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.tr","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.tr","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.tr","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.tr","metric":"candidate_accuracy","value":0.7,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.tr","metric":"candidate_ece","value":0.1450413915514946,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.ur","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.ur","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.ur","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.ur","metric":"candidate_accuracy","value":0.62,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.ur","metric":"candidate_ece","value":0.0880414754152298,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.vi","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.vi","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.vi","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.vi","metric":"candidate_accuracy","value":0.7,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.vi","metric":"candidate_ece","value":0.1799289581179619,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.zh","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.zh","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.zh","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.zh","metric":"candidate_accuracy","value":0.63,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.zh","metric":"candidate_ece","value":0.12515602439641954,"lo":null,"hi":null,"n":100,"of":null},{"category":"robustness/shuffle","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/shuffle","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/shuffle","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/shuffle","metric":"candidate_accuracy","value":0.8972709890491918,"lo":null,"hi":null,"n":5753,"of":null},{"category":"robustness/shuffle","metric":"candidate_ece","value":0.039063116139574534,"lo":null,"hi":null,"n":5753,"of":null},{"category":"robustness/alias","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/alias","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/alias","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/alias","metric":"candidate_accuracy","value":0.848531684698609,"lo":null,"hi":null,"n":647,"of":null},{"category":"robustness/alias","metric":"candidate_ece","value":0.059081201299080746,"lo":null,"hi":null,"n":647,"of":null},{"category":"robustness/swap","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/swap","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/swap","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/swap","metric":"candidate_accuracy","value":0.8469860896445132,"lo":null,"hi":null,"n":647,"of":null},{"category":"robustness/swap","metric":"candidate_ece","value":0.05197275265291265,"lo":null,"hi":null,"n":647,"of":null},{"category":"robustness/subset","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/subset","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/subset","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/subset","metric":"candidate_accuracy","value":0.9490566037735849,"lo":null,"hi":null,"n":5300,"of":null},{"category":"robustness/subset","metric":"candidate_ece","value":0.018249458639689256,"lo":null,"hi":null,"n":5300,"of":null},{"category":"robustness/near","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/near","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/near","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/near","metric":"candidate_accuracy","value":0.9233962264150943,"lo":null,"hi":null,"n":5300,"of":null},{"category":"robustness/near","metric":"candidate_ece","value":0.023286296534088497,"lo":null,"hi":null,"n":5300,"of":null},{"category":"parity","metric":"pass","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"parity","metric":"labels","value":108,"lo":null,"hi":null,"n":null,"of":null},{"category":"parity","metric":"equal","value":108,"lo":null,"hi":null,"n":null,"of":null},{"category":"parity","metric":"logit","value":0.0000286102294921875,"lo":null,"hi":null,"n":null,"of":null},{"category":"parity","metric":"bound","value":0.0001,"lo":null,"hi":null,"n":null,"of":null}],"notes":"tolerance accuracy 0.02, ece 0.03; gate sha256:67beae53f262b02808776065230c36d25b6031e117070254886e2934f52d6581","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"gate.laya-drawn-a8.0cfeef05-kai","benchmark":"gate.laya","split":"drawn","system":"kai","version":"a8@0cfeef05c879d7f24bbde275a6c23e85e80b6baf06cf9311d529010688d4a697","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"11474208ce49e6834bad21db4af8bb7ddf33b0ada6138b3661938522cafc815f","store_digest":"","facts_digest":"","commit":"a1a4a73d524cecf8038e30c408ad30b66e71d490","questions":null,"answered":null,"when":"","ended":"2026-09-28T16:33:11Z","by":"","completion":"unknown","measures":[{"category":"all","metric":"pass","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"suites","value":62,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"candidate","value":0.8932258064516131,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"baseline","value":0.43848387096774183,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"delta","value":0.45474193548387126,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"regressions","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"cases","value":100,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/jev.ag_news","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/jev.ag_news","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/jev.ag_news","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/jev.ag_news","metric":"candidate_accuracy","value":0.96,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/jev.ag_news","metric":"candidate_ece","value":0.03911538153886794,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/jev.ag_news","metric":"baseline_accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/jev.ag_news","metric":"baseline_ece","value":0.04963111966848367,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/jev.emotion","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/jev.emotion","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/jev.emotion","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/jev.emotion","metric":"candidate_accuracy","value":0.95,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/jev.emotion","metric":"candidate_ece","value":0.042020905315876,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/jev.emotion","metric":"baseline_accuracy","value":0.63,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/jev.emotion","metric":"baseline_ece","value":0.29464753001928334,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/jev.banking77_full","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/jev.banking77_full","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/jev.banking77_full","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/jev.banking77_full","metric":"candidate_accuracy","value":0.89,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/jev.banking77_full","metric":"candidate_ece","value":0.09766138911247257,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/jev.banking77_full","metric":"baseline_accuracy","value":0.42,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/jev.banking77_full","metric":"baseline_ece","value":0.39392668925225743,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.support_triage","metric":"regressed","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.support_triage","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.support_triage","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.support_triage","metric":"candidate_accuracy","value":0.33,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.support_triage","metric":"candidate_ece","value":0.4952091199159622,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.support_triage","metric":"baseline_accuracy","value":0.48,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.support_triage","metric":"baseline_ece","value":0.11383459806442263,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.email_spam","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.email_spam","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.email_spam","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.email_spam","metric":"candidate_accuracy","value":0.99,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.email_spam","metric":"candidate_ece","value":0.009769555330276458,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.email_spam","metric":"baseline_accuracy","value":1,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.email_spam","metric":"baseline_ece","value":0.0038367044925689377,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.phishing","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.phishing","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.phishing","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.phishing","metric":"candidate_accuracy","value":1,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.phishing","metric":"candidate_ece","value":0.0000613886117935003,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.phishing","metric":"baseline_accuracy","value":1,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.phishing","metric":"baseline_ece","value":0.010809252858161901,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.guardrails_jailbreak","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.guardrails_jailbreak","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.guardrails_jailbreak","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.guardrails_jailbreak","metric":"candidate_accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.guardrails_jailbreak","metric":"candidate_ece","value":0.05118291318416589,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.guardrails_jailbreak","metric":"baseline_accuracy","value":0.7,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.guardrails_jailbreak","metric":"baseline_ece","value":0.2747894221544265,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.moderation_toxicity","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.moderation_toxicity","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.moderation_toxicity","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.moderation_toxicity","metric":"candidate_accuracy","value":0.81,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.moderation_toxicity","metric":"candidate_ece","value":0.16926764965057373,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.moderation_toxicity","metric":"baseline_accuracy","value":0.6,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.moderation_toxicity","metric":"baseline_ece","value":0.2243971598148346,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.rag_relevance","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.rag_relevance","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.rag_relevance","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.rag_relevance","metric":"candidate_accuracy","value":0.57,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.rag_relevance","metric":"candidate_ece","value":0.12333745658397677,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.rag_relevance","metric":"baseline_accuracy","value":0.48,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.rag_relevance","metric":"baseline_ece","value":0.26150231361389165,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.model_routing_domain","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.model_routing_domain","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.model_routing_domain","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.model_routing_domain","metric":"candidate_accuracy","value":1,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.model_routing_domain","metric":"candidate_ece","value":0.003844951391220098,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.model_routing_domain","metric":"baseline_accuracy","value":0.7,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.model_routing_domain","metric":"baseline_ece","value":0.09083123058080676,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/typed_decisions","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/typed_decisions","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/typed_decisions","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/typed_decisions","metric":"candidate_accuracy","value":0.75,"lo":null,"hi":null,"n":500,"of":null},{"category":"harness/typed_decisions","metric":"candidate_ece","value":0.1543838354349136,"lo":null,"hi":null,"n":500,"of":null},{"category":"harness/typed_decisions","metric":"baseline_accuracy","value":0.746,"lo":null,"hi":null,"n":500,"of":null},{"category":"harness/typed_decisions","metric":"baseline_ece","value":0.19970310807228087,"lo":null,"hi":null,"n":500,"of":null},{"category":"harness/massive.af","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.af","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.af","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.af","metric":"candidate_accuracy","value":0.89,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.af","metric":"candidate_ece","value":0.05268498986959454,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.af","metric":"baseline_accuracy","value":0.34,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.af","metric":"baseline_ece","value":0.48447977453470237,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.am","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.am","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.am","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.am","metric":"candidate_accuracy","value":0.85,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.am","metric":"candidate_ece","value":0.11598066598176962,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.am","metric":"baseline_accuracy","value":0.15,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.am","metric":"baseline_ece","value":0.48387789905071266,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ar","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ar","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ar","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ar","metric":"candidate_accuracy","value":0.89,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ar","metric":"candidate_ece","value":0.07288791090250017,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ar","metric":"baseline_accuracy","value":0.46,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ar","metric":"baseline_ece","value":0.3150437718629837,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.az","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.az","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.az","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.az","metric":"candidate_accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.az","metric":"candidate_ece","value":0.06024763107299812,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.az","metric":"baseline_accuracy","value":0.36,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.az","metric":"baseline_ece","value":0.39286278426647187,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.bn","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.bn","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.bn","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.bn","metric":"candidate_accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.bn","metric":"candidate_ece","value":0.05731592342257498,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.bn","metric":"baseline_accuracy","value":0.45,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.bn","metric":"baseline_ece","value":0.3539693360030651,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.cy","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.cy","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.cy","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.cy","metric":"candidate_accuracy","value":0.87,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.cy","metric":"candidate_ece","value":0.08664740681648261,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.cy","metric":"baseline_accuracy","value":0.16,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.cy","metric":"baseline_ece","value":0.6107046373188496,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.da","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.da","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.da","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.da","metric":"candidate_accuracy","value":0.91,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.da","metric":"candidate_ece","value":0.053155619204044345,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.da","metric":"baseline_accuracy","value":0.46,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.da","metric":"baseline_ece","value":0.45181164443492894,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.de","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.de","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.de","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.de","metric":"candidate_accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.de","metric":"candidate_ece","value":0.05135965853929523,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.de","metric":"baseline_accuracy","value":0.53,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.de","metric":"baseline_ece","value":0.30368388086557385,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.el","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.el","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.el","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.el","metric":"candidate_accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.el","metric":"candidate_ece","value":0.0822948729991913,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.el","metric":"baseline_accuracy","value":0.44,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.el","metric":"baseline_ece","value":0.4296881093084812,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.en","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.en","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.en","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.en","metric":"candidate_accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.en","metric":"candidate_ece","value":0.05915291845798494,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.en","metric":"baseline_accuracy","value":0.82,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.en","metric":"baseline_ece","value":0.13824885368347167,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.es","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.es","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.es","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.es","metric":"candidate_accuracy","value":0.91,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.es","metric":"candidate_ece","value":0.06837773680686952,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.es","metric":"baseline_accuracy","value":0.53,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.es","metric":"baseline_ece","value":0.34762571096420286,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.fa","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.fa","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.fa","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.fa","metric":"candidate_accuracy","value":0.95,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.fa","metric":"candidate_ece","value":0.04708155661821374,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.fa","metric":"baseline_accuracy","value":0.51,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.fa","metric":"baseline_ece","value":0.2985081049799919,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.fi","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.fi","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.fi","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.fi","metric":"candidate_accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.fi","metric":"candidate_ece","value":0.056733679771423406,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.fi","metric":"baseline_accuracy","value":0.3,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.fi","metric":"baseline_ece","value":0.4618027746677399,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.fr","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.fr","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.fr","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.fr","metric":"candidate_accuracy","value":0.9,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.fr","metric":"candidate_ece","value":0.07309922486543659,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.fr","metric":"baseline_accuracy","value":0.6,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.fr","metric":"baseline_ece","value":0.26832883805036545,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.he","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.he","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.he","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.he","metric":"candidate_accuracy","value":0.91,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.he","metric":"candidate_ece","value":0.07781283855438227,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.he","metric":"baseline_accuracy","value":0.37,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.he","metric":"baseline_ece","value":0.4185565242916346,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.hi","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.hi","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.hi","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.hi","metric":"candidate_accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.hi","metric":"candidate_ece","value":0.043011419475078645,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.hi","metric":"baseline_accuracy","value":0.46,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.hi","metric":"baseline_ece","value":0.36848925307393077,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.hu","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.hu","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.hu","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.hu","metric":"candidate_accuracy","value":0.91,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.hu","metric":"candidate_ece","value":0.08298780947923662,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.hu","metric":"baseline_accuracy","value":0.34,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.hu","metric":"baseline_ece","value":0.4373521935939789,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.hy","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.hy","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.hy","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.hy","metric":"candidate_accuracy","value":0.89,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.hy","metric":"candidate_ece","value":0.05867982119321828,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.hy","metric":"baseline_accuracy","value":0.25,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.hy","metric":"baseline_ece","value":0.5127132028341294,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.id","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.id","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.id","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.id","metric":"candidate_accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.id","metric":"candidate_ece","value":0.04446434766054152,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.id","metric":"baseline_accuracy","value":0.36,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.id","metric":"baseline_ece","value":0.48560510724782946,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.is","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.is","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.is","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.is","metric":"candidate_accuracy","value":0.88,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.is","metric":"candidate_ece","value":0.08488803684711461,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.is","metric":"baseline_accuracy","value":0.33,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.is","metric":"baseline_ece","value":0.48597560346126556,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.it","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.it","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.it","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.it","metric":"candidate_accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.it","metric":"candidate_ece","value":0.05702402949333189,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.it","metric":"baseline_accuracy","value":0.47,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.it","metric":"baseline_ece","value":0.38000325053930284,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ja","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ja","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ja","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ja","metric":"candidate_accuracy","value":0.96,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ja","metric":"candidate_ece","value":0.034759365916252095,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ja","metric":"baseline_accuracy","value":0.64,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ja","metric":"baseline_ece","value":0.23371360868215557,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.jv","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.jv","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.jv","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.jv","metric":"candidate_accuracy","value":0.88,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.jv","metric":"candidate_ece","value":0.06856640487909312,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.jv","metric":"baseline_accuracy","value":0.16,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.jv","metric":"baseline_ece","value":0.6633555415272713,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ka","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ka","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ka","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ka","metric":"candidate_accuracy","value":0.8,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ka","metric":"candidate_ece","value":0.1308203849196434,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ka","metric":"baseline_accuracy","value":0.15,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ka","metric":"baseline_ece","value":0.5422389303892852,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.km","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.km","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.km","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.km","metric":"candidate_accuracy","value":0.82,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.km","metric":"candidate_ece","value":0.14559533908963204,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.km","metric":"baseline_accuracy","value":0.2,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.km","metric":"baseline_ece","value":0.44535648457705973,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.kn","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.kn","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.kn","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.kn","metric":"candidate_accuracy","value":0.9,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.kn","metric":"candidate_ece","value":0.07564685180783273,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.kn","metric":"baseline_accuracy","value":0.3,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.kn","metric":"baseline_ece","value":0.3880462409555912,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ko","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ko","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ko","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ko","metric":"candidate_accuracy","value":0.9,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ko","metric":"candidate_ece","value":0.07660935476422309,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ko","metric":"baseline_accuracy","value":0.47,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ko","metric":"baseline_ece","value":0.31081631831824785,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.lv","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.lv","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.lv","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.lv","metric":"candidate_accuracy","value":0.89,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.lv","metric":"candidate_ece","value":0.09463837064802645,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.lv","metric":"baseline_accuracy","value":0.3,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.lv","metric":"baseline_ece","value":0.4859770181775093,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ml","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ml","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ml","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ml","metric":"candidate_accuracy","value":0.88,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ml","metric":"candidate_ece","value":0.07211435198783879,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ml","metric":"baseline_accuracy","value":0.28,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ml","metric":"baseline_ece","value":0.4433153660595417,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.mn","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.mn","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.mn","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.mn","metric":"candidate_accuracy","value":0.88,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.mn","metric":"candidate_ece","value":0.09503076672554014,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.mn","metric":"baseline_accuracy","value":0.16,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.mn","metric":"baseline_ece","value":0.5769782765209674,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ms","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ms","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ms","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ms","metric":"candidate_accuracy","value":0.9,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ms","metric":"candidate_ece","value":0.07233261793851858,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ms","metric":"baseline_accuracy","value":0.27,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ms","metric":"baseline_ece","value":0.5063951462507248,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.my","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.my","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.my","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.my","metric":"candidate_accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.my","metric":"candidate_ece","value":0.059497409313917074,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.my","metric":"baseline_accuracy","value":0.16,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.my","metric":"baseline_ece","value":0.4766131702065468,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.nb","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.nb","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.nb","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.nb","metric":"candidate_accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.nb","metric":"candidate_ece","value":0.05309729158878326,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.nb","metric":"baseline_accuracy","value":0.5,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.nb","metric":"baseline_ece","value":0.36785355895757677,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.nl","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.nl","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.nl","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.nl","metric":"candidate_accuracy","value":0.88,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.nl","metric":"candidate_ece","value":0.08047885030508038,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.nl","metric":"baseline_accuracy","value":0.4,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.nl","metric":"baseline_ece","value":0.479215635061264,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.pl","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.pl","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.pl","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.pl","metric":"candidate_accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.pl","metric":"candidate_ece","value":0.04532576993107795,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.pl","metric":"baseline_accuracy","value":0.47,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.pl","metric":"baseline_ece","value":0.37366658061742786,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.pt","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.pt","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.pt","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.pt","metric":"candidate_accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.pt","metric":"candidate_ece","value":0.04678081497550015,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.pt","metric":"baseline_accuracy","value":0.44,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.pt","metric":"baseline_ece","value":0.3956287080049515,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ro","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ro","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ro","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ro","metric":"candidate_accuracy","value":0.96,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ro","metric":"candidate_ece","value":0.03957485139369964,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ro","metric":"baseline_accuracy","value":0.37,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ro","metric":"baseline_ece","value":0.472951978892088,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ru","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ru","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ru","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ru","metric":"candidate_accuracy","value":0.95,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ru","metric":"candidate_ece","value":0.04580723613500593,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ru","metric":"baseline_accuracy","value":0.57,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ru","metric":"baseline_ece","value":0.2959438425302505,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sl","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.sl","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.sl","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.sl","metric":"candidate_accuracy","value":0.88,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sl","metric":"candidate_ece","value":0.06133252531290058,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sl","metric":"baseline_accuracy","value":0.33,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sl","metric":"baseline_ece","value":0.48708411037921906,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sq","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.sq","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.sq","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.sq","metric":"candidate_accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sq","metric":"candidate_ece","value":0.054145977795124055,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sq","metric":"baseline_accuracy","value":0.29,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sq","metric":"baseline_ece","value":0.4996085746586323,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sv","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.sv","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.sv","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.sv","metric":"candidate_accuracy","value":0.91,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sv","metric":"candidate_ece","value":0.06203487917780875,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sv","metric":"baseline_accuracy","value":0.48,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sv","metric":"baseline_ece","value":0.3698970249295235,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sw","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.sw","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.sw","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.sw","metric":"candidate_accuracy","value":0.85,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sw","metric":"candidate_ece","value":0.0911237014085054,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sw","metric":"baseline_accuracy","value":0.13,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sw","metric":"baseline_ece","value":0.6278604088723658,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ta","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ta","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ta","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ta","metric":"candidate_accuracy","value":0.89,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ta","metric":"candidate_ece","value":0.06441951379179953,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ta","metric":"baseline_accuracy","value":0.31,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ta","metric":"baseline_ece","value":0.4479472939670086,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.te","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.te","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.te","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.te","metric":"candidate_accuracy","value":0.87,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.te","metric":"candidate_ece","value":0.08435814142227176,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.te","metric":"baseline_accuracy","value":0.22,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.te","metric":"baseline_ece","value":0.4968701080977917,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.th","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.th","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.th","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.th","metric":"candidate_accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.th","metric":"candidate_ece","value":0.044177895784378036,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.th","metric":"baseline_accuracy","value":0.48,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.th","metric":"baseline_ece","value":0.356314766779542,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.tl","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.tl","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.tl","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.tl","metric":"candidate_accuracy","value":0.88,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.tl","metric":"candidate_ece","value":0.08951716601848603,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.tl","metric":"baseline_accuracy","value":0.3,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.tl","metric":"baseline_ece","value":0.49216231718659403,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.tr","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.tr","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.tr","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.tr","metric":"candidate_accuracy","value":0.87,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.tr","metric":"candidate_ece","value":0.10064284861087798,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.tr","metric":"baseline_accuracy","value":0.41,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.tr","metric":"baseline_ece","value":0.3854488131403923,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ur","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ur","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ur","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ur","metric":"candidate_accuracy","value":0.91,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ur","metric":"candidate_ece","value":0.06861960574984545,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ur","metric":"baseline_accuracy","value":0.42,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ur","metric":"baseline_ece","value":0.33551987260580063,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.vi","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.vi","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.vi","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.vi","metric":"candidate_accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.vi","metric":"candidate_ece","value":0.04812568008899687,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.vi","metric":"baseline_accuracy","value":0.33,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.vi","metric":"baseline_ece","value":0.4669697824120522,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.zh-CN","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.zh-CN","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.zh-CN","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.zh-CN","metric":"candidate_accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.zh-CN","metric":"candidate_ece","value":0.05050742074847214,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.zh-CN","metric":"baseline_accuracy","value":0.65,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.zh-CN","metric":"baseline_ece","value":0.2191513399779796,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.zh-TW","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.zh-TW","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.zh-TW","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.zh-TW","metric":"candidate_accuracy","value":0.96,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.zh-TW","metric":"candidate_ece","value":0.03419531375169753,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.zh-TW","metric":"baseline_accuracy","value":0.61,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.zh-TW","metric":"baseline_ece","value":0.2661561845242977,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/clinc150","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/clinc150","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/clinc150","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/clinc150","metric":"candidate_accuracy","value":0.78,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/clinc150","metric":"candidate_ece","value":0.15651181817054743,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/sst5","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/sst5","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/sst5","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/sst5","metric":"candidate_accuracy","value":0.5,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/sst5","metric":"candidate_ece","value":0.1442073956131935,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.ar","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.ar","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.ar","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.ar","metric":"candidate_accuracy","value":0.62,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.ar","metric":"candidate_ece","value":0.13826530635356904,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.bg","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.bg","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.bg","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.bg","metric":"candidate_accuracy","value":0.76,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.bg","metric":"candidate_ece","value":0.12432977795600889,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.de","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.de","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.de","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.de","metric":"candidate_accuracy","value":0.76,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.de","metric":"candidate_ece","value":0.12723101794719702,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.el","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.el","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.el","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.el","metric":"candidate_accuracy","value":0.73,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.el","metric":"candidate_ece","value":0.11346075385808947,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.en","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.en","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.en","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.en","metric":"candidate_accuracy","value":0.76,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.en","metric":"candidate_ece","value":0.12963819324970244,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.es","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.es","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.es","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.es","metric":"candidate_accuracy","value":0.77,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.es","metric":"candidate_ece","value":0.15921563208103184,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.fr","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.fr","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.fr","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.fr","metric":"candidate_accuracy","value":0.79,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.fr","metric":"candidate_ece","value":0.14418751090764997,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.hi","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.hi","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.hi","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.hi","metric":"candidate_accuracy","value":0.59,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.hi","metric":"candidate_ece","value":0.05843946486711503,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.ru","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.ru","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.ru","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.ru","metric":"candidate_accuracy","value":0.72,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.ru","metric":"candidate_ece","value":0.11137971103191374,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.sw","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.sw","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.sw","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.sw","metric":"candidate_accuracy","value":0.58,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.sw","metric":"candidate_ece","value":0.12170465260744094,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.th","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.th","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.th","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.th","metric":"candidate_accuracy","value":0.61,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.th","metric":"candidate_ece","value":0.1322586926817894,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.tr","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.tr","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.tr","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.tr","metric":"candidate_accuracy","value":0.72,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.tr","metric":"candidate_ece","value":0.12287913411855698,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.ur","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.ur","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.ur","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.ur","metric":"candidate_accuracy","value":0.5,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.ur","metric":"candidate_ece","value":0.1103538781404495,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.vi","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.vi","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.vi","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.vi","metric":"candidate_accuracy","value":0.71,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.vi","metric":"candidate_ece","value":0.11963065892457966,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.zh","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.zh","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.zh","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.zh","metric":"candidate_accuracy","value":0.67,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.zh","metric":"candidate_ece","value":0.10528100371360781,"lo":null,"hi":null,"n":100,"of":null},{"category":"robustness/shuffle","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/shuffle","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/shuffle","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/shuffle","metric":"candidate_accuracy","value":0.8932730749174344,"lo":null,"hi":null,"n":5753,"of":null},{"category":"robustness/shuffle","metric":"candidate_ece","value":0.04835260770055868,"lo":null,"hi":null,"n":5753,"of":null},{"category":"robustness/alias","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/alias","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/alias","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/alias","metric":"candidate_accuracy","value":0.848531684698609,"lo":null,"hi":null,"n":647,"of":null},{"category":"robustness/alias","metric":"candidate_ece","value":0.04920363361721247,"lo":null,"hi":null,"n":647,"of":null},{"category":"robustness/swap","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/swap","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/swap","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/swap","metric":"candidate_accuracy","value":0.8500772797527048,"lo":null,"hi":null,"n":647,"of":null},{"category":"robustness/swap","metric":"candidate_ece","value":0.05356464936891703,"lo":null,"hi":null,"n":647,"of":null},{"category":"robustness/subset","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/subset","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/subset","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/subset","metric":"candidate_accuracy","value":0.9507547169811321,"lo":null,"hi":null,"n":5300,"of":null},{"category":"robustness/subset","metric":"candidate_ece","value":0.018969453463576848,"lo":null,"hi":null,"n":5300,"of":null},{"category":"robustness/near","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/near","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/near","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/near","metric":"candidate_accuracy","value":0.9222641509433962,"lo":null,"hi":null,"n":5300,"of":null},{"category":"robustness/near","metric":"candidate_ece","value":0.029289239268257973,"lo":null,"hi":null,"n":5300,"of":null},{"category":"parity","metric":"pass","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"parity","metric":"labels","value":108,"lo":null,"hi":null,"n":null,"of":null},{"category":"parity","metric":"equal","value":108,"lo":null,"hi":null,"n":null,"of":null},{"category":"parity","metric":"logit","value":0.0000286102294921875,"lo":null,"hi":null,"n":null,"of":null},{"category":"parity","metric":"bound","value":0.0001,"lo":null,"hi":null,"n":null,"of":null}],"notes":"tolerance accuracy 0.02, ece 0.03; gate sha256:b58055bcea72b8fb0f083115586ee46a0f16a47942d20da91a7eaa3d5be38282","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"jev.ag_news-harness-20260917-jev","benchmark":"jev.ag_news","split":"harness","system":"jev","version":"typesafe/jev-1.13-20260917","baseline":true,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"e95c6ee6362fd66d6d80c711095e03f30bbeeffd","questions":400,"answered":400,"when":"","ended":"2026-09-25T02:55:39Z","by":"","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.86,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"brier","value":0.2323,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"ece","value":0.1023,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"aurc","value":0.0832,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"zero_prob","value":0.0575,"lo":null,"hi":null,"n":400,"of":400}],"notes":"preds sha256:9bafe8f26aecc4d0f77933b073a226aa21b0c509add1e8807b2540cfe61b1971","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"jev.ag_news-harness-a5.df1c16a2-kai","benchmark":"jev.ag_news","split":"harness","system":"kai","version":"a5@df1c16a2fef7ec93f2b34a6a194f6dc3c2eee70e936473b802151543c5d46e83","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"611059b60f64ab686d6475cd76778b3fac52cd98","questions":400,"answered":400,"when":"","ended":"2026-09-27T21:09:54Z","by":"","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.95,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"brier","value":0.0992,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"ece","value":0.0702,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"aurc","value":0.0102,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":400,"of":400}],"notes":"device metal; dtype bf16; preds sha256:f28954731ec991ecbed3c1fbfc45980eeffef7744d8c12bcfc4cb160cb6f3eac; scores sha256:071b556a8eafd2c19d4f9a948b4f10251c9b253107c6afb21ad959ef72af4a77","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"jev.ag_news-harness-a6.a211cc70-kai","benchmark":"jev.ag_news","split":"harness","system":"kai","version":"a6@a211cc70103825bfbc9bfb0fbe970f8bf08da2efe0953cd8045a3cd07397f4bf","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"a1a4a73d524cecf8038e30c408ad30b66e71d490","questions":400,"answered":400,"when":"","ended":"2026-09-28T03:18:42Z","by":"","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.94,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"brier","value":0.0932,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"ece","value":0.0282,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"aurc","value":0.0097,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":400,"of":400}],"notes":"device metal; dtype bf16; preds sha256:43b1bd476a72a536ee96fbc934a065a808fdb35395e694d9af873e462b3d8866; scores sha256:3760df4b2c4cdcccb225bbb446b2ddcff998e09ab723e36ee466f771dc355a7e","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"jev.ag_news-harness-a7.0834a74f-kai","benchmark":"jev.ag_news","split":"harness","system":"kai","version":"a7@0834a74f2d140642a453373da09e5a128e3d2c8d9e8bb1dcaf86d7210a4ecdfc","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"a1a4a73d524cecf8038e30c408ad30b66e71d490","questions":400,"answered":400,"when":"","ended":"2026-09-28T12:24:45Z","by":"","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.9425,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"brier","value":0.0907,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"ece","value":0.0336,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"aurc","value":0.0079,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":400,"of":400}],"notes":"device metal; dtype bf16; preds sha256:25bfec2f5ad49bc050cc9015161c7ef2dbf07f599f35e9dad153b4a5b6f7b543; scores sha256:ddca48bcf9816e7df2a187251d9f0f00de3b2fc24f22805b34b7bd650a048c81","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"jev.ag_news-harness-a8.0cfeef05-kai","benchmark":"jev.ag_news","split":"harness","system":"kai","version":"a8@0cfeef05c879d7f24bbde275a6c23e85e80b6baf06cf9311d529010688d4a697","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"a1a4a73d524cecf8038e30c408ad30b66e71d490","questions":400,"answered":400,"when":"","ended":"2026-09-28T16:33:11Z","by":"","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.9425,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"brier","value":0.0909,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"ece","value":0.024,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"aurc","value":0.0084,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":400,"of":400}],"notes":"device metal; dtype bf16; preds sha256:d0a81ccbdc9eb7305fc196d7ad2c6be8c1df1ce5966ac964dabeeea231ae41d6; scores sha256:86b58e1f53447a8fb3b7963ea69f2bee40ffa97b9bda7b74b038064188998f10","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"jev.ag_news-harness-english-laya","benchmark":"jev.ag_news","split":"harness","system":"laya","version":"convaiinnovations/laya@55cf4c4ebb4ebe31b2550e8bdf3bd21b99753851","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"735e240323cd7c9f0c880b7d8c62d89f40be2164","questions":400,"answered":400,"when":"","ended":"2026-09-25T17:33:56Z","by":"","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.95,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"brier","value":0.0808,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"ece","value":0.0316,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"aurc","value":0.0067,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":400,"of":400}],"notes":"runtime laya 0.3.20; device cpu; dtype f32; preds sha256:f59f9f2498a9eebab040635371bdbd85b2eb8267f5c7bcbfb5d4e74866a323e4","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"jev.ag_news-harness-kai-1.0834a74f-kai","benchmark":"jev.ag_news","split":"harness","system":"kai","version":"kai-1@0834a74f2d140642a453373da09e5a128e3d2c8d9e8bb1dcaf86d7210a4ecdfc","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"2c74e2d61addc885e847358975fb866b03cf17dc","questions":400,"answered":400,"when":"","ended":"2026-10-03T00:06:45Z","by":"","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.9425,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"brier","value":0.0908,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"ece","value":0.0332,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"aurc","value":0.008,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":400,"of":400}],"notes":"device cuda; dtype bf16; preds sha256:63708a6eba59d6289ada94f33ee52886b8a99208a1be71d51af033a5e42426de; scores sha256:1ec811cfb085ac209c7b094b408b00ef95d3d3c73b7ae7bdf6c52fc314d112e0","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"jev.ag_news-harness-multilingual-laya","benchmark":"jev.ag_news","split":"harness","system":"laya","version":"convaiinnovations/laya-multilingual@e4e9ddf21a7b1903b7acffd8814ad4307bf63a67","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"735e240323cd7c9f0c880b7d8c62d89f40be2164","questions":400,"answered":400,"when":"","ended":"2026-09-25T17:33:56Z","by":"","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.935,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"brier","value":0.0978,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"ece","value":0.038,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"aurc","value":0.0192,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":400,"of":400}],"notes":"runtime laya 0.3.20; device cpu; dtype f32; preds sha256:f59f9f2498a9eebab040635371bdbd85b2eb8267f5c7bcbfb5d4e74866a323e4","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"jev.ag_news-harness-routed-laya","benchmark":"jev.ag_news","split":"harness","system":"laya","version":"convaiinnovations/laya@55cf4c4ebb4ebe31b2550e8bdf3bd21b99753851+convaiinnovations/laya-multilingual@e4e9ddf21a7b1903b7acffd8814ad4307bf63a67","baseline":true,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"735e240323cd7c9f0c880b7d8c62d89f40be2164","questions":400,"answered":400,"when":"","ended":"2026-09-25T17:33:56Z","by":"","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.95,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"brier","value":0.0808,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"ece","value":0.0315,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"aurc","value":0.0067,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":400,"of":400}],"notes":"runtime laya 0.3.20; device cpu; dtype f32; preds sha256:f59f9f2498a9eebab040635371bdbd85b2eb8267f5c7bcbfb5d4e74866a323e4","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"jev.ag_news-harness-typed-decisions-laya","benchmark":"jev.ag_news","split":"harness","system":"laya","version":"convaiinnovations/laya-typed-decisions@1a793eb568e6718f15941d08f85432581df534e3","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"735e240323cd7c9f0c880b7d8c62d89f40be2164","questions":400,"answered":400,"when":"","ended":"2026-09-25T17:33:56Z","by":"","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.9525,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"brier","value":0.1137,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"ece","value":0.155,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"aurc","value":0.006,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":400,"of":400}],"notes":"runtime laya 0.3.20; device cpu; dtype f32; preds sha256:f59f9f2498a9eebab040635371bdbd85b2eb8267f5c7bcbfb5d4e74866a323e4","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"jev.banking77_full-harness-20260917-jev","benchmark":"jev.banking77_full","split":"harness","system":"jev","version":"typesafe/jev-1.13-20260917","baseline":true,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"e95c6ee6362fd66d6d80c711095e03f30bbeeffd","questions":400,"answered":400,"when":"","ended":"2026-09-25T02:55:39Z","by":"","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.835,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"brier","value":0.257,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"ece","value":0.0729,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"aurc","value":0.0579,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"zero_prob","value":0.035,"lo":null,"hi":null,"n":400,"of":400}],"notes":"preds sha256:ac2f1566f1d2e00ba3beb42d6f6a727a4b84a1efab7d359e722abfafd7062eba","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"jev.banking77_full-harness-a5.df1c16a2-kai","benchmark":"jev.banking77_full","split":"harness","system":"kai","version":"a5@df1c16a2fef7ec93f2b34a6a194f6dc3c2eee70e936473b802151543c5d46e83","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"611059b60f64ab686d6475cd76778b3fac52cd98","questions":400,"answered":400,"when":"","ended":"2026-09-27T21:09:54Z","by":"","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.9075,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"brier","value":0.1549,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"ece","value":0.0649,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"aurc","value":0.024,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":400,"of":400}],"notes":"device metal; dtype bf16; preds sha256:f28954731ec991ecbed3c1fbfc45980eeffef7744d8c12bcfc4cb160cb6f3eac; scores sha256:071b556a8eafd2c19d4f9a948b4f10251c9b253107c6afb21ad959ef72af4a77","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"jev.banking77_full-harness-a6.a211cc70-kai","benchmark":"jev.banking77_full","split":"harness","system":"kai","version":"a6@a211cc70103825bfbc9bfb0fbe970f8bf08da2efe0953cd8045a3cd07397f4bf","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"a1a4a73d524cecf8038e30c408ad30b66e71d490","questions":400,"answered":400,"when":"","ended":"2026-09-28T03:18:42Z","by":"","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.9225,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"brier","value":0.1437,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"ece","value":0.061,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"aurc","value":0.0211,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"zero_prob","value":0.015,"lo":null,"hi":null,"n":400,"of":400}],"notes":"device metal; dtype bf16; preds sha256:43b1bd476a72a536ee96fbc934a065a808fdb35395e694d9af873e462b3d8866; scores sha256:3760df4b2c4cdcccb225bbb446b2ddcff998e09ab723e36ee466f771dc355a7e","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"jev.banking77_full-harness-a7.0834a74f-kai","benchmark":"jev.banking77_full","split":"harness","system":"kai","version":"a7@0834a74f2d140642a453373da09e5a128e3d2c8d9e8bb1dcaf86d7210a4ecdfc","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"a1a4a73d524cecf8038e30c408ad30b66e71d490","questions":400,"answered":400,"when":"","ended":"2026-09-28T12:24:45Z","by":"","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.9075,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"brier","value":0.165,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"ece","value":0.0758,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"aurc","value":0.0236,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"zero_prob","value":0.015,"lo":null,"hi":null,"n":400,"of":400}],"notes":"device metal; dtype bf16; preds sha256:25bfec2f5ad49bc050cc9015161c7ef2dbf07f599f35e9dad153b4a5b6f7b543; scores sha256:ddca48bcf9816e7df2a187251d9f0f00de3b2fc24f22805b34b7bd650a048c81","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"jev.banking77_full-harness-a8.0cfeef05-kai","benchmark":"jev.banking77_full","split":"harness","system":"kai","version":"a8@0cfeef05c879d7f24bbde275a6c23e85e80b6baf06cf9311d529010688d4a697","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"a1a4a73d524cecf8038e30c408ad30b66e71d490","questions":400,"answered":400,"when":"","ended":"2026-09-28T16:33:11Z","by":"","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.8975,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"brier","value":0.1772,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"ece","value":0.0851,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"aurc","value":0.0216,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"zero_prob","value":0.0275,"lo":null,"hi":null,"n":400,"of":400}],"notes":"device metal; dtype bf16; preds sha256:d0a81ccbdc9eb7305fc196d7ad2c6be8c1df1ce5966ac964dabeeea231ae41d6; scores sha256:86b58e1f53447a8fb3b7963ea69f2bee40ffa97b9bda7b74b038064188998f10","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"jev.banking77_full-harness-english-laya","benchmark":"jev.banking77_full","split":"harness","system":"laya","version":"convaiinnovations/laya@55cf4c4ebb4ebe31b2550e8bdf3bd21b99753851","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"735e240323cd7c9f0c880b7d8c62d89f40be2164","questions":400,"answered":400,"when":"","ended":"2026-09-25T17:33:56Z","by":"","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.425,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"brier","value":0.9246,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"ece","value":0.3809,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"aurc","value":0.3559,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"zero_prob","value":0.0975,"lo":null,"hi":null,"n":400,"of":400}],"notes":"runtime laya 0.3.20; device cpu; dtype f32; preds sha256:d2a9fb23e2f1df20c2602de33a25ca82d48b9d6b9a2e5ab9a758f384ae73abc6","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"jev.banking77_full-harness-kai-1.0834a74f-kai","benchmark":"jev.banking77_full","split":"harness","system":"kai","version":"kai-1@0834a74f2d140642a453373da09e5a128e3d2c8d9e8bb1dcaf86d7210a4ecdfc","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"2c74e2d61addc885e847358975fb866b03cf17dc","questions":400,"answered":400,"when":"","ended":"2026-10-03T00:06:45Z","by":"","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.9075,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"brier","value":0.1647,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"ece","value":0.0743,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"aurc","value":0.0264,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"zero_prob","value":0.015,"lo":null,"hi":null,"n":400,"of":400}],"notes":"device cuda; dtype bf16; preds sha256:d4e8118ae65a4f9b309bedf919fc3d2792d80a5e987ae16feae32d8b29c57620; scores sha256:1ec811cfb085ac209c7b094b408b00ef95d3d3c73b7ae7bdf6c52fc314d112e0","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"jev.banking77_full-harness-multilingual-laya","benchmark":"jev.banking77_full","split":"harness","system":"laya","version":"convaiinnovations/laya-multilingual@e4e9ddf21a7b1903b7acffd8814ad4307bf63a67","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"735e240323cd7c9f0c880b7d8c62d89f40be2164","questions":400,"answered":400,"when":"","ended":"2026-09-25T17:33:56Z","by":"","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.47,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"brier","value":0.8859,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"ece","value":0.3851,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"aurc","value":0.3111,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"zero_prob","value":0.07,"lo":null,"hi":null,"n":400,"of":400}],"notes":"runtime laya 0.3.20; device cpu; dtype f32; preds sha256:d2a9fb23e2f1df20c2602de33a25ca82d48b9d6b9a2e5ab9a758f384ae73abc6","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"jev.banking77_full-harness-routed-laya","benchmark":"jev.banking77_full","split":"harness","system":"laya","version":"convaiinnovations/laya@55cf4c4ebb4ebe31b2550e8bdf3bd21b99753851+convaiinnovations/laya-multilingual@e4e9ddf21a7b1903b7acffd8814ad4307bf63a67","baseline":true,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"735e240323cd7c9f0c880b7d8c62d89f40be2164","questions":400,"answered":400,"when":"","ended":"2026-09-25T17:33:56Z","by":"","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.425,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"brier","value":0.9246,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"ece","value":0.3809,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"aurc","value":0.3559,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"zero_prob","value":0.0975,"lo":null,"hi":null,"n":400,"of":400}],"notes":"runtime laya 0.3.20; device cpu; dtype f32; preds sha256:d2a9fb23e2f1df20c2602de33a25ca82d48b9d6b9a2e5ab9a758f384ae73abc6","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"jev.banking77_full-harness-typed-decisions-laya","benchmark":"jev.banking77_full","split":"harness","system":"laya","version":"convaiinnovations/laya-typed-decisions@1a793eb568e6718f15941d08f85432581df534e3","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"735e240323cd7c9f0c880b7d8c62d89f40be2164","questions":400,"answered":400,"when":"","ended":"2026-09-25T17:33:56Z","by":"","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.4925,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"brier","value":0.6744,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"ece","value":0.0946,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"aurc","value":0.2984,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":400,"of":400}],"notes":"runtime laya 0.3.20; device cpu; dtype f32; preds sha256:d2a9fb23e2f1df20c2602de33a25ca82d48b9d6b9a2e5ab9a758f384ae73abc6","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"jev.emotion-harness-20260917-jev","benchmark":"jev.emotion","split":"harness","system":"jev","version":"typesafe/jev-1.13-20260917","baseline":true,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"e95c6ee6362fd66d6d80c711095e03f30bbeeffd","questions":400,"answered":400,"when":"","ended":"2026-09-25T02:55:39Z","by":"","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.6025,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"brier","value":0.6464,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"ece","value":0.2795,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"aurc","value":0.2311,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"zero_prob","value":0.1425,"lo":null,"hi":null,"n":400,"of":400}],"notes":"preds sha256:bd949448efbeeed4ef0f1e710fb63b63b8415063a4fa3b9421b3eb37238fa39b","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"jev.emotion-harness-a5.df1c16a2-kai","benchmark":"jev.emotion","split":"harness","system":"kai","version":"a5@df1c16a2fef7ec93f2b34a6a194f6dc3c2eee70e936473b802151543c5d46e83","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"611059b60f64ab686d6475cd76778b3fac52cd98","questions":400,"answered":400,"when":"","ended":"2026-09-27T21:09:54Z","by":"","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.925,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"brier","value":0.0895,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"ece","value":0.041,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"aurc","value":0.0062,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":400,"of":400}],"notes":"device metal; dtype bf16; preds sha256:f28954731ec991ecbed3c1fbfc45980eeffef7744d8c12bcfc4cb160cb6f3eac; scores sha256:071b556a8eafd2c19d4f9a948b4f10251c9b253107c6afb21ad959ef72af4a77","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"jev.emotion-harness-a6.a211cc70-kai","benchmark":"jev.emotion","split":"harness","system":"kai","version":"a6@a211cc70103825bfbc9bfb0fbe970f8bf08da2efe0953cd8045a3cd07397f4bf","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"a1a4a73d524cecf8038e30c408ad30b66e71d490","questions":400,"answered":400,"when":"","ended":"2026-09-28T03:18:42Z","by":"","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.935,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"brier","value":0.0806,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"ece","value":0.0249,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"aurc","value":0.0048,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":400,"of":400}],"notes":"device metal; dtype bf16; preds sha256:43b1bd476a72a536ee96fbc934a065a808fdb35395e694d9af873e462b3d8866; scores sha256:3760df4b2c4cdcccb225bbb446b2ddcff998e09ab723e36ee466f771dc355a7e","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"jev.emotion-harness-a7.0834a74f-kai","benchmark":"jev.emotion","split":"harness","system":"kai","version":"a7@0834a74f2d140642a453373da09e5a128e3d2c8d9e8bb1dcaf86d7210a4ecdfc","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"a1a4a73d524cecf8038e30c408ad30b66e71d490","questions":400,"answered":400,"when":"","ended":"2026-09-28T12:24:45Z","by":"","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.935,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"brier","value":0.08,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"ece","value":0.0188,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"aurc","value":0.0045,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":400,"of":400}],"notes":"device metal; dtype bf16; preds sha256:25bfec2f5ad49bc050cc9015161c7ef2dbf07f599f35e9dad153b4a5b6f7b543; scores sha256:ddca48bcf9816e7df2a187251d9f0f00de3b2fc24f22805b34b7bd650a048c81","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"jev.emotion-harness-a8.0cfeef05-kai","benchmark":"jev.emotion","split":"harness","system":"kai","version":"a8@0cfeef05c879d7f24bbde275a6c23e85e80b6baf06cf9311d529010688d4a697","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"a1a4a73d524cecf8038e30c408ad30b66e71d490","questions":400,"answered":400,"when":"","ended":"2026-09-28T16:33:11Z","by":"","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.9375,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"brier","value":0.0796,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"ece","value":0.0234,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"aurc","value":0.0057,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":400,"of":400}],"notes":"device metal; dtype bf16; preds sha256:d0a81ccbdc9eb7305fc196d7ad2c6be8c1df1ce5966ac964dabeeea231ae41d6; scores sha256:86b58e1f53447a8fb3b7963ea69f2bee40ffa97b9bda7b74b038064188998f10","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"jev.emotion-harness-english-laya","benchmark":"jev.emotion","split":"harness","system":"laya","version":"convaiinnovations/laya@55cf4c4ebb4ebe31b2550e8bdf3bd21b99753851","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"735e240323cd7c9f0c880b7d8c62d89f40be2164","questions":400,"answered":400,"when":"","ended":"2026-09-25T17:33:56Z","by":"","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.595,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"brier","value":0.6963,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"ece","value":0.3063,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"aurc","value":0.2655,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":400,"of":400}],"notes":"runtime laya 0.3.20; device cpu; dtype f32; preds sha256:d52672beecb1c074cb47dca968ddea0ac3bf9d8183d87369c249f29a77129740","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"jev.emotion-harness-kai-1.0834a74f-kai","benchmark":"jev.emotion","split":"harness","system":"kai","version":"kai-1@0834a74f2d140642a453373da09e5a128e3d2c8d9e8bb1dcaf86d7210a4ecdfc","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"2c74e2d61addc885e847358975fb866b03cf17dc","questions":400,"answered":400,"when":"","ended":"2026-10-03T00:06:45Z","by":"","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.935,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"brier","value":0.08,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"ece","value":0.0182,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"aurc","value":0.0046,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":400,"of":400}],"notes":"device cuda; dtype bf16; preds sha256:eb57f32a8e40650a18ad2654bcddf6688e2901e7cc4ab65d81275bf55b44e3c6; scores sha256:1ec811cfb085ac209c7b094b408b00ef95d3d3c73b7ae7bdf6c52fc314d112e0","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"jev.emotion-harness-multilingual-laya","benchmark":"jev.emotion","split":"harness","system":"laya","version":"convaiinnovations/laya-multilingual@e4e9ddf21a7b1903b7acffd8814ad4307bf63a67","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"735e240323cd7c9f0c880b7d8c62d89f40be2164","questions":400,"answered":400,"when":"","ended":"2026-09-25T17:33:56Z","by":"","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.5375,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"brier","value":0.7802,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"ece","value":0.3323,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"aurc","value":0.3246,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":400,"of":400}],"notes":"runtime laya 0.3.20; device cpu; dtype f32; preds sha256:d52672beecb1c074cb47dca968ddea0ac3bf9d8183d87369c249f29a77129740","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"jev.emotion-harness-routed-laya","benchmark":"jev.emotion","split":"harness","system":"laya","version":"convaiinnovations/laya@55cf4c4ebb4ebe31b2550e8bdf3bd21b99753851+convaiinnovations/laya-multilingual@e4e9ddf21a7b1903b7acffd8814ad4307bf63a67","baseline":true,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"735e240323cd7c9f0c880b7d8c62d89f40be2164","questions":400,"answered":400,"when":"","ended":"2026-09-25T17:33:56Z","by":"","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.595,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"brier","value":0.6963,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"ece","value":0.3063,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"aurc","value":0.2655,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":400,"of":400}],"notes":"runtime laya 0.3.20; device cpu; dtype f32; preds sha256:d52672beecb1c074cb47dca968ddea0ac3bf9d8183d87369c249f29a77129740","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"jev.emotion-harness-typed-decisions-laya","benchmark":"jev.emotion","split":"harness","system":"laya","version":"convaiinnovations/laya-typed-decisions@1a793eb568e6718f15941d08f85432581df534e3","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"735e240323cd7c9f0c880b7d8c62d89f40be2164","questions":400,"answered":400,"when":"","ended":"2026-09-25T17:33:56Z","by":"","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.6,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"brier","value":0.6113,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"ece","value":0.2005,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"aurc","value":0.2631,"lo":null,"hi":null,"n":400,"of":400},{"category":"all","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":400,"of":400}],"notes":"runtime laya 0.3.20; device cpu; dtype f32; preds sha256:d52672beecb1c074cb47dca968ddea0ac3bf9d8183d87369c249f29a77129740","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"kai.ag_news-val:0b8b6476822b343c-a7.0834a74f-kai","benchmark":"kai.ag_news","split":"val:0b8b6476822b343c","system":"kai","version":"a7@0834a74f2d140642a453373da09e5a128e3d2c8d9e8bb1dcaf86d7210a4ecdfc","baseline":true,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"sha256:0b8b6476822b343c2c1e21bbaaa1350966cadb9ff716f7f0472ea311e57cebff","store_digest":"","facts_digest":"","commit":"","questions":2735,"answered":2735,"when":"2026-09-29T07:42:05Z","ended":"2026-09-29T07:42:18Z","by":"evo","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.950274223034735,"lo":null,"hi":null,"n":2735,"of":2735},{"category":"all","metric":"nll","value":0.1625286015884605,"lo":null,"hi":null,"n":2735,"of":2735},{"category":"all","metric":"ece","value":0.021158426083579266,"lo":null,"hi":null,"n":2735,"of":2735}],"notes":"job job_30bfc6e5af09a9f7; device rocm; dtype bf16; build 5c359f428ef2637d","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"kai.ag_news-val:0b8b6476822b343c-triage+job_30bfc6e5af09a9f7.0cac9e85-kai","benchmark":"kai.ag_news","split":"val:0b8b6476822b343c","system":"kai","version":"triage+job_30bfc6e5af09a9f7@0cac9e85f926b3a88b7176d456a084ca77238581940522dd17ab6451b61e12f5","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"sha256:0b8b6476822b343c2c1e21bbaaa1350966cadb9ff716f7f0472ea311e57cebff","store_digest":"","facts_digest":"","commit":"","questions":2735,"answered":2735,"when":"2026-09-29T07:50:09Z","ended":"2026-09-29T07:50:22Z","by":"evo","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.9477148080438756,"lo":null,"hi":null,"n":2735,"of":2735},{"category":"all","metric":"nll","value":0.1732962789175048,"lo":null,"hi":null,"n":2735,"of":2735},{"category":"all","metric":"ece","value":0.025656094317175672,"lo":null,"hi":null,"n":2735,"of":2735}],"notes":"job job_30bfc6e5af09a9f7; device rocm; dtype bf16; build 5c359f428ef2637d","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"kai.ag_news-val:0b8b6476822b343c-triage+job_cccd2fcfeae9ecb9.57535ef4-kai","benchmark":"kai.ag_news","split":"val:0b8b6476822b343c","system":"kai","version":"triage+job_cccd2fcfeae9ecb9@57535ef405f7bd9032cd90330b2fdfbd623479c04d5715fac472be1708b111be","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"sha256:0b8b6476822b343c2c1e21bbaaa1350966cadb9ff716f7f0472ea311e57cebff","store_digest":"","facts_digest":"","commit":"","questions":2735,"answered":2735,"when":"2026-09-29T05:27:21Z","ended":"2026-09-29T05:27:34Z","by":"evo","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.9455210237659963,"lo":null,"hi":null,"n":2735,"of":2735},{"category":"all","metric":"nll","value":0.1759071827288776,"lo":null,"hi":null,"n":2735,"of":2735},{"category":"all","metric":"ece","value":0.02509483155553141,"lo":null,"hi":null,"n":2735,"of":2735}],"notes":"job job_cccd2fcfeae9ecb9; device rocm; dtype bf16; build d7b503724e5adf0d","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"kai.support_triage-val:0b8b6476822b343c-a7.0834a74f-kai","benchmark":"kai.support_triage","split":"val:0b8b6476822b343c","system":"kai","version":"a7@0834a74f2d140642a453373da09e5a128e3d2c8d9e8bb1dcaf86d7210a4ecdfc","baseline":true,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"sha256:0b8b6476822b343c2c1e21bbaaa1350966cadb9ff716f7f0472ea311e57cebff","store_digest":"","facts_digest":"","commit":"","questions":518,"answered":518,"when":"2026-09-29T07:42:05Z","ended":"2026-09-29T07:42:18Z","by":"evo","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.6640926640926641,"lo":null,"hi":null,"n":518,"of":518},{"category":"all","metric":"nll","value":2.299691470318066,"lo":null,"hi":null,"n":518,"of":518},{"category":"all","metric":"ece","value":0.2618000212177394,"lo":null,"hi":null,"n":518,"of":518}],"notes":"job job_30bfc6e5af09a9f7; device rocm; dtype bf16; build 5c359f428ef2637d","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"kai.support_triage-val:0b8b6476822b343c-triage+job_30bfc6e5af09a9f7.0cac9e85-kai","benchmark":"kai.support_triage","split":"val:0b8b6476822b343c","system":"kai","version":"triage+job_30bfc6e5af09a9f7@0cac9e85f926b3a88b7176d456a084ca77238581940522dd17ab6451b61e12f5","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"sha256:0b8b6476822b343c2c1e21bbaaa1350966cadb9ff716f7f0472ea311e57cebff","store_digest":"","facts_digest":"","commit":"","questions":518,"answered":518,"when":"2026-09-29T07:50:09Z","ended":"2026-09-29T07:50:22Z","by":"evo","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.6486486486486487,"lo":null,"hi":null,"n":518,"of":518},{"category":"all","metric":"nll","value":2.974795825644426,"lo":null,"hi":null,"n":518,"of":518},{"category":"all","metric":"ece","value":0.2824697971650246,"lo":null,"hi":null,"n":518,"of":518}],"notes":"job job_30bfc6e5af09a9f7; device rocm; dtype bf16; build 5c359f428ef2637d","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"kai.support_triage-val:0b8b6476822b343c-triage+job_cccd2fcfeae9ecb9.57535ef4-kai","benchmark":"kai.support_triage","split":"val:0b8b6476822b343c","system":"kai","version":"triage+job_cccd2fcfeae9ecb9@57535ef405f7bd9032cd90330b2fdfbd623479c04d5715fac472be1708b111be","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"sha256:0b8b6476822b343c2c1e21bbaaa1350966cadb9ff716f7f0472ea311e57cebff","store_digest":"","facts_digest":"","commit":"","questions":518,"answered":518,"when":"2026-09-29T05:27:21Z","ended":"2026-09-29T05:27:34Z","by":"evo","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.6583011583011583,"lo":null,"hi":null,"n":518,"of":518},{"category":"all","metric":"nll","value":2.6301461855114128,"lo":null,"hi":null,"n":518,"of":518},{"category":"all","metric":"ece","value":0.27758026148484644,"lo":null,"hi":null,"n":518,"of":518}],"notes":"job job_cccd2fcfeae9ecb9; device rocm; dtype bf16; build d7b503724e5adf0d","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"kai.typed_decisions-val:0b8b6476822b343c-a7.0834a74f-kai","benchmark":"kai.typed_decisions","split":"val:0b8b6476822b343c","system":"kai","version":"a7@0834a74f2d140642a453373da09e5a128e3d2c8d9e8bb1dcaf86d7210a4ecdfc","baseline":true,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"sha256:0b8b6476822b343c2c1e21bbaaa1350966cadb9ff716f7f0472ea311e57cebff","store_digest":"","facts_digest":"","commit":"","questions":310,"answered":310,"when":"2026-09-29T07:42:05Z","ended":"2026-09-29T07:42:18Z","by":"evo","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.7967741935483871,"lo":null,"hi":null,"n":310,"of":310},{"category":"all","metric":"nll","value":0.8629567513161102,"lo":null,"hi":null,"n":310,"of":310},{"category":"all","metric":"ece","value":0.1790709034007661,"lo":null,"hi":null,"n":310,"of":310}],"notes":"job job_30bfc6e5af09a9f7; device rocm; dtype bf16; build 5c359f428ef2637d","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"kai.typed_decisions-val:0b8b6476822b343c-triage+job_30bfc6e5af09a9f7.0cac9e85-kai","benchmark":"kai.typed_decisions","split":"val:0b8b6476822b343c","system":"kai","version":"triage+job_30bfc6e5af09a9f7@0cac9e85f926b3a88b7176d456a084ca77238581940522dd17ab6451b61e12f5","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"sha256:0b8b6476822b343c2c1e21bbaaa1350966cadb9ff716f7f0472ea311e57cebff","store_digest":"","facts_digest":"","commit":"","questions":310,"answered":310,"when":"2026-09-29T07:50:09Z","ended":"2026-09-29T07:50:22Z","by":"evo","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.8064516129032258,"lo":null,"hi":null,"n":310,"of":310},{"category":"all","metric":"nll","value":0.8606986499437324,"lo":null,"hi":null,"n":310,"of":310},{"category":"all","metric":"ece","value":0.1821982464086091,"lo":null,"hi":null,"n":310,"of":310}],"notes":"job job_30bfc6e5af09a9f7; device rocm; dtype bf16; build 5c359f428ef2637d","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"kai.typed_decisions-val:0b8b6476822b343c-triage+job_cccd2fcfeae9ecb9.57535ef4-kai","benchmark":"kai.typed_decisions","split":"val:0b8b6476822b343c","system":"kai","version":"triage+job_cccd2fcfeae9ecb9@57535ef405f7bd9032cd90330b2fdfbd623479c04d5715fac472be1708b111be","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"sha256:0b8b6476822b343c2c1e21bbaaa1350966cadb9ff716f7f0472ea311e57cebff","store_digest":"","facts_digest":"","commit":"","questions":310,"answered":310,"when":"2026-09-29T05:27:21Z","ended":"2026-09-29T05:27:34Z","by":"evo","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.8,"lo":null,"hi":null,"n":310,"of":310},{"category":"all","metric":"nll","value":0.8621494598395679,"lo":null,"hi":null,"n":310,"of":310},{"category":"all","metric":"ece","value":0.17046724903593222,"lo":null,"hi":null,"n":310,"of":310}],"notes":"job job_cccd2fcfeae9ecb9; device rocm; dtype bf16; build d7b503724e5adf0d","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"latency-pool-a7.0834a74f-kai","benchmark":"latency","split":"pool","system":"kai","version":"a7@0834a74f2d140642a453373da09e5a128e3d2c8d9e8bb1dcaf86d7210a4ecdfc","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"","store_digest":"","facts_digest":"","commit":"a1a4a73d524cecf8038e30c408ad30b66e71d490","questions":null,"answered":null,"when":"","ended":"2026-09-28T12:09:15Z","by":"evo","completion":"unknown","measures":[{"category":"cold/rocm-bf16","metric":"p50ms","value":19.8161,"lo":null,"hi":null,"n":10,"of":null},{"category":"cold/rocm-bf16","metric":"p95ms","value":28.971,"lo":null,"hi":null,"n":10,"of":null}],"notes":"cold/rocm-bf16: encoder 768 wide, 22 layers","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"latency-pool-multilingual-laya","benchmark":"latency","split":"pool","system":"laya","version":"convaiinnovations/laya-multilingual@e4e9ddf21a7b1903b7acffd8814ad4307bf63a67","baseline":true,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"","store_digest":"","facts_digest":"","commit":"a1a4a73d524cecf8038e30c408ad30b66e71d490","questions":null,"answered":null,"when":"","ended":"2026-09-28T12:09:15Z","by":"evo","completion":"unknown","measures":[{"category":"cold/rocm-bf16","metric":"p50ms","value":16.4573,"lo":null,"hi":null,"n":10,"of":null},{"category":"cold/rocm-bf16","metric":"p95ms","value":18.3214,"lo":null,"hi":null,"n":10,"of":null}],"notes":"cold/rocm-bf16: encoder 768 wide, 22 layers","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"massive-harness-20260917-jev","benchmark":"massive","split":"harness","system":"jev","version":"typesafe/jev-1.13-20260917","baseline":true,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"e95c6ee6362fd66d6d80c711095e03f30bbeeffd","questions":5100,"answered":5099,"when":"","ended":"2026-09-25T02:55:39Z","by":"","completion":"partial","measures":[{"category":"all","metric":"macro_accuracy","value":0.8902,"lo":null,"hi":null,"n":5099,"of":5100},{"category":"all","metric":"macro_brier","value":0.1606,"lo":null,"hi":null,"n":5099,"of":5100},{"category":"all","metric":"macro_ece","value":0.0638,"lo":null,"hi":null,"n":5099,"of":5100},{"category":"all","metric":"macro_aurc","value":0.0199,"lo":null,"hi":null,"n":5099,"of":5100},{"category":"all","metric":"macro_zero_prob","value":0.0176,"lo":null,"hi":null,"n":5099,"of":5100},{"category":"af","metric":"accuracy","value":0.87,"lo":null,"hi":null,"n":100,"of":100},{"category":"af","metric":"brier","value":0.2058,"lo":null,"hi":null,"n":100,"of":100},{"category":"af","metric":"ece","value":0.0873,"lo":null,"hi":null,"n":100,"of":100},{"category":"af","metric":"aurc","value":0.0259,"lo":null,"hi":null,"n":100,"of":100},{"category":"af","metric":"zero_prob","value":0.02,"lo":null,"hi":null,"n":100,"of":100},{"category":"am","metric":"accuracy","value":0.85,"lo":null,"hi":null,"n":100,"of":100},{"category":"am","metric":"brier","value":0.2195,"lo":null,"hi":null,"n":100,"of":100},{"category":"am","metric":"ece","value":0.096,"lo":null,"hi":null,"n":100,"of":100},{"category":"am","metric":"aurc","value":0.0284,"lo":null,"hi":null,"n":100,"of":100},{"category":"am","metric":"zero_prob","value":0.02,"lo":null,"hi":null,"n":100,"of":100},{"category":"ar","metric":"accuracy","value":0.87,"lo":null,"hi":null,"n":100,"of":100},{"category":"ar","metric":"brier","value":0.1962,"lo":null,"hi":null,"n":100,"of":100},{"category":"ar","metric":"ece","value":0.0756,"lo":null,"hi":null,"n":100,"of":100},{"category":"ar","metric":"aurc","value":0.0202,"lo":null,"hi":null,"n":100,"of":100},{"category":"ar","metric":"zero_prob","value":0.02,"lo":null,"hi":null,"n":100,"of":100},{"category":"az","metric":"accuracy","value":0.88,"lo":null,"hi":null,"n":100,"of":100},{"category":"az","metric":"brier","value":0.1693,"lo":null,"hi":null,"n":100,"of":100},{"category":"az","metric":"ece","value":0.0482,"lo":null,"hi":null,"n":100,"of":100},{"category":"az","metric":"aurc","value":0.0135,"lo":null,"hi":null,"n":100,"of":100},{"category":"az","metric":"zero_prob","value":0.03,"lo":null,"hi":null,"n":100,"of":100},{"category":"bn","metric":"accuracy","value":0.86,"lo":null,"hi":null,"n":100,"of":100},{"category":"bn","metric":"brier","value":0.1895,"lo":null,"hi":null,"n":100,"of":100},{"category":"bn","metric":"ece","value":0.0672,"lo":null,"hi":null,"n":100,"of":100},{"category":"bn","metric":"aurc","value":0.0265,"lo":null,"hi":null,"n":100,"of":100},{"category":"bn","metric":"zero_prob","value":0.01,"lo":null,"hi":null,"n":100,"of":100},{"category":"cy","metric":"accuracy","value":0.76,"lo":null,"hi":null,"n":100,"of":100},{"category":"cy","metric":"brier","value":0.3358,"lo":null,"hi":null,"n":100,"of":100},{"category":"cy","metric":"ece","value":0.0878,"lo":null,"hi":null,"n":100,"of":100},{"category":"cy","metric":"aurc","value":0.0565,"lo":null,"hi":null,"n":100,"of":100},{"category":"cy","metric":"zero_prob","value":0.04,"lo":null,"hi":null,"n":100,"of":100},{"category":"da","metric":"accuracy","value":0.88,"lo":null,"hi":null,"n":100,"of":100},{"category":"da","metric":"brier","value":0.1538,"lo":null,"hi":null,"n":100,"of":100},{"category":"da","metric":"ece","value":0.0439,"lo":null,"hi":null,"n":100,"of":100},{"category":"da","metric":"aurc","value":0.0153,"lo":null,"hi":null,"n":100,"of":100},{"category":"da","metric":"zero_prob","value":0.02,"lo":null,"hi":null,"n":100,"of":100},{"category":"de","metric":"accuracy","value":0.9394,"lo":null,"hi":null,"n":99,"of":100},{"category":"de","metric":"brier","value":0.087,"lo":null,"hi":null,"n":99,"of":100},{"category":"de","metric":"ece","value":0.057,"lo":null,"hi":null,"n":99,"of":100},{"category":"de","metric":"aurc","value":0.0073,"lo":null,"hi":null,"n":99,"of":100},{"category":"de","metric":"zero_prob","value":0.0101,"lo":null,"hi":null,"n":99,"of":100},{"category":"el","metric":"accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":100},{"category":"el","metric":"brier","value":0.094,"lo":null,"hi":null,"n":100,"of":100},{"category":"el","metric":"ece","value":0.0571,"lo":null,"hi":null,"n":100,"of":100},{"category":"el","metric":"aurc","value":0.0085,"lo":null,"hi":null,"n":100,"of":100},{"category":"el","metric":"zero_prob","value":0.01,"lo":null,"hi":null,"n":100,"of":100},{"category":"en","metric":"accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":100},{"category":"en","metric":"brier","value":0.108,"lo":null,"hi":null,"n":100,"of":100},{"category":"en","metric":"ece","value":0.0253,"lo":null,"hi":null,"n":100,"of":100},{"category":"en","metric":"aurc","value":0.008,"lo":null,"hi":null,"n":100,"of":100},{"category":"en","metric":"zero_prob","value":0.02,"lo":null,"hi":null,"n":100,"of":100},{"category":"es","metric":"accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":100},{"category":"es","metric":"brier","value":0.1223,"lo":null,"hi":null,"n":100,"of":100},{"category":"es","metric":"ece","value":0.0628,"lo":null,"hi":null,"n":100,"of":100},{"category":"es","metric":"aurc","value":0.0132,"lo":null,"hi":null,"n":100,"of":100},{"category":"es","metric":"zero_prob","value":0.01,"lo":null,"hi":null,"n":100,"of":100},{"category":"fa","metric":"accuracy","value":0.95,"lo":null,"hi":null,"n":100,"of":100},{"category":"fa","metric":"brier","value":0.0779,"lo":null,"hi":null,"n":100,"of":100},{"category":"fa","metric":"ece","value":0.0535,"lo":null,"hi":null,"n":100,"of":100},{"category":"fa","metric":"aurc","value":0.0059,"lo":null,"hi":null,"n":100,"of":100},{"category":"fa","metric":"zero_prob","value":0.01,"lo":null,"hi":null,"n":100,"of":100},{"category":"fi","metric":"accuracy","value":0.89,"lo":null,"hi":null,"n":100,"of":100},{"category":"fi","metric":"brier","value":0.1513,"lo":null,"hi":null,"n":100,"of":100},{"category":"fi","metric":"ece","value":0.0383,"lo":null,"hi":null,"n":100,"of":100},{"category":"fi","metric":"aurc","value":0.016,"lo":null,"hi":null,"n":100,"of":100},{"category":"fi","metric":"zero_prob","value":0.02,"lo":null,"hi":null,"n":100,"of":100},{"category":"fr","metric":"accuracy","value":0.9,"lo":null,"hi":null,"n":100,"of":100},{"category":"fr","metric":"brier","value":0.1302,"lo":null,"hi":null,"n":100,"of":100},{"category":"fr","metric":"ece","value":0.0652,"lo":null,"hi":null,"n":100,"of":100},{"category":"fr","metric":"aurc","value":0.0117,"lo":null,"hi":null,"n":100,"of":100},{"category":"fr","metric":"zero_prob","value":0.01,"lo":null,"hi":null,"n":100,"of":100},{"category":"he","metric":"accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":100},{"category":"he","metric":"brier","value":0.0928,"lo":null,"hi":null,"n":100,"of":100},{"category":"he","metric":"ece","value":0.0427,"lo":null,"hi":null,"n":100,"of":100},{"category":"he","metric":"aurc","value":0.0069,"lo":null,"hi":null,"n":100,"of":100},{"category":"he","metric":"zero_prob","value":0.01,"lo":null,"hi":null,"n":100,"of":100},{"category":"hi","metric":"accuracy","value":0.95,"lo":null,"hi":null,"n":100,"of":100},{"category":"hi","metric":"brier","value":0.0827,"lo":null,"hi":null,"n":100,"of":100},{"category":"hi","metric":"ece","value":0.029,"lo":null,"hi":null,"n":100,"of":100},{"category":"hi","metric":"aurc","value":0.0068,"lo":null,"hi":null,"n":100,"of":100},{"category":"hi","metric":"zero_prob","value":0.01,"lo":null,"hi":null,"n":100,"of":100},{"category":"hu","metric":"accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":100},{"category":"hu","metric":"brier","value":0.1328,"lo":null,"hi":null,"n":100,"of":100},{"category":"hu","metric":"ece","value":0.0387,"lo":null,"hi":null,"n":100,"of":100},{"category":"hu","metric":"aurc","value":0.009,"lo":null,"hi":null,"n":100,"of":100},{"category":"hu","metric":"zero_prob","value":0.03,"lo":null,"hi":null,"n":100,"of":100},{"category":"hy","metric":"accuracy","value":0.84,"lo":null,"hi":null,"n":100,"of":100},{"category":"hy","metric":"brier","value":0.1861,"lo":null,"hi":null,"n":100,"of":100},{"category":"hy","metric":"ece","value":0.0901,"lo":null,"hi":null,"n":100,"of":100},{"category":"hy","metric":"aurc","value":0.0227,"lo":null,"hi":null,"n":100,"of":100},{"category":"hy","metric":"zero_prob","value":0.01,"lo":null,"hi":null,"n":100,"of":100},{"category":"id","metric":"accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":100},{"category":"id","metric":"brier","value":0.1195,"lo":null,"hi":null,"n":100,"of":100},{"category":"id","metric":"ece","value":0.0453,"lo":null,"hi":null,"n":100,"of":100},{"category":"id","metric":"aurc","value":0.0092,"lo":null,"hi":null,"n":100,"of":100},{"category":"id","metric":"zero_prob","value":0.02,"lo":null,"hi":null,"n":100,"of":100},{"category":"is","metric":"accuracy","value":0.84,"lo":null,"hi":null,"n":100,"of":100},{"category":"is","metric":"brier","value":0.2212,"lo":null,"hi":null,"n":100,"of":100},{"category":"is","metric":"ece","value":0.073,"lo":null,"hi":null,"n":100,"of":100},{"category":"is","metric":"aurc","value":0.0248,"lo":null,"hi":null,"n":100,"of":100},{"category":"is","metric":"zero_prob","value":0.02,"lo":null,"hi":null,"n":100,"of":100},{"category":"it","metric":"accuracy","value":0.96,"lo":null,"hi":null,"n":100,"of":100},{"category":"it","metric":"brier","value":0.0568,"lo":null,"hi":null,"n":100,"of":100},{"category":"it","metric":"ece","value":0.0502,"lo":null,"hi":null,"n":100,"of":100},{"category":"it","metric":"aurc","value":0.0047,"lo":null,"hi":null,"n":100,"of":100},{"category":"it","metric":"zero_prob","value":0.01,"lo":null,"hi":null,"n":100,"of":100},{"category":"ja","metric":"accuracy","value":0.97,"lo":null,"hi":null,"n":100,"of":100},{"category":"ja","metric":"brier","value":0.0602,"lo":null,"hi":null,"n":100,"of":100},{"category":"ja","metric":"ece","value":0.0587,"lo":null,"hi":null,"n":100,"of":100},{"category":"ja","metric":"aurc","value":0.0035,"lo":null,"hi":null,"n":100,"of":100},{"category":"ja","metric":"zero_prob","value":0.01,"lo":null,"hi":null,"n":100,"of":100},{"category":"jv","metric":"accuracy","value":0.75,"lo":null,"hi":null,"n":100,"of":100},{"category":"jv","metric":"brier","value":0.3814,"lo":null,"hi":null,"n":100,"of":100},{"category":"jv","metric":"ece","value":0.0856,"lo":null,"hi":null,"n":100,"of":100},{"category":"jv","metric":"aurc","value":0.0908,"lo":null,"hi":null,"n":100,"of":100},{"category":"jv","metric":"zero_prob","value":0.04,"lo":null,"hi":null,"n":100,"of":100},{"category":"ka","metric":"accuracy","value":0.88,"lo":null,"hi":null,"n":100,"of":100},{"category":"ka","metric":"brier","value":0.2376,"lo":null,"hi":null,"n":100,"of":100},{"category":"ka","metric":"ece","value":0.1081,"lo":null,"hi":null,"n":100,"of":100},{"category":"ka","metric":"aurc","value":0.039,"lo":null,"hi":null,"n":100,"of":100},{"category":"ka","metric":"zero_prob","value":0.02,"lo":null,"hi":null,"n":100,"of":100},{"category":"km","metric":"accuracy","value":0.81,"lo":null,"hi":null,"n":100,"of":100},{"category":"km","metric":"brier","value":0.2688,"lo":null,"hi":null,"n":100,"of":100},{"category":"km","metric":"ece","value":0.0731,"lo":null,"hi":null,"n":100,"of":100},{"category":"km","metric":"aurc","value":0.0484,"lo":null,"hi":null,"n":100,"of":100},{"category":"km","metric":"zero_prob","value":0.03,"lo":null,"hi":null,"n":100,"of":100},{"category":"kn","metric":"accuracy","value":0.87,"lo":null,"hi":null,"n":100,"of":100},{"category":"kn","metric":"brier","value":0.158,"lo":null,"hi":null,"n":100,"of":100},{"category":"kn","metric":"ece","value":0.07,"lo":null,"hi":null,"n":100,"of":100},{"category":"kn","metric":"aurc","value":0.0211,"lo":null,"hi":null,"n":100,"of":100},{"category":"kn","metric":"zero_prob","value":0.01,"lo":null,"hi":null,"n":100,"of":100},{"category":"ko","metric":"accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":100},{"category":"ko","metric":"brier","value":0.0812,"lo":null,"hi":null,"n":100,"of":100},{"category":"ko","metric":"ece","value":0.0423,"lo":null,"hi":null,"n":100,"of":100},{"category":"ko","metric":"aurc","value":0.007,"lo":null,"hi":null,"n":100,"of":100},{"category":"ko","metric":"zero_prob","value":0.01,"lo":null,"hi":null,"n":100,"of":100},{"category":"lv","metric":"accuracy","value":0.85,"lo":null,"hi":null,"n":100,"of":100},{"category":"lv","metric":"brier","value":0.2047,"lo":null,"hi":null,"n":100,"of":100},{"category":"lv","metric":"ece","value":0.0748,"lo":null,"hi":null,"n":100,"of":100},{"category":"lv","metric":"aurc","value":0.0253,"lo":null,"hi":null,"n":100,"of":100},{"category":"lv","metric":"zero_prob","value":0.02,"lo":null,"hi":null,"n":100,"of":100},{"category":"ml","metric":"accuracy","value":0.91,"lo":null,"hi":null,"n":100,"of":100},{"category":"ml","metric":"brier","value":0.1368,"lo":null,"hi":null,"n":100,"of":100},{"category":"ml","metric":"ece","value":0.0495,"lo":null,"hi":null,"n":100,"of":100},{"category":"ml","metric":"aurc","value":0.0111,"lo":null,"hi":null,"n":100,"of":100},{"category":"ml","metric":"zero_prob","value":0.01,"lo":null,"hi":null,"n":100,"of":100},{"category":"mn","metric":"accuracy","value":0.84,"lo":null,"hi":null,"n":100,"of":100},{"category":"mn","metric":"brier","value":0.2063,"lo":null,"hi":null,"n":100,"of":100},{"category":"mn","metric":"ece","value":0.0825,"lo":null,"hi":null,"n":100,"of":100},{"category":"mn","metric":"aurc","value":0.0252,"lo":null,"hi":null,"n":100,"of":100},{"category":"mn","metric":"zero_prob","value":0.01,"lo":null,"hi":null,"n":100,"of":100},{"category":"ms","metric":"accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":100},{"category":"ms","metric":"brier","value":0.1374,"lo":null,"hi":null,"n":100,"of":100},{"category":"ms","metric":"ece","value":0.0684,"lo":null,"hi":null,"n":100,"of":100},{"category":"ms","metric":"aurc","value":0.0143,"lo":null,"hi":null,"n":100,"of":100},{"category":"ms","metric":"zero_prob","value":0.03,"lo":null,"hi":null,"n":100,"of":100},{"category":"my","metric":"accuracy","value":0.89,"lo":null,"hi":null,"n":100,"of":100},{"category":"my","metric":"brier","value":0.1661,"lo":null,"hi":null,"n":100,"of":100},{"category":"my","metric":"ece","value":0.0572,"lo":null,"hi":null,"n":100,"of":100},{"category":"my","metric":"aurc","value":0.0182,"lo":null,"hi":null,"n":100,"of":100},{"category":"my","metric":"zero_prob","value":0.01,"lo":null,"hi":null,"n":100,"of":100},{"category":"nb","metric":"accuracy","value":0.86,"lo":null,"hi":null,"n":100,"of":100},{"category":"nb","metric":"brier","value":0.1846,"lo":null,"hi":null,"n":100,"of":100},{"category":"nb","metric":"ece","value":0.0713,"lo":null,"hi":null,"n":100,"of":100},{"category":"nb","metric":"aurc","value":0.0223,"lo":null,"hi":null,"n":100,"of":100},{"category":"nb","metric":"zero_prob","value":0.01,"lo":null,"hi":null,"n":100,"of":100},{"category":"nl","metric":"accuracy","value":0.9,"lo":null,"hi":null,"n":100,"of":100},{"category":"nl","metric":"brier","value":0.1544,"lo":null,"hi":null,"n":100,"of":100},{"category":"nl","metric":"ece","value":0.0551,"lo":null,"hi":null,"n":100,"of":100},{"category":"nl","metric":"aurc","value":0.0142,"lo":null,"hi":null,"n":100,"of":100},{"category":"nl","metric":"zero_prob","value":0.02,"lo":null,"hi":null,"n":100,"of":100},{"category":"pl","metric":"accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":100},{"category":"pl","metric":"brier","value":0.1231,"lo":null,"hi":null,"n":100,"of":100},{"category":"pl","metric":"ece","value":0.0358,"lo":null,"hi":null,"n":100,"of":100},{"category":"pl","metric":"aurc","value":0.0105,"lo":null,"hi":null,"n":100,"of":100},{"category":"pl","metric":"zero_prob","value":0.01,"lo":null,"hi":null,"n":100,"of":100},{"category":"pt","metric":"accuracy","value":0.9,"lo":null,"hi":null,"n":100,"of":100},{"category":"pt","metric":"brier","value":0.159,"lo":null,"hi":null,"n":100,"of":100},{"category":"pt","metric":"ece","value":0.0985,"lo":null,"hi":null,"n":100,"of":100},{"category":"pt","metric":"aurc","value":0.0447,"lo":null,"hi":null,"n":100,"of":100},{"category":"pt","metric":"zero_prob","value":0.03,"lo":null,"hi":null,"n":100,"of":100},{"category":"ro","metric":"accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":100},{"category":"ro","metric":"brier","value":0.102,"lo":null,"hi":null,"n":100,"of":100},{"category":"ro","metric":"ece","value":0.0822,"lo":null,"hi":null,"n":100,"of":100},{"category":"ro","metric":"aurc","value":0.0071,"lo":null,"hi":null,"n":100,"of":100},{"category":"ro","metric":"zero_prob","value":0.02,"lo":null,"hi":null,"n":100,"of":100},{"category":"ru","metric":"accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":100},{"category":"ru","metric":"brier","value":0.0781,"lo":null,"hi":null,"n":100,"of":100},{"category":"ru","metric":"ece","value":0.0325,"lo":null,"hi":null,"n":100,"of":100},{"category":"ru","metric":"aurc","value":0.0052,"lo":null,"hi":null,"n":100,"of":100},{"category":"ru","metric":"zero_prob","value":0.01,"lo":null,"hi":null,"n":100,"of":100},{"category":"sl","metric":"accuracy","value":0.89,"lo":null,"hi":null,"n":100,"of":100},{"category":"sl","metric":"brier","value":0.16,"lo":null,"hi":null,"n":100,"of":100},{"category":"sl","metric":"ece","value":0.0579,"lo":null,"hi":null,"n":100,"of":100},{"category":"sl","metric":"aurc","value":0.0148,"lo":null,"hi":null,"n":100,"of":100},{"category":"sl","metric":"zero_prob","value":0.01,"lo":null,"hi":null,"n":100,"of":100},{"category":"sq","metric":"accuracy","value":0.76,"lo":null,"hi":null,"n":100,"of":100},{"category":"sq","metric":"brier","value":0.3067,"lo":null,"hi":null,"n":100,"of":100},{"category":"sq","metric":"ece","value":0.094,"lo":null,"hi":null,"n":100,"of":100},{"category":"sq","metric":"aurc","value":0.0531,"lo":null,"hi":null,"n":100,"of":100},{"category":"sq","metric":"zero_prob","value":0.02,"lo":null,"hi":null,"n":100,"of":100},{"category":"sv","metric":"accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":100},{"category":"sv","metric":"brier","value":0.1188,"lo":null,"hi":null,"n":100,"of":100},{"category":"sv","metric":"ece","value":0.0737,"lo":null,"hi":null,"n":100,"of":100},{"category":"sv","metric":"aurc","value":0.0103,"lo":null,"hi":null,"n":100,"of":100},{"category":"sv","metric":"zero_prob","value":0.01,"lo":null,"hi":null,"n":100,"of":100},{"category":"sw","metric":"accuracy","value":0.7,"lo":null,"hi":null,"n":100,"of":100},{"category":"sw","metric":"brier","value":0.3897,"lo":null,"hi":null,"n":100,"of":100},{"category":"sw","metric":"ece","value":0.1124,"lo":null,"hi":null,"n":100,"of":100},{"category":"sw","metric":"aurc","value":0.0724,"lo":null,"hi":null,"n":100,"of":100},{"category":"sw","metric":"zero_prob","value":0.07,"lo":null,"hi":null,"n":100,"of":100},{"category":"ta","metric":"accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":100},{"category":"ta","metric":"brier","value":0.171,"lo":null,"hi":null,"n":100,"of":100},{"category":"ta","metric":"ece","value":0.0924,"lo":null,"hi":null,"n":100,"of":100},{"category":"ta","metric":"aurc","value":0.0173,"lo":null,"hi":null,"n":100,"of":100},{"category":"ta","metric":"zero_prob","value":0.02,"lo":null,"hi":null,"n":100,"of":100},{"category":"te","metric":"accuracy","value":0.91,"lo":null,"hi":null,"n":100,"of":100},{"category":"te","metric":"brier","value":0.1252,"lo":null,"hi":null,"n":100,"of":100},{"category":"te","metric":"ece","value":0.0375,"lo":null,"hi":null,"n":100,"of":100},{"category":"te","metric":"aurc","value":0.0111,"lo":null,"hi":null,"n":100,"of":100},{"category":"te","metric":"zero_prob","value":0.01,"lo":null,"hi":null,"n":100,"of":100},{"category":"th","metric":"accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":100},{"category":"th","metric":"brier","value":0.0858,"lo":null,"hi":null,"n":100,"of":100},{"category":"th","metric":"ece","value":0.047,"lo":null,"hi":null,"n":100,"of":100},{"category":"th","metric":"aurc","value":0.0035,"lo":null,"hi":null,"n":100,"of":100},{"category":"th","metric":"zero_prob","value":0.01,"lo":null,"hi":null,"n":100,"of":100},{"category":"tl","metric":"accuracy","value":0.85,"lo":null,"hi":null,"n":100,"of":100},{"category":"tl","metric":"brier","value":0.1962,"lo":null,"hi":null,"n":100,"of":100},{"category":"tl","metric":"ece","value":0.0845,"lo":null,"hi":null,"n":100,"of":100},{"category":"tl","metric":"aurc","value":0.0212,"lo":null,"hi":null,"n":100,"of":100},{"category":"tl","metric":"zero_prob","value":0.03,"lo":null,"hi":null,"n":100,"of":100},{"category":"tr","metric":"accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":100},{"category":"tr","metric":"brier","value":0.1492,"lo":null,"hi":null,"n":100,"of":100},{"category":"tr","metric":"ece","value":0.061,"lo":null,"hi":null,"n":100,"of":100},{"category":"tr","metric":"aurc","value":0.012,"lo":null,"hi":null,"n":100,"of":100},{"category":"tr","metric":"zero_prob","value":0.02,"lo":null,"hi":null,"n":100,"of":100},{"category":"ur","metric":"accuracy","value":0.9,"lo":null,"hi":null,"n":100,"of":100},{"category":"ur","metric":"brier","value":0.1499,"lo":null,"hi":null,"n":100,"of":100},{"category":"ur","metric":"ece","value":0.0638,"lo":null,"hi":null,"n":100,"of":100},{"category":"ur","metric":"aurc","value":0.0178,"lo":null,"hi":null,"n":100,"of":100},{"category":"ur","metric":"zero_prob","value":0.01,"lo":null,"hi":null,"n":100,"of":100},{"category":"vi","metric":"accuracy","value":0.89,"lo":null,"hi":null,"n":100,"of":100},{"category":"vi","metric":"brier","value":0.1479,"lo":null,"hi":null,"n":100,"of":100},{"category":"vi","metric":"ece","value":0.0518,"lo":null,"hi":null,"n":100,"of":100},{"category":"vi","metric":"aurc","value":0.0142,"lo":null,"hi":null,"n":100,"of":100},{"category":"vi","metric":"zero_prob","value":0.01,"lo":null,"hi":null,"n":100,"of":100},{"category":"zh-CN","metric":"accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":100},{"category":"zh-CN","metric":"brier","value":0.1018,"lo":null,"hi":null,"n":100,"of":100},{"category":"zh-CN","metric":"ece","value":0.0363,"lo":null,"hi":null,"n":100,"of":100},{"category":"zh-CN","metric":"aurc","value":0.007,"lo":null,"hi":null,"n":100,"of":100},{"category":"zh-CN","metric":"zero_prob","value":0.01,"lo":null,"hi":null,"n":100,"of":100},{"category":"zh-TW","metric":"accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":100},{"category":"zh-TW","metric":"brier","value":0.1145,"lo":null,"hi":null,"n":100,"of":100},{"category":"zh-TW","metric":"ece","value":0.0621,"lo":null,"hi":null,"n":100,"of":100},{"category":"zh-TW","metric":"aurc","value":0.0101,"lo":null,"hi":null,"n":100,"of":100},{"category":"zh-TW","metric":"zero_prob","value":0.01,"lo":null,"hi":null,"n":100,"of":100}],"notes":"preds sha256:6e7dfa6d3a9bda215e61b7f3e278639d5327ed244f14d445a00d7d96e784b738","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"massive-harness-a5.df1c16a2-kai","benchmark":"massive","split":"harness","system":"kai","version":"a5@df1c16a2fef7ec93f2b34a6a194f6dc3c2eee70e936473b802151543c5d46e83","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"611059b60f64ab686d6475cd76778b3fac52cd98","questions":5100,"answered":5100,"when":"","ended":"2026-09-27T21:09:54Z","by":"","completion":"complete","measures":[{"category":"all","metric":"macro_accuracy","value":0.8882,"lo":null,"hi":null,"n":5100,"of":5100},{"category":"all","metric":"macro_brier","value":0.1572,"lo":null,"hi":null,"n":5100,"of":5100},{"category":"all","metric":"macro_ece","value":0.0655,"lo":null,"hi":null,"n":5100,"of":5100},{"category":"all","metric":"macro_aurc","value":0.0168,"lo":null,"hi":null,"n":5100,"of":5100},{"category":"all","metric":"macro_zero_prob","value":0,"lo":null,"hi":null,"n":5100,"of":5100},{"category":"af","metric":"accuracy","value":0.89,"lo":null,"hi":null,"n":100,"of":100},{"category":"af","metric":"brier","value":0.158,"lo":null,"hi":null,"n":100,"of":100},{"category":"af","metric":"ece","value":0.0595,"lo":null,"hi":null,"n":100,"of":100},{"category":"af","metric":"aurc","value":0.0151,"lo":null,"hi":null,"n":100,"of":100},{"category":"af","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"am","metric":"accuracy","value":0.84,"lo":null,"hi":null,"n":100,"of":100},{"category":"am","metric":"brier","value":0.2354,"lo":null,"hi":null,"n":100,"of":100},{"category":"am","metric":"ece","value":0.0683,"lo":null,"hi":null,"n":100,"of":100},{"category":"am","metric":"aurc","value":0.0281,"lo":null,"hi":null,"n":100,"of":100},{"category":"am","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"ar","metric":"accuracy","value":0.88,"lo":null,"hi":null,"n":100,"of":100},{"category":"ar","metric":"brier","value":0.1453,"lo":null,"hi":null,"n":100,"of":100},{"category":"ar","metric":"ece","value":0.085,"lo":null,"hi":null,"n":100,"of":100},{"category":"ar","metric":"aurc","value":0.0118,"lo":null,"hi":null,"n":100,"of":100},{"category":"ar","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"az","metric":"accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":100},{"category":"az","metric":"brier","value":0.1414,"lo":null,"hi":null,"n":100,"of":100},{"category":"az","metric":"ece","value":0.0609,"lo":null,"hi":null,"n":100,"of":100},{"category":"az","metric":"aurc","value":0.0137,"lo":null,"hi":null,"n":100,"of":100},{"category":"az","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"bn","metric":"accuracy","value":0.87,"lo":null,"hi":null,"n":100,"of":100},{"category":"bn","metric":"brier","value":0.1658,"lo":null,"hi":null,"n":100,"of":100},{"category":"bn","metric":"ece","value":0.0521,"lo":null,"hi":null,"n":100,"of":100},{"category":"bn","metric":"aurc","value":0.0205,"lo":null,"hi":null,"n":100,"of":100},{"category":"bn","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"cy","metric":"accuracy","value":0.82,"lo":null,"hi":null,"n":100,"of":100},{"category":"cy","metric":"brier","value":0.2431,"lo":null,"hi":null,"n":100,"of":100},{"category":"cy","metric":"ece","value":0.0679,"lo":null,"hi":null,"n":100,"of":100},{"category":"cy","metric":"aurc","value":0.0339,"lo":null,"hi":null,"n":100,"of":100},{"category":"cy","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"da","metric":"accuracy","value":0.91,"lo":null,"hi":null,"n":100,"of":100},{"category":"da","metric":"brier","value":0.1295,"lo":null,"hi":null,"n":100,"of":100},{"category":"da","metric":"ece","value":0.0491,"lo":null,"hi":null,"n":100,"of":100},{"category":"da","metric":"aurc","value":0.0079,"lo":null,"hi":null,"n":100,"of":100},{"category":"da","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"de","metric":"accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":100},{"category":"de","metric":"brier","value":0.0974,"lo":null,"hi":null,"n":100,"of":100},{"category":"de","metric":"ece","value":0.0687,"lo":null,"hi":null,"n":100,"of":100},{"category":"de","metric":"aurc","value":0.0046,"lo":null,"hi":null,"n":100,"of":100},{"category":"de","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"el","metric":"accuracy","value":0.9,"lo":null,"hi":null,"n":100,"of":100},{"category":"el","metric":"brier","value":0.1356,"lo":null,"hi":null,"n":100,"of":100},{"category":"el","metric":"ece","value":0.0537,"lo":null,"hi":null,"n":100,"of":100},{"category":"el","metric":"aurc","value":0.0088,"lo":null,"hi":null,"n":100,"of":100},{"category":"el","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"en","metric":"accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":100},{"category":"en","metric":"brier","value":0.11,"lo":null,"hi":null,"n":100,"of":100},{"category":"en","metric":"ece","value":0.0516,"lo":null,"hi":null,"n":100,"of":100},{"category":"en","metric":"aurc","value":0.007,"lo":null,"hi":null,"n":100,"of":100},{"category":"en","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"es","metric":"accuracy","value":0.9,"lo":null,"hi":null,"n":100,"of":100},{"category":"es","metric":"brier","value":0.1371,"lo":null,"hi":null,"n":100,"of":100},{"category":"es","metric":"ece","value":0.0433,"lo":null,"hi":null,"n":100,"of":100},{"category":"es","metric":"aurc","value":0.0169,"lo":null,"hi":null,"n":100,"of":100},{"category":"es","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"fa","metric":"accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":100},{"category":"fa","metric":"brier","value":0.1042,"lo":null,"hi":null,"n":100,"of":100},{"category":"fa","metric":"ece","value":0.0569,"lo":null,"hi":null,"n":100,"of":100},{"category":"fa","metric":"aurc","value":0.0051,"lo":null,"hi":null,"n":100,"of":100},{"category":"fa","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"fi","metric":"accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":100},{"category":"fi","metric":"brier","value":0.1274,"lo":null,"hi":null,"n":100,"of":100},{"category":"fi","metric":"ece","value":0.06,"lo":null,"hi":null,"n":100,"of":100},{"category":"fi","metric":"aurc","value":0.0168,"lo":null,"hi":null,"n":100,"of":100},{"category":"fi","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"fr","metric":"accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":100},{"category":"fr","metric":"brier","value":0.1028,"lo":null,"hi":null,"n":100,"of":100},{"category":"fr","metric":"ece","value":0.0502,"lo":null,"hi":null,"n":100,"of":100},{"category":"fr","metric":"aurc","value":0.0047,"lo":null,"hi":null,"n":100,"of":100},{"category":"fr","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"he","metric":"accuracy","value":0.87,"lo":null,"hi":null,"n":100,"of":100},{"category":"he","metric":"brier","value":0.1592,"lo":null,"hi":null,"n":100,"of":100},{"category":"he","metric":"ece","value":0.0729,"lo":null,"hi":null,"n":100,"of":100},{"category":"he","metric":"aurc","value":0.0137,"lo":null,"hi":null,"n":100,"of":100},{"category":"he","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"hi","metric":"accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":100},{"category":"hi","metric":"brier","value":0.141,"lo":null,"hi":null,"n":100,"of":100},{"category":"hi","metric":"ece","value":0.0735,"lo":null,"hi":null,"n":100,"of":100},{"category":"hi","metric":"aurc","value":0.0101,"lo":null,"hi":null,"n":100,"of":100},{"category":"hi","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"hu","metric":"accuracy","value":0.87,"lo":null,"hi":null,"n":100,"of":100},{"category":"hu","metric":"brier","value":0.1731,"lo":null,"hi":null,"n":100,"of":100},{"category":"hu","metric":"ece","value":0.0504,"lo":null,"hi":null,"n":100,"of":100},{"category":"hu","metric":"aurc","value":0.0141,"lo":null,"hi":null,"n":100,"of":100},{"category":"hu","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"hy","metric":"accuracy","value":0.84,"lo":null,"hi":null,"n":100,"of":100},{"category":"hy","metric":"brier","value":0.2068,"lo":null,"hi":null,"n":100,"of":100},{"category":"hy","metric":"ece","value":0.0573,"lo":null,"hi":null,"n":100,"of":100},{"category":"hy","metric":"aurc","value":0.0194,"lo":null,"hi":null,"n":100,"of":100},{"category":"hy","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"id","metric":"accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":100},{"category":"id","metric":"brier","value":0.108,"lo":null,"hi":null,"n":100,"of":100},{"category":"id","metric":"ece","value":0.059,"lo":null,"hi":null,"n":100,"of":100},{"category":"id","metric":"aurc","value":0.0077,"lo":null,"hi":null,"n":100,"of":100},{"category":"id","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"is","metric":"accuracy","value":0.86,"lo":null,"hi":null,"n":100,"of":100},{"category":"is","metric":"brier","value":0.1917,"lo":null,"hi":null,"n":100,"of":100},{"category":"is","metric":"ece","value":0.0591,"lo":null,"hi":null,"n":100,"of":100},{"category":"is","metric":"aurc","value":0.0225,"lo":null,"hi":null,"n":100,"of":100},{"category":"is","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"it","metric":"accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":100},{"category":"it","metric":"brier","value":0.1196,"lo":null,"hi":null,"n":100,"of":100},{"category":"it","metric":"ece","value":0.0505,"lo":null,"hi":null,"n":100,"of":100},{"category":"it","metric":"aurc","value":0.0131,"lo":null,"hi":null,"n":100,"of":100},{"category":"it","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"ja","metric":"accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":100},{"category":"ja","metric":"brier","value":0.0778,"lo":null,"hi":null,"n":100,"of":100},{"category":"ja","metric":"ece","value":0.0628,"lo":null,"hi":null,"n":100,"of":100},{"category":"ja","metric":"aurc","value":0.0036,"lo":null,"hi":null,"n":100,"of":100},{"category":"ja","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"jv","metric":"accuracy","value":0.83,"lo":null,"hi":null,"n":100,"of":100},{"category":"jv","metric":"brier","value":0.2101,"lo":null,"hi":null,"n":100,"of":100},{"category":"jv","metric":"ece","value":0.1025,"lo":null,"hi":null,"n":100,"of":100},{"category":"jv","metric":"aurc","value":0.0212,"lo":null,"hi":null,"n":100,"of":100},{"category":"jv","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"ka","metric":"accuracy","value":0.79,"lo":null,"hi":null,"n":100,"of":100},{"category":"ka","metric":"brier","value":0.2927,"lo":null,"hi":null,"n":100,"of":100},{"category":"ka","metric":"ece","value":0.101,"lo":null,"hi":null,"n":100,"of":100},{"category":"ka","metric":"aurc","value":0.0528,"lo":null,"hi":null,"n":100,"of":100},{"category":"ka","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"km","metric":"accuracy","value":0.8,"lo":null,"hi":null,"n":100,"of":100},{"category":"km","metric":"brier","value":0.249,"lo":null,"hi":null,"n":100,"of":100},{"category":"km","metric":"ece","value":0.0999,"lo":null,"hi":null,"n":100,"of":100},{"category":"km","metric":"aurc","value":0.0387,"lo":null,"hi":null,"n":100,"of":100},{"category":"km","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"kn","metric":"accuracy","value":0.87,"lo":null,"hi":null,"n":100,"of":100},{"category":"kn","metric":"brier","value":0.187,"lo":null,"hi":null,"n":100,"of":100},{"category":"kn","metric":"ece","value":0.057,"lo":null,"hi":null,"n":100,"of":100},{"category":"kn","metric":"aurc","value":0.0264,"lo":null,"hi":null,"n":100,"of":100},{"category":"kn","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"ko","metric":"accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":100},{"category":"ko","metric":"brier","value":0.1236,"lo":null,"hi":null,"n":100,"of":100},{"category":"ko","metric":"ece","value":0.0653,"lo":null,"hi":null,"n":100,"of":100},{"category":"ko","metric":"aurc","value":0.0088,"lo":null,"hi":null,"n":100,"of":100},{"category":"ko","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"lv","metric":"accuracy","value":0.85,"lo":null,"hi":null,"n":100,"of":100},{"category":"lv","metric":"brier","value":0.2325,"lo":null,"hi":null,"n":100,"of":100},{"category":"lv","metric":"ece","value":0.0748,"lo":null,"hi":null,"n":100,"of":100},{"category":"lv","metric":"aurc","value":0.0324,"lo":null,"hi":null,"n":100,"of":100},{"category":"lv","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"ml","metric":"accuracy","value":0.82,"lo":null,"hi":null,"n":100,"of":100},{"category":"ml","metric":"brier","value":0.2341,"lo":null,"hi":null,"n":100,"of":100},{"category":"ml","metric":"ece","value":0.0637,"lo":null,"hi":null,"n":100,"of":100},{"category":"ml","metric":"aurc","value":0.0429,"lo":null,"hi":null,"n":100,"of":100},{"category":"ml","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"mn","metric":"accuracy","value":0.88,"lo":null,"hi":null,"n":100,"of":100},{"category":"mn","metric":"brier","value":0.2164,"lo":null,"hi":null,"n":100,"of":100},{"category":"mn","metric":"ece","value":0.1092,"lo":null,"hi":null,"n":100,"of":100},{"category":"mn","metric":"aurc","value":0.0262,"lo":null,"hi":null,"n":100,"of":100},{"category":"mn","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"ms","metric":"accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":100},{"category":"ms","metric":"brier","value":0.1204,"lo":null,"hi":null,"n":100,"of":100},{"category":"ms","metric":"ece","value":0.0593,"lo":null,"hi":null,"n":100,"of":100},{"category":"ms","metric":"aurc","value":0.01,"lo":null,"hi":null,"n":100,"of":100},{"category":"ms","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"my","metric":"accuracy","value":0.9,"lo":null,"hi":null,"n":100,"of":100},{"category":"my","metric":"brier","value":0.1818,"lo":null,"hi":null,"n":100,"of":100},{"category":"my","metric":"ece","value":0.067,"lo":null,"hi":null,"n":100,"of":100},{"category":"my","metric":"aurc","value":0.0245,"lo":null,"hi":null,"n":100,"of":100},{"category":"my","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"nb","metric":"accuracy","value":0.91,"lo":null,"hi":null,"n":100,"of":100},{"category":"nb","metric":"brier","value":0.1343,"lo":null,"hi":null,"n":100,"of":100},{"category":"nb","metric":"ece","value":0.0674,"lo":null,"hi":null,"n":100,"of":100},{"category":"nb","metric":"aurc","value":0.0105,"lo":null,"hi":null,"n":100,"of":100},{"category":"nb","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"nl","metric":"accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":100},{"category":"nl","metric":"brier","value":0.118,"lo":null,"hi":null,"n":100,"of":100},{"category":"nl","metric":"ece","value":0.0697,"lo":null,"hi":null,"n":100,"of":100},{"category":"nl","metric":"aurc","value":0.0068,"lo":null,"hi":null,"n":100,"of":100},{"category":"nl","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"pl","metric":"accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":100},{"category":"pl","metric":"brier","value":0.0968,"lo":null,"hi":null,"n":100,"of":100},{"category":"pl","metric":"ece","value":0.0659,"lo":null,"hi":null,"n":100,"of":100},{"category":"pl","metric":"aurc","value":0.0057,"lo":null,"hi":null,"n":100,"of":100},{"category":"pl","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"pt","metric":"accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":100},{"category":"pt","metric":"brier","value":0.1196,"lo":null,"hi":null,"n":100,"of":100},{"category":"pt","metric":"ece","value":0.0486,"lo":null,"hi":null,"n":100,"of":100},{"category":"pt","metric":"aurc","value":0.0104,"lo":null,"hi":null,"n":100,"of":100},{"category":"pt","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"ro","metric":"accuracy","value":0.9,"lo":null,"hi":null,"n":100,"of":100},{"category":"ro","metric":"brier","value":0.1167,"lo":null,"hi":null,"n":100,"of":100},{"category":"ro","metric":"ece","value":0.0578,"lo":null,"hi":null,"n":100,"of":100},{"category":"ro","metric":"aurc","value":0.0071,"lo":null,"hi":null,"n":100,"of":100},{"category":"ro","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"ru","metric":"accuracy","value":0.96,"lo":null,"hi":null,"n":100,"of":100},{"category":"ru","metric":"brier","value":0.0542,"lo":null,"hi":null,"n":100,"of":100},{"category":"ru","metric":"ece","value":0.0651,"lo":null,"hi":null,"n":100,"of":100},{"category":"ru","metric":"aurc","value":0.0015,"lo":null,"hi":null,"n":100,"of":100},{"category":"ru","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"sl","metric":"accuracy","value":0.9,"lo":null,"hi":null,"n":100,"of":100},{"category":"sl","metric":"brier","value":0.1283,"lo":null,"hi":null,"n":100,"of":100},{"category":"sl","metric":"ece","value":0.052,"lo":null,"hi":null,"n":100,"of":100},{"category":"sl","metric":"aurc","value":0.0085,"lo":null,"hi":null,"n":100,"of":100},{"category":"sl","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"sq","metric":"accuracy","value":0.88,"lo":null,"hi":null,"n":100,"of":100},{"category":"sq","metric":"brier","value":0.1785,"lo":null,"hi":null,"n":100,"of":100},{"category":"sq","metric":"ece","value":0.0739,"lo":null,"hi":null,"n":100,"of":100},{"category":"sq","metric":"aurc","value":0.018,"lo":null,"hi":null,"n":100,"of":100},{"category":"sq","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"sv","metric":"accuracy","value":0.88,"lo":null,"hi":null,"n":100,"of":100},{"category":"sv","metric":"brier","value":0.1347,"lo":null,"hi":null,"n":100,"of":100},{"category":"sv","metric":"ece","value":0.0602,"lo":null,"hi":null,"n":100,"of":100},{"category":"sv","metric":"aurc","value":0.0103,"lo":null,"hi":null,"n":100,"of":100},{"category":"sv","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"sw","metric":"accuracy","value":0.81,"lo":null,"hi":null,"n":100,"of":100},{"category":"sw","metric":"brier","value":0.2677,"lo":null,"hi":null,"n":100,"of":100},{"category":"sw","metric":"ece","value":0.0674,"lo":null,"hi":null,"n":100,"of":100},{"category":"sw","metric":"aurc","value":0.0388,"lo":null,"hi":null,"n":100,"of":100},{"category":"sw","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"ta","metric":"accuracy","value":0.84,"lo":null,"hi":null,"n":100,"of":100},{"category":"ta","metric":"brier","value":0.1925,"lo":null,"hi":null,"n":100,"of":100},{"category":"ta","metric":"ece","value":0.0881,"lo":null,"hi":null,"n":100,"of":100},{"category":"ta","metric":"aurc","value":0.0204,"lo":null,"hi":null,"n":100,"of":100},{"category":"ta","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"te","metric":"accuracy","value":0.83,"lo":null,"hi":null,"n":100,"of":100},{"category":"te","metric":"brier","value":0.2539,"lo":null,"hi":null,"n":100,"of":100},{"category":"te","metric":"ece","value":0.0734,"lo":null,"hi":null,"n":100,"of":100},{"category":"te","metric":"aurc","value":0.0438,"lo":null,"hi":null,"n":100,"of":100},{"category":"te","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"th","metric":"accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":100},{"category":"th","metric":"brier","value":0.1101,"lo":null,"hi":null,"n":100,"of":100},{"category":"th","metric":"ece","value":0.0498,"lo":null,"hi":null,"n":100,"of":100},{"category":"th","metric":"aurc","value":0.0073,"lo":null,"hi":null,"n":100,"of":100},{"category":"th","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"tl","metric":"accuracy","value":0.86,"lo":null,"hi":null,"n":100,"of":100},{"category":"tl","metric":"brier","value":0.1696,"lo":null,"hi":null,"n":100,"of":100},{"category":"tl","metric":"ece","value":0.0501,"lo":null,"hi":null,"n":100,"of":100},{"category":"tl","metric":"aurc","value":0.0229,"lo":null,"hi":null,"n":100,"of":100},{"category":"tl","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"tr","metric":"accuracy","value":0.88,"lo":null,"hi":null,"n":100,"of":100},{"category":"tr","metric":"brier","value":0.1871,"lo":null,"hi":null,"n":100,"of":100},{"category":"tr","metric":"ece","value":0.0763,"lo":null,"hi":null,"n":100,"of":100},{"category":"tr","metric":"aurc","value":0.0206,"lo":null,"hi":null,"n":100,"of":100},{"category":"tr","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"ur","metric":"accuracy","value":0.86,"lo":null,"hi":null,"n":100,"of":100},{"category":"ur","metric":"brier","value":0.1859,"lo":null,"hi":null,"n":100,"of":100},{"category":"ur","metric":"ece","value":0.0737,"lo":null,"hi":null,"n":100,"of":100},{"category":"ur","metric":"aurc","value":0.0229,"lo":null,"hi":null,"n":100,"of":100},{"category":"ur","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"vi","metric":"accuracy","value":0.89,"lo":null,"hi":null,"n":100,"of":100},{"category":"vi","metric":"brier","value":0.159,"lo":null,"hi":null,"n":100,"of":100},{"category":"vi","metric":"ece","value":0.0596,"lo":null,"hi":null,"n":100,"of":100},{"category":"vi","metric":"aurc","value":0.0134,"lo":null,"hi":null,"n":100,"of":100},{"category":"vi","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"zh-CN","metric":"accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":100},{"category":"zh-CN","metric":"brier","value":0.089,"lo":null,"hi":null,"n":100,"of":100},{"category":"zh-CN","metric":"ece","value":0.0504,"lo":null,"hi":null,"n":100,"of":100},{"category":"zh-CN","metric":"aurc","value":0.004,"lo":null,"hi":null,"n":100,"of":100},{"category":"zh-CN","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"zh-TW","metric":"accuracy","value":0.97,"lo":null,"hi":null,"n":100,"of":100},{"category":"zh-TW","metric":"brier","value":0.0627,"lo":null,"hi":null,"n":100,"of":100},{"category":"zh-TW","metric":"ece","value":0.0762,"lo":null,"hi":null,"n":100,"of":100},{"category":"zh-TW","metric":"aurc","value":0.0016,"lo":null,"hi":null,"n":100,"of":100},{"category":"zh-TW","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100}],"notes":"device metal; dtype bf16; preds sha256:f28954731ec991ecbed3c1fbfc45980eeffef7744d8c12bcfc4cb160cb6f3eac; scores sha256:071b556a8eafd2c19d4f9a948b4f10251c9b253107c6afb21ad959ef72af4a77","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"massive-harness-a6.a211cc70-kai","benchmark":"massive","split":"harness","system":"kai","version":"a6@a211cc70103825bfbc9bfb0fbe970f8bf08da2efe0953cd8045a3cd07397f4bf","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"a1a4a73d524cecf8038e30c408ad30b66e71d490","questions":5100,"answered":5100,"when":"","ended":"2026-09-28T03:18:42Z","by":"","completion":"complete","measures":[{"category":"all","metric":"macro_accuracy","value":0.9096,"lo":null,"hi":null,"n":5100,"of":5100},{"category":"all","metric":"macro_brier","value":0.1358,"lo":null,"hi":null,"n":5100,"of":5100},{"category":"all","metric":"macro_ece","value":0.0589,"lo":null,"hi":null,"n":5100,"of":5100},{"category":"all","metric":"macro_aurc","value":0.0128,"lo":null,"hi":null,"n":5100,"of":5100},{"category":"all","metric":"macro_zero_prob","value":0,"lo":null,"hi":null,"n":5100,"of":5100},{"category":"af","metric":"accuracy","value":0.9,"lo":null,"hi":null,"n":100,"of":100},{"category":"af","metric":"brier","value":0.1488,"lo":null,"hi":null,"n":100,"of":100},{"category":"af","metric":"ece","value":0.0615,"lo":null,"hi":null,"n":100,"of":100},{"category":"af","metric":"aurc","value":0.0125,"lo":null,"hi":null,"n":100,"of":100},{"category":"af","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"am","metric":"accuracy","value":0.85,"lo":null,"hi":null,"n":100,"of":100},{"category":"am","metric":"brier","value":0.2307,"lo":null,"hi":null,"n":100,"of":100},{"category":"am","metric":"ece","value":0.0715,"lo":null,"hi":null,"n":100,"of":100},{"category":"am","metric":"aurc","value":0.0278,"lo":null,"hi":null,"n":100,"of":100},{"category":"am","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"ar","metric":"accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":100},{"category":"ar","metric":"brier","value":0.1195,"lo":null,"hi":null,"n":100,"of":100},{"category":"ar","metric":"ece","value":0.0688,"lo":null,"hi":null,"n":100,"of":100},{"category":"ar","metric":"aurc","value":0.006,"lo":null,"hi":null,"n":100,"of":100},{"category":"ar","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"az","metric":"accuracy","value":0.95,"lo":null,"hi":null,"n":100,"of":100},{"category":"az","metric":"brier","value":0.0814,"lo":null,"hi":null,"n":100,"of":100},{"category":"az","metric":"ece","value":0.048,"lo":null,"hi":null,"n":100,"of":100},{"category":"az","metric":"aurc","value":0.0039,"lo":null,"hi":null,"n":100,"of":100},{"category":"az","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"bn","metric":"accuracy","value":0.9,"lo":null,"hi":null,"n":100,"of":100},{"category":"bn","metric":"brier","value":0.1677,"lo":null,"hi":null,"n":100,"of":100},{"category":"bn","metric":"ece","value":0.0867,"lo":null,"hi":null,"n":100,"of":100},{"category":"bn","metric":"aurc","value":0.0197,"lo":null,"hi":null,"n":100,"of":100},{"category":"bn","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"cy","metric":"accuracy","value":0.86,"lo":null,"hi":null,"n":100,"of":100},{"category":"cy","metric":"brier","value":0.1902,"lo":null,"hi":null,"n":100,"of":100},{"category":"cy","metric":"ece","value":0.0491,"lo":null,"hi":null,"n":100,"of":100},{"category":"cy","metric":"aurc","value":0.0273,"lo":null,"hi":null,"n":100,"of":100},{"category":"cy","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"da","metric":"accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":100},{"category":"da","metric":"brier","value":0.1097,"lo":null,"hi":null,"n":100,"of":100},{"category":"da","metric":"ece","value":0.0335,"lo":null,"hi":null,"n":100,"of":100},{"category":"da","metric":"aurc","value":0.0055,"lo":null,"hi":null,"n":100,"of":100},{"category":"da","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"de","metric":"accuracy","value":0.91,"lo":null,"hi":null,"n":100,"of":100},{"category":"de","metric":"brier","value":0.1031,"lo":null,"hi":null,"n":100,"of":100},{"category":"de","metric":"ece","value":0.0629,"lo":null,"hi":null,"n":100,"of":100},{"category":"de","metric":"aurc","value":0.0054,"lo":null,"hi":null,"n":100,"of":100},{"category":"de","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"el","metric":"accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":100},{"category":"el","metric":"brier","value":0.1016,"lo":null,"hi":null,"n":100,"of":100},{"category":"el","metric":"ece","value":0.0497,"lo":null,"hi":null,"n":100,"of":100},{"category":"el","metric":"aurc","value":0.0045,"lo":null,"hi":null,"n":100,"of":100},{"category":"el","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"en","metric":"accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":100},{"category":"en","metric":"brier","value":0.1112,"lo":null,"hi":null,"n":100,"of":100},{"category":"en","metric":"ece","value":0.0599,"lo":null,"hi":null,"n":100,"of":100},{"category":"en","metric":"aurc","value":0.0066,"lo":null,"hi":null,"n":100,"of":100},{"category":"en","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"es","metric":"accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":100},{"category":"es","metric":"brier","value":0.1241,"lo":null,"hi":null,"n":100,"of":100},{"category":"es","metric":"ece","value":0.0607,"lo":null,"hi":null,"n":100,"of":100},{"category":"es","metric":"aurc","value":0.0123,"lo":null,"hi":null,"n":100,"of":100},{"category":"es","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"fa","metric":"accuracy","value":0.95,"lo":null,"hi":null,"n":100,"of":100},{"category":"fa","metric":"brier","value":0.0864,"lo":null,"hi":null,"n":100,"of":100},{"category":"fa","metric":"ece","value":0.0394,"lo":null,"hi":null,"n":100,"of":100},{"category":"fa","metric":"aurc","value":0.0037,"lo":null,"hi":null,"n":100,"of":100},{"category":"fa","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"fi","metric":"accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":100},{"category":"fi","metric":"brier","value":0.0929,"lo":null,"hi":null,"n":100,"of":100},{"category":"fi","metric":"ece","value":0.0416,"lo":null,"hi":null,"n":100,"of":100},{"category":"fi","metric":"aurc","value":0.0204,"lo":null,"hi":null,"n":100,"of":100},{"category":"fi","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"fr","metric":"accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":100},{"category":"fr","metric":"brier","value":0.1136,"lo":null,"hi":null,"n":100,"of":100},{"category":"fr","metric":"ece","value":0.0565,"lo":null,"hi":null,"n":100,"of":100},{"category":"fr","metric":"aurc","value":0.0067,"lo":null,"hi":null,"n":100,"of":100},{"category":"fr","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"he","metric":"accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":100},{"category":"he","metric":"brier","value":0.1315,"lo":null,"hi":null,"n":100,"of":100},{"category":"he","metric":"ece","value":0.062,"lo":null,"hi":null,"n":100,"of":100},{"category":"he","metric":"aurc","value":0.0095,"lo":null,"hi":null,"n":100,"of":100},{"category":"he","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"hi","metric":"accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":100},{"category":"hi","metric":"brier","value":0.1119,"lo":null,"hi":null,"n":100,"of":100},{"category":"hi","metric":"ece","value":0.0523,"lo":null,"hi":null,"n":100,"of":100},{"category":"hi","metric":"aurc","value":0.0068,"lo":null,"hi":null,"n":100,"of":100},{"category":"hi","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"hu","metric":"accuracy","value":0.86,"lo":null,"hi":null,"n":100,"of":100},{"category":"hu","metric":"brier","value":0.1782,"lo":null,"hi":null,"n":100,"of":100},{"category":"hu","metric":"ece","value":0.0787,"lo":null,"hi":null,"n":100,"of":100},{"category":"hu","metric":"aurc","value":0.0172,"lo":null,"hi":null,"n":100,"of":100},{"category":"hu","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"hy","metric":"accuracy","value":0.87,"lo":null,"hi":null,"n":100,"of":100},{"category":"hy","metric":"brier","value":0.18,"lo":null,"hi":null,"n":100,"of":100},{"category":"hy","metric":"ece","value":0.0651,"lo":null,"hi":null,"n":100,"of":100},{"category":"hy","metric":"aurc","value":0.0155,"lo":null,"hi":null,"n":100,"of":100},{"category":"hy","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"id","metric":"accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":100},{"category":"id","metric":"brier","value":0.0874,"lo":null,"hi":null,"n":100,"of":100},{"category":"id","metric":"ece","value":0.0365,"lo":null,"hi":null,"n":100,"of":100},{"category":"id","metric":"aurc","value":0.0075,"lo":null,"hi":null,"n":100,"of":100},{"category":"id","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"is","metric":"accuracy","value":0.87,"lo":null,"hi":null,"n":100,"of":100},{"category":"is","metric":"brier","value":0.1892,"lo":null,"hi":null,"n":100,"of":100},{"category":"is","metric":"ece","value":0.0637,"lo":null,"hi":null,"n":100,"of":100},{"category":"is","metric":"aurc","value":0.0254,"lo":null,"hi":null,"n":100,"of":100},{"category":"is","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"it","metric":"accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":100},{"category":"it","metric":"brier","value":0.1113,"lo":null,"hi":null,"n":100,"of":100},{"category":"it","metric":"ece","value":0.0568,"lo":null,"hi":null,"n":100,"of":100},{"category":"it","metric":"aurc","value":0.0078,"lo":null,"hi":null,"n":100,"of":100},{"category":"it","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"ja","metric":"accuracy","value":0.97,"lo":null,"hi":null,"n":100,"of":100},{"category":"ja","metric":"brier","value":0.0632,"lo":null,"hi":null,"n":100,"of":100},{"category":"ja","metric":"ece","value":0.0329,"lo":null,"hi":null,"n":100,"of":100},{"category":"ja","metric":"aurc","value":0.0016,"lo":null,"hi":null,"n":100,"of":100},{"category":"ja","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"jv","metric":"accuracy","value":0.85,"lo":null,"hi":null,"n":100,"of":100},{"category":"jv","metric":"brier","value":0.2089,"lo":null,"hi":null,"n":100,"of":100},{"category":"jv","metric":"ece","value":0.0702,"lo":null,"hi":null,"n":100,"of":100},{"category":"jv","metric":"aurc","value":0.0225,"lo":null,"hi":null,"n":100,"of":100},{"category":"jv","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"ka","metric":"accuracy","value":0.83,"lo":null,"hi":null,"n":100,"of":100},{"category":"ka","metric":"brier","value":0.2511,"lo":null,"hi":null,"n":100,"of":100},{"category":"ka","metric":"ece","value":0.0845,"lo":null,"hi":null,"n":100,"of":100},{"category":"ka","metric":"aurc","value":0.0478,"lo":null,"hi":null,"n":100,"of":100},{"category":"ka","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"km","metric":"accuracy","value":0.85,"lo":null,"hi":null,"n":100,"of":100},{"category":"km","metric":"brier","value":0.2361,"lo":null,"hi":null,"n":100,"of":100},{"category":"km","metric":"ece","value":0.0893,"lo":null,"hi":null,"n":100,"of":100},{"category":"km","metric":"aurc","value":0.0335,"lo":null,"hi":null,"n":100,"of":100},{"category":"km","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"kn","metric":"accuracy","value":0.9,"lo":null,"hi":null,"n":100,"of":100},{"category":"kn","metric":"brier","value":0.1807,"lo":null,"hi":null,"n":100,"of":100},{"category":"kn","metric":"ece","value":0.0847,"lo":null,"hi":null,"n":100,"of":100},{"category":"kn","metric":"aurc","value":0.016,"lo":null,"hi":null,"n":100,"of":100},{"category":"kn","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"ko","metric":"accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":100},{"category":"ko","metric":"brier","value":0.1132,"lo":null,"hi":null,"n":100,"of":100},{"category":"ko","metric":"ece","value":0.0326,"lo":null,"hi":null,"n":100,"of":100},{"category":"ko","metric":"aurc","value":0.0066,"lo":null,"hi":null,"n":100,"of":100},{"category":"ko","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"lv","metric":"accuracy","value":0.89,"lo":null,"hi":null,"n":100,"of":100},{"category":"lv","metric":"brier","value":0.1668,"lo":null,"hi":null,"n":100,"of":100},{"category":"lv","metric":"ece","value":0.0716,"lo":null,"hi":null,"n":100,"of":100},{"category":"lv","metric":"aurc","value":0.0122,"lo":null,"hi":null,"n":100,"of":100},{"category":"lv","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"ml","metric":"accuracy","value":0.87,"lo":null,"hi":null,"n":100,"of":100},{"category":"ml","metric":"brier","value":0.2236,"lo":null,"hi":null,"n":100,"of":100},{"category":"ml","metric":"ece","value":0.0836,"lo":null,"hi":null,"n":100,"of":100},{"category":"ml","metric":"aurc","value":0.0332,"lo":null,"hi":null,"n":100,"of":100},{"category":"ml","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"mn","metric":"accuracy","value":0.88,"lo":null,"hi":null,"n":100,"of":100},{"category":"mn","metric":"brier","value":0.1881,"lo":null,"hi":null,"n":100,"of":100},{"category":"mn","metric":"ece","value":0.0808,"lo":null,"hi":null,"n":100,"of":100},{"category":"mn","metric":"aurc","value":0.019,"lo":null,"hi":null,"n":100,"of":100},{"category":"mn","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"ms","metric":"accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":100},{"category":"ms","metric":"brier","value":0.122,"lo":null,"hi":null,"n":100,"of":100},{"category":"ms","metric":"ece","value":0.0641,"lo":null,"hi":null,"n":100,"of":100},{"category":"ms","metric":"aurc","value":0.0172,"lo":null,"hi":null,"n":100,"of":100},{"category":"ms","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"my","metric":"accuracy","value":0.95,"lo":null,"hi":null,"n":100,"of":100},{"category":"my","metric":"brier","value":0.0927,"lo":null,"hi":null,"n":100,"of":100},{"category":"my","metric":"ece","value":0.0478,"lo":null,"hi":null,"n":100,"of":100},{"category":"my","metric":"aurc","value":0.0042,"lo":null,"hi":null,"n":100,"of":100},{"category":"my","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"nb","metric":"accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":100},{"category":"nb","metric":"brier","value":0.1151,"lo":null,"hi":null,"n":100,"of":100},{"category":"nb","metric":"ece","value":0.0496,"lo":null,"hi":null,"n":100,"of":100},{"category":"nb","metric":"aurc","value":0.0065,"lo":null,"hi":null,"n":100,"of":100},{"category":"nb","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"nl","metric":"accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":100},{"category":"nl","metric":"brier","value":0.0939,"lo":null,"hi":null,"n":100,"of":100},{"category":"nl","metric":"ece","value":0.0539,"lo":null,"hi":null,"n":100,"of":100},{"category":"nl","metric":"aurc","value":0.0049,"lo":null,"hi":null,"n":100,"of":100},{"category":"nl","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"pl","metric":"accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":100},{"category":"pl","metric":"brier","value":0.0982,"lo":null,"hi":null,"n":100,"of":100},{"category":"pl","metric":"ece","value":0.0296,"lo":null,"hi":null,"n":100,"of":100},{"category":"pl","metric":"aurc","value":0.0065,"lo":null,"hi":null,"n":100,"of":100},{"category":"pl","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"pt","metric":"accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":100},{"category":"pt","metric":"brier","value":0.1098,"lo":null,"hi":null,"n":100,"of":100},{"category":"pt","metric":"ece","value":0.0312,"lo":null,"hi":null,"n":100,"of":100},{"category":"pt","metric":"aurc","value":0.0074,"lo":null,"hi":null,"n":100,"of":100},{"category":"pt","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"ro","metric":"accuracy","value":0.95,"lo":null,"hi":null,"n":100,"of":100},{"category":"ro","metric":"brier","value":0.0839,"lo":null,"hi":null,"n":100,"of":100},{"category":"ro","metric":"ece","value":0.0474,"lo":null,"hi":null,"n":100,"of":100},{"category":"ro","metric":"aurc","value":0.0035,"lo":null,"hi":null,"n":100,"of":100},{"category":"ro","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"ru","metric":"accuracy","value":0.98,"lo":null,"hi":null,"n":100,"of":100},{"category":"ru","metric":"brier","value":0.0454,"lo":null,"hi":null,"n":100,"of":100},{"category":"ru","metric":"ece","value":0.0447,"lo":null,"hi":null,"n":100,"of":100},{"category":"ru","metric":"aurc","value":0.0009,"lo":null,"hi":null,"n":100,"of":100},{"category":"ru","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"sl","metric":"accuracy","value":0.89,"lo":null,"hi":null,"n":100,"of":100},{"category":"sl","metric":"brier","value":0.1272,"lo":null,"hi":null,"n":100,"of":100},{"category":"sl","metric":"ece","value":0.0486,"lo":null,"hi":null,"n":100,"of":100},{"category":"sl","metric":"aurc","value":0.0099,"lo":null,"hi":null,"n":100,"of":100},{"category":"sl","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"sq","metric":"accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":100},{"category":"sq","metric":"brier","value":0.1304,"lo":null,"hi":null,"n":100,"of":100},{"category":"sq","metric":"ece","value":0.0477,"lo":null,"hi":null,"n":100,"of":100},{"category":"sq","metric":"aurc","value":0.01,"lo":null,"hi":null,"n":100,"of":100},{"category":"sq","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"sv","metric":"accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":100},{"category":"sv","metric":"brier","value":0.115,"lo":null,"hi":null,"n":100,"of":100},{"category":"sv","metric":"ece","value":0.0596,"lo":null,"hi":null,"n":100,"of":100},{"category":"sv","metric":"aurc","value":0.0067,"lo":null,"hi":null,"n":100,"of":100},{"category":"sv","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"sw","metric":"accuracy","value":0.83,"lo":null,"hi":null,"n":100,"of":100},{"category":"sw","metric":"brier","value":0.2301,"lo":null,"hi":null,"n":100,"of":100},{"category":"sw","metric":"ece","value":0.0788,"lo":null,"hi":null,"n":100,"of":100},{"category":"sw","metric":"aurc","value":0.0245,"lo":null,"hi":null,"n":100,"of":100},{"category":"sw","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"ta","metric":"accuracy","value":0.91,"lo":null,"hi":null,"n":100,"of":100},{"category":"ta","metric":"brier","value":0.1316,"lo":null,"hi":null,"n":100,"of":100},{"category":"ta","metric":"ece","value":0.0491,"lo":null,"hi":null,"n":100,"of":100},{"category":"ta","metric":"aurc","value":0.0068,"lo":null,"hi":null,"n":100,"of":100},{"category":"ta","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"te","metric":"accuracy","value":0.85,"lo":null,"hi":null,"n":100,"of":100},{"category":"te","metric":"brier","value":0.2266,"lo":null,"hi":null,"n":100,"of":100},{"category":"te","metric":"ece","value":0.1197,"lo":null,"hi":null,"n":100,"of":100},{"category":"te","metric":"aurc","value":0.0337,"lo":null,"hi":null,"n":100,"of":100},{"category":"te","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"th","metric":"accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":100},{"category":"th","metric":"brier","value":0.118,"lo":null,"hi":null,"n":100,"of":100},{"category":"th","metric":"ece","value":0.0342,"lo":null,"hi":null,"n":100,"of":100},{"category":"th","metric":"aurc","value":0.0111,"lo":null,"hi":null,"n":100,"of":100},{"category":"th","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"tl","metric":"accuracy","value":0.91,"lo":null,"hi":null,"n":100,"of":100},{"category":"tl","metric":"brier","value":0.1509,"lo":null,"hi":null,"n":100,"of":100},{"category":"tl","metric":"ece","value":0.0774,"lo":null,"hi":null,"n":100,"of":100},{"category":"tl","metric":"aurc","value":0.0165,"lo":null,"hi":null,"n":100,"of":100},{"category":"tl","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"tr","metric":"accuracy","value":0.9,"lo":null,"hi":null,"n":100,"of":100},{"category":"tr","metric":"brier","value":0.1558,"lo":null,"hi":null,"n":100,"of":100},{"category":"tr","metric":"ece","value":0.0737,"lo":null,"hi":null,"n":100,"of":100},{"category":"tr","metric":"aurc","value":0.0205,"lo":null,"hi":null,"n":100,"of":100},{"category":"tr","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"ur","metric":"accuracy","value":0.91,"lo":null,"hi":null,"n":100,"of":100},{"category":"ur","metric":"brier","value":0.13,"lo":null,"hi":null,"n":100,"of":100},{"category":"ur","metric":"ece","value":0.0697,"lo":null,"hi":null,"n":100,"of":100},{"category":"ur","metric":"aurc","value":0.0092,"lo":null,"hi":null,"n":100,"of":100},{"category":"ur","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"vi","metric":"accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":100},{"category":"vi","metric":"brier","value":0.1164,"lo":null,"hi":null,"n":100,"of":100},{"category":"vi","metric":"ece","value":0.0603,"lo":null,"hi":null,"n":100,"of":100},{"category":"vi","metric":"aurc","value":0.0058,"lo":null,"hi":null,"n":100,"of":100},{"category":"vi","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"zh-CN","metric":"accuracy","value":0.95,"lo":null,"hi":null,"n":100,"of":100},{"category":"zh-CN","metric":"brier","value":0.0793,"lo":null,"hi":null,"n":100,"of":100},{"category":"zh-CN","metric":"ece","value":0.0399,"lo":null,"hi":null,"n":100,"of":100},{"category":"zh-CN","metric":"aurc","value":0.0026,"lo":null,"hi":null,"n":100,"of":100},{"category":"zh-CN","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"zh-TW","metric":"accuracy","value":0.96,"lo":null,"hi":null,"n":100,"of":100},{"category":"zh-TW","metric":"brier","value":0.0525,"lo":null,"hi":null,"n":100,"of":100},{"category":"zh-TW","metric":"ece","value":0.041,"lo":null,"hi":null,"n":100,"of":100},{"category":"zh-TW","metric":"aurc","value":0.0013,"lo":null,"hi":null,"n":100,"of":100},{"category":"zh-TW","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100}],"notes":"device metal; dtype bf16; preds sha256:43b1bd476a72a536ee96fbc934a065a808fdb35395e694d9af873e462b3d8866; scores sha256:3760df4b2c4cdcccb225bbb446b2ddcff998e09ab723e36ee466f771dc355a7e","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"massive-harness-a7.0834a74f-kai","benchmark":"massive","split":"harness","system":"kai","version":"a7@0834a74f2d140642a453373da09e5a128e3d2c8d9e8bb1dcaf86d7210a4ecdfc","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"a1a4a73d524cecf8038e30c408ad30b66e71d490","questions":5100,"answered":5100,"when":"","ended":"2026-09-28T12:24:45Z","by":"","completion":"complete","measures":[{"category":"all","metric":"macro_accuracy","value":0.9094,"lo":null,"hi":null,"n":5100,"of":5100},{"category":"all","metric":"macro_brier","value":0.1361,"lo":null,"hi":null,"n":5100,"of":5100},{"category":"all","metric":"macro_ece","value":0.0597,"lo":null,"hi":null,"n":5100,"of":5100},{"category":"all","metric":"macro_aurc","value":0.0112,"lo":null,"hi":null,"n":5100,"of":5100},{"category":"all","metric":"macro_zero_prob","value":0,"lo":null,"hi":null,"n":5100,"of":5100},{"category":"af","metric":"accuracy","value":0.91,"lo":null,"hi":null,"n":100,"of":100},{"category":"af","metric":"brier","value":0.1474,"lo":null,"hi":null,"n":100,"of":100},{"category":"af","metric":"ece","value":0.0474,"lo":null,"hi":null,"n":100,"of":100},{"category":"af","metric":"aurc","value":0.0088,"lo":null,"hi":null,"n":100,"of":100},{"category":"af","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"am","metric":"accuracy","value":0.85,"lo":null,"hi":null,"n":100,"of":100},{"category":"am","metric":"brier","value":0.2195,"lo":null,"hi":null,"n":100,"of":100},{"category":"am","metric":"ece","value":0.0853,"lo":null,"hi":null,"n":100,"of":100},{"category":"am","metric":"aurc","value":0.0208,"lo":null,"hi":null,"n":100,"of":100},{"category":"am","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"ar","metric":"accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":100},{"category":"ar","metric":"brier","value":0.1231,"lo":null,"hi":null,"n":100,"of":100},{"category":"ar","metric":"ece","value":0.064,"lo":null,"hi":null,"n":100,"of":100},{"category":"ar","metric":"aurc","value":0.0063,"lo":null,"hi":null,"n":100,"of":100},{"category":"ar","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"az","metric":"accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":100},{"category":"az","metric":"brier","value":0.0771,"lo":null,"hi":null,"n":100,"of":100},{"category":"az","metric":"ece","value":0.036,"lo":null,"hi":null,"n":100,"of":100},{"category":"az","metric":"aurc","value":0.0041,"lo":null,"hi":null,"n":100,"of":100},{"category":"az","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"bn","metric":"accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":100},{"category":"bn","metric":"brier","value":0.1418,"lo":null,"hi":null,"n":100,"of":100},{"category":"bn","metric":"ece","value":0.0658,"lo":null,"hi":null,"n":100,"of":100},{"category":"bn","metric":"aurc","value":0.0128,"lo":null,"hi":null,"n":100,"of":100},{"category":"bn","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"cy","metric":"accuracy","value":0.86,"lo":null,"hi":null,"n":100,"of":100},{"category":"cy","metric":"brier","value":0.1966,"lo":null,"hi":null,"n":100,"of":100},{"category":"cy","metric":"ece","value":0.0817,"lo":null,"hi":null,"n":100,"of":100},{"category":"cy","metric":"aurc","value":0.0224,"lo":null,"hi":null,"n":100,"of":100},{"category":"cy","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"da","metric":"accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":100},{"category":"da","metric":"brier","value":0.1216,"lo":null,"hi":null,"n":100,"of":100},{"category":"da","metric":"ece","value":0.0506,"lo":null,"hi":null,"n":100,"of":100},{"category":"da","metric":"aurc","value":0.0066,"lo":null,"hi":null,"n":100,"of":100},{"category":"da","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"de","metric":"accuracy","value":0.91,"lo":null,"hi":null,"n":100,"of":100},{"category":"de","metric":"brier","value":0.1135,"lo":null,"hi":null,"n":100,"of":100},{"category":"de","metric":"ece","value":0.0419,"lo":null,"hi":null,"n":100,"of":100},{"category":"de","metric":"aurc","value":0.0062,"lo":null,"hi":null,"n":100,"of":100},{"category":"de","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"el","metric":"accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":100},{"category":"el","metric":"brier","value":0.0984,"lo":null,"hi":null,"n":100,"of":100},{"category":"el","metric":"ece","value":0.0314,"lo":null,"hi":null,"n":100,"of":100},{"category":"el","metric":"aurc","value":0.005,"lo":null,"hi":null,"n":100,"of":100},{"category":"el","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"en","metric":"accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":100},{"category":"en","metric":"brier","value":0.1031,"lo":null,"hi":null,"n":100,"of":100},{"category":"en","metric":"ece","value":0.0479,"lo":null,"hi":null,"n":100,"of":100},{"category":"en","metric":"aurc","value":0.0049,"lo":null,"hi":null,"n":100,"of":100},{"category":"en","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"es","metric":"accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":100},{"category":"es","metric":"brier","value":0.1299,"lo":null,"hi":null,"n":100,"of":100},{"category":"es","metric":"ece","value":0.0503,"lo":null,"hi":null,"n":100,"of":100},{"category":"es","metric":"aurc","value":0.0127,"lo":null,"hi":null,"n":100,"of":100},{"category":"es","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"fa","metric":"accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":100},{"category":"fa","metric":"brier","value":0.0886,"lo":null,"hi":null,"n":100,"of":100},{"category":"fa","metric":"ece","value":0.0319,"lo":null,"hi":null,"n":100,"of":100},{"category":"fa","metric":"aurc","value":0.0037,"lo":null,"hi":null,"n":100,"of":100},{"category":"fa","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"fi","metric":"accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":100},{"category":"fi","metric":"brier","value":0.1117,"lo":null,"hi":null,"n":100,"of":100},{"category":"fi","metric":"ece","value":0.0551,"lo":null,"hi":null,"n":100,"of":100},{"category":"fi","metric":"aurc","value":0.0196,"lo":null,"hi":null,"n":100,"of":100},{"category":"fi","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"fr","metric":"accuracy","value":0.91,"lo":null,"hi":null,"n":100,"of":100},{"category":"fr","metric":"brier","value":0.1375,"lo":null,"hi":null,"n":100,"of":100},{"category":"fr","metric":"ece","value":0.0739,"lo":null,"hi":null,"n":100,"of":100},{"category":"fr","metric":"aurc","value":0.0107,"lo":null,"hi":null,"n":100,"of":100},{"category":"fr","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"he","metric":"accuracy","value":0.91,"lo":null,"hi":null,"n":100,"of":100},{"category":"he","metric":"brier","value":0.145,"lo":null,"hi":null,"n":100,"of":100},{"category":"he","metric":"ece","value":0.0554,"lo":null,"hi":null,"n":100,"of":100},{"category":"he","metric":"aurc","value":0.0122,"lo":null,"hi":null,"n":100,"of":100},{"category":"he","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"hi","metric":"accuracy","value":0.91,"lo":null,"hi":null,"n":100,"of":100},{"category":"hi","metric":"brier","value":0.1275,"lo":null,"hi":null,"n":100,"of":100},{"category":"hi","metric":"ece","value":0.0493,"lo":null,"hi":null,"n":100,"of":100},{"category":"hi","metric":"aurc","value":0.0071,"lo":null,"hi":null,"n":100,"of":100},{"category":"hi","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"hu","metric":"accuracy","value":0.9,"lo":null,"hi":null,"n":100,"of":100},{"category":"hu","metric":"brier","value":0.1636,"lo":null,"hi":null,"n":100,"of":100},{"category":"hu","metric":"ece","value":0.0794,"lo":null,"hi":null,"n":100,"of":100},{"category":"hu","metric":"aurc","value":0.015,"lo":null,"hi":null,"n":100,"of":100},{"category":"hu","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"hy","metric":"accuracy","value":0.89,"lo":null,"hi":null,"n":100,"of":100},{"category":"hy","metric":"brier","value":0.1609,"lo":null,"hi":null,"n":100,"of":100},{"category":"hy","metric":"ece","value":0.0602,"lo":null,"hi":null,"n":100,"of":100},{"category":"hy","metric":"aurc","value":0.012,"lo":null,"hi":null,"n":100,"of":100},{"category":"hy","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"id","metric":"accuracy","value":0.96,"lo":null,"hi":null,"n":100,"of":100},{"category":"id","metric":"brier","value":0.0749,"lo":null,"hi":null,"n":100,"of":100},{"category":"id","metric":"ece","value":0.0517,"lo":null,"hi":null,"n":100,"of":100},{"category":"id","metric":"aurc","value":0.0051,"lo":null,"hi":null,"n":100,"of":100},{"category":"id","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"is","metric":"accuracy","value":0.85,"lo":null,"hi":null,"n":100,"of":100},{"category":"is","metric":"brier","value":0.1975,"lo":null,"hi":null,"n":100,"of":100},{"category":"is","metric":"ece","value":0.0713,"lo":null,"hi":null,"n":100,"of":100},{"category":"is","metric":"aurc","value":0.0175,"lo":null,"hi":null,"n":100,"of":100},{"category":"is","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"it","metric":"accuracy","value":0.91,"lo":null,"hi":null,"n":100,"of":100},{"category":"it","metric":"brier","value":0.1252,"lo":null,"hi":null,"n":100,"of":100},{"category":"it","metric":"ece","value":0.0568,"lo":null,"hi":null,"n":100,"of":100},{"category":"it","metric":"aurc","value":0.0109,"lo":null,"hi":null,"n":100,"of":100},{"category":"it","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"ja","metric":"accuracy","value":0.97,"lo":null,"hi":null,"n":100,"of":100},{"category":"ja","metric":"brier","value":0.0695,"lo":null,"hi":null,"n":100,"of":100},{"category":"ja","metric":"ece","value":0.0371,"lo":null,"hi":null,"n":100,"of":100},{"category":"ja","metric":"aurc","value":0.0023,"lo":null,"hi":null,"n":100,"of":100},{"category":"ja","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"jv","metric":"accuracy","value":0.89,"lo":null,"hi":null,"n":100,"of":100},{"category":"jv","metric":"brier","value":0.1847,"lo":null,"hi":null,"n":100,"of":100},{"category":"jv","metric":"ece","value":0.0737,"lo":null,"hi":null,"n":100,"of":100},{"category":"jv","metric":"aurc","value":0.0147,"lo":null,"hi":null,"n":100,"of":100},{"category":"jv","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"ka","metric":"accuracy","value":0.83,"lo":null,"hi":null,"n":100,"of":100},{"category":"ka","metric":"brier","value":0.2411,"lo":null,"hi":null,"n":100,"of":100},{"category":"ka","metric":"ece","value":0.0763,"lo":null,"hi":null,"n":100,"of":100},{"category":"ka","metric":"aurc","value":0.0359,"lo":null,"hi":null,"n":100,"of":100},{"category":"ka","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"km","metric":"accuracy","value":0.84,"lo":null,"hi":null,"n":100,"of":100},{"category":"km","metric":"brier","value":0.2347,"lo":null,"hi":null,"n":100,"of":100},{"category":"km","metric":"ece","value":0.079,"lo":null,"hi":null,"n":100,"of":100},{"category":"km","metric":"aurc","value":0.0318,"lo":null,"hi":null,"n":100,"of":100},{"category":"km","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"kn","metric":"accuracy","value":0.91,"lo":null,"hi":null,"n":100,"of":100},{"category":"kn","metric":"brier","value":0.1616,"lo":null,"hi":null,"n":100,"of":100},{"category":"kn","metric":"ece","value":0.0891,"lo":null,"hi":null,"n":100,"of":100},{"category":"kn","metric":"aurc","value":0.0136,"lo":null,"hi":null,"n":100,"of":100},{"category":"kn","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"ko","metric":"accuracy","value":0.91,"lo":null,"hi":null,"n":100,"of":100},{"category":"ko","metric":"brier","value":0.1246,"lo":null,"hi":null,"n":100,"of":100},{"category":"ko","metric":"ece","value":0.0584,"lo":null,"hi":null,"n":100,"of":100},{"category":"ko","metric":"aurc","value":0.0076,"lo":null,"hi":null,"n":100,"of":100},{"category":"ko","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"lv","metric":"accuracy","value":0.89,"lo":null,"hi":null,"n":100,"of":100},{"category":"lv","metric":"brier","value":0.1672,"lo":null,"hi":null,"n":100,"of":100},{"category":"lv","metric":"ece","value":0.0868,"lo":null,"hi":null,"n":100,"of":100},{"category":"lv","metric":"aurc","value":0.0121,"lo":null,"hi":null,"n":100,"of":100},{"category":"lv","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"ml","metric":"accuracy","value":0.87,"lo":null,"hi":null,"n":100,"of":100},{"category":"ml","metric":"brier","value":0.2188,"lo":null,"hi":null,"n":100,"of":100},{"category":"ml","metric":"ece","value":0.0854,"lo":null,"hi":null,"n":100,"of":100},{"category":"ml","metric":"aurc","value":0.0251,"lo":null,"hi":null,"n":100,"of":100},{"category":"ml","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"mn","metric":"accuracy","value":0.88,"lo":null,"hi":null,"n":100,"of":100},{"category":"mn","metric":"brier","value":0.1963,"lo":null,"hi":null,"n":100,"of":100},{"category":"mn","metric":"ece","value":0.1116,"lo":null,"hi":null,"n":100,"of":100},{"category":"mn","metric":"aurc","value":0.0191,"lo":null,"hi":null,"n":100,"of":100},{"category":"mn","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"ms","metric":"accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":100},{"category":"ms","metric":"brier","value":0.1112,"lo":null,"hi":null,"n":100,"of":100},{"category":"ms","metric":"ece","value":0.0394,"lo":null,"hi":null,"n":100,"of":100},{"category":"ms","metric":"aurc","value":0.0113,"lo":null,"hi":null,"n":100,"of":100},{"category":"ms","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"my","metric":"accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":100},{"category":"my","metric":"brier","value":0.1114,"lo":null,"hi":null,"n":100,"of":100},{"category":"my","metric":"ece","value":0.0732,"lo":null,"hi":null,"n":100,"of":100},{"category":"my","metric":"aurc","value":0.0097,"lo":null,"hi":null,"n":100,"of":100},{"category":"my","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"nb","metric":"accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":100},{"category":"nb","metric":"brier","value":0.0881,"lo":null,"hi":null,"n":100,"of":100},{"category":"nb","metric":"ece","value":0.0371,"lo":null,"hi":null,"n":100,"of":100},{"category":"nb","metric":"aurc","value":0.0044,"lo":null,"hi":null,"n":100,"of":100},{"category":"nb","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"nl","metric":"accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":100},{"category":"nl","metric":"brier","value":0.1199,"lo":null,"hi":null,"n":100,"of":100},{"category":"nl","metric":"ece","value":0.0496,"lo":null,"hi":null,"n":100,"of":100},{"category":"nl","metric":"aurc","value":0.0059,"lo":null,"hi":null,"n":100,"of":100},{"category":"nl","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"pl","metric":"accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":100},{"category":"pl","metric":"brier","value":0.103,"lo":null,"hi":null,"n":100,"of":100},{"category":"pl","metric":"ece","value":0.0521,"lo":null,"hi":null,"n":100,"of":100},{"category":"pl","metric":"aurc","value":0.0053,"lo":null,"hi":null,"n":100,"of":100},{"category":"pl","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"pt","metric":"accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":100},{"category":"pt","metric":"brier","value":0.1065,"lo":null,"hi":null,"n":100,"of":100},{"category":"pt","metric":"ece","value":0.0376,"lo":null,"hi":null,"n":100,"of":100},{"category":"pt","metric":"aurc","value":0.0071,"lo":null,"hi":null,"n":100,"of":100},{"category":"pt","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"ro","metric":"accuracy","value":0.91,"lo":null,"hi":null,"n":100,"of":100},{"category":"ro","metric":"brier","value":0.0926,"lo":null,"hi":null,"n":100,"of":100},{"category":"ro","metric":"ece","value":0.0526,"lo":null,"hi":null,"n":100,"of":100},{"category":"ro","metric":"aurc","value":0.005,"lo":null,"hi":null,"n":100,"of":100},{"category":"ro","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"ru","metric":"accuracy","value":0.97,"lo":null,"hi":null,"n":100,"of":100},{"category":"ru","metric":"brier","value":0.0428,"lo":null,"hi":null,"n":100,"of":100},{"category":"ru","metric":"ece","value":0.0352,"lo":null,"hi":null,"n":100,"of":100},{"category":"ru","metric":"aurc","value":0.0008,"lo":null,"hi":null,"n":100,"of":100},{"category":"ru","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"sl","metric":"accuracy","value":0.91,"lo":null,"hi":null,"n":100,"of":100},{"category":"sl","metric":"brier","value":0.1259,"lo":null,"hi":null,"n":100,"of":100},{"category":"sl","metric":"ece","value":0.0537,"lo":null,"hi":null,"n":100,"of":100},{"category":"sl","metric":"aurc","value":0.0077,"lo":null,"hi":null,"n":100,"of":100},{"category":"sl","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"sq","metric":"accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":100},{"category":"sq","metric":"brier","value":0.1093,"lo":null,"hi":null,"n":100,"of":100},{"category":"sq","metric":"ece","value":0.0472,"lo":null,"hi":null,"n":100,"of":100},{"category":"sq","metric":"aurc","value":0.0047,"lo":null,"hi":null,"n":100,"of":100},{"category":"sq","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"sv","metric":"accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":100},{"category":"sv","metric":"brier","value":0.0972,"lo":null,"hi":null,"n":100,"of":100},{"category":"sv","metric":"ece","value":0.043,"lo":null,"hi":null,"n":100,"of":100},{"category":"sv","metric":"aurc","value":0.004,"lo":null,"hi":null,"n":100,"of":100},{"category":"sv","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"sw","metric":"accuracy","value":0.84,"lo":null,"hi":null,"n":100,"of":100},{"category":"sw","metric":"brier","value":0.2192,"lo":null,"hi":null,"n":100,"of":100},{"category":"sw","metric":"ece","value":0.0862,"lo":null,"hi":null,"n":100,"of":100},{"category":"sw","metric":"aurc","value":0.019,"lo":null,"hi":null,"n":100,"of":100},{"category":"sw","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"ta","metric":"accuracy","value":0.88,"lo":null,"hi":null,"n":100,"of":100},{"category":"ta","metric":"brier","value":0.1507,"lo":null,"hi":null,"n":100,"of":100},{"category":"ta","metric":"ece","value":0.0623,"lo":null,"hi":null,"n":100,"of":100},{"category":"ta","metric":"aurc","value":0.01,"lo":null,"hi":null,"n":100,"of":100},{"category":"ta","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"te","metric":"accuracy","value":0.87,"lo":null,"hi":null,"n":100,"of":100},{"category":"te","metric":"brier","value":0.1933,"lo":null,"hi":null,"n":100,"of":100},{"category":"te","metric":"ece","value":0.0855,"lo":null,"hi":null,"n":100,"of":100},{"category":"te","metric":"aurc","value":0.0218,"lo":null,"hi":null,"n":100,"of":100},{"category":"te","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"th","metric":"accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":100},{"category":"th","metric":"brier","value":0.0999,"lo":null,"hi":null,"n":100,"of":100},{"category":"th","metric":"ece","value":0.0392,"lo":null,"hi":null,"n":100,"of":100},{"category":"th","metric":"aurc","value":0.0056,"lo":null,"hi":null,"n":100,"of":100},{"category":"th","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"tl","metric":"accuracy","value":0.91,"lo":null,"hi":null,"n":100,"of":100},{"category":"tl","metric":"brier","value":0.1494,"lo":null,"hi":null,"n":100,"of":100},{"category":"tl","metric":"ece","value":0.0668,"lo":null,"hi":null,"n":100,"of":100},{"category":"tl","metric":"aurc","value":0.0195,"lo":null,"hi":null,"n":100,"of":100},{"category":"tl","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"tr","metric":"accuracy","value":0.9,"lo":null,"hi":null,"n":100,"of":100},{"category":"tr","metric":"brier","value":0.1739,"lo":null,"hi":null,"n":100,"of":100},{"category":"tr","metric":"ece","value":0.0752,"lo":null,"hi":null,"n":100,"of":100},{"category":"tr","metric":"aurc","value":0.0189,"lo":null,"hi":null,"n":100,"of":100},{"category":"tr","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"ur","metric":"accuracy","value":0.9,"lo":null,"hi":null,"n":100,"of":100},{"category":"ur","metric":"brier","value":0.1488,"lo":null,"hi":null,"n":100,"of":100},{"category":"ur","metric":"ece","value":0.0657,"lo":null,"hi":null,"n":100,"of":100},{"category":"ur","metric":"aurc","value":0.0096,"lo":null,"hi":null,"n":100,"of":100},{"category":"ur","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"vi","metric":"accuracy","value":0.91,"lo":null,"hi":null,"n":100,"of":100},{"category":"vi","metric":"brier","value":0.1374,"lo":null,"hi":null,"n":100,"of":100},{"category":"vi","metric":"ece","value":0.0681,"lo":null,"hi":null,"n":100,"of":100},{"category":"vi","metric":"aurc","value":0.0088,"lo":null,"hi":null,"n":100,"of":100},{"category":"vi","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"zh-CN","metric":"accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":100},{"category":"zh-CN","metric":"brier","value":0.0896,"lo":null,"hi":null,"n":100,"of":100},{"category":"zh-CN","metric":"ece","value":0.0587,"lo":null,"hi":null,"n":100,"of":100},{"category":"zh-CN","metric":"aurc","value":0.0032,"lo":null,"hi":null,"n":100,"of":100},{"category":"zh-CN","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"zh-TW","metric":"accuracy","value":0.95,"lo":null,"hi":null,"n":100,"of":100},{"category":"zh-TW","metric":"brier","value":0.0683,"lo":null,"hi":null,"n":100,"of":100},{"category":"zh-TW","metric":"ece","value":0.0321,"lo":null,"hi":null,"n":100,"of":100},{"category":"zh-TW","metric":"aurc","value":0.0029,"lo":null,"hi":null,"n":100,"of":100},{"category":"zh-TW","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100}],"notes":"device metal; dtype bf16; preds sha256:25bfec2f5ad49bc050cc9015161c7ef2dbf07f599f35e9dad153b4a5b6f7b543; scores sha256:ddca48bcf9816e7df2a187251d9f0f00de3b2fc24f22805b34b7bd650a048c81","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"massive-harness-a8.0cfeef05-kai","benchmark":"massive","split":"harness","system":"kai","version":"a8@0cfeef05c879d7f24bbde275a6c23e85e80b6baf06cf9311d529010688d4a697","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"a1a4a73d524cecf8038e30c408ad30b66e71d490","questions":5100,"answered":5100,"when":"","ended":"2026-09-28T16:33:11Z","by":"","completion":"complete","measures":[{"category":"all","metric":"macro_accuracy","value":0.9057,"lo":null,"hi":null,"n":5100,"of":5100},{"category":"all","metric":"macro_brier","value":0.1455,"lo":null,"hi":null,"n":5100,"of":5100},{"category":"all","metric":"macro_ece","value":0.0681,"lo":null,"hi":null,"n":5100,"of":5100},{"category":"all","metric":"macro_aurc","value":0.0126,"lo":null,"hi":null,"n":5100,"of":5100},{"category":"all","metric":"macro_zero_prob","value":0,"lo":null,"hi":null,"n":5100,"of":5100},{"category":"af","metric":"accuracy","value":0.89,"lo":null,"hi":null,"n":100,"of":100},{"category":"af","metric":"brier","value":0.1624,"lo":null,"hi":null,"n":100,"of":100},{"category":"af","metric":"ece","value":0.0527,"lo":null,"hi":null,"n":100,"of":100},{"category":"af","metric":"aurc","value":0.0113,"lo":null,"hi":null,"n":100,"of":100},{"category":"af","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"am","metric":"accuracy","value":0.85,"lo":null,"hi":null,"n":100,"of":100},{"category":"am","metric":"brier","value":0.2444,"lo":null,"hi":null,"n":100,"of":100},{"category":"am","metric":"ece","value":0.116,"lo":null,"hi":null,"n":100,"of":100},{"category":"am","metric":"aurc","value":0.0323,"lo":null,"hi":null,"n":100,"of":100},{"category":"am","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"ar","metric":"accuracy","value":0.89,"lo":null,"hi":null,"n":100,"of":100},{"category":"ar","metric":"brier","value":0.1475,"lo":null,"hi":null,"n":100,"of":100},{"category":"ar","metric":"ece","value":0.0729,"lo":null,"hi":null,"n":100,"of":100},{"category":"ar","metric":"aurc","value":0.0102,"lo":null,"hi":null,"n":100,"of":100},{"category":"ar","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"az","metric":"accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":100},{"category":"az","metric":"brier","value":0.116,"lo":null,"hi":null,"n":100,"of":100},{"category":"az","metric":"ece","value":0.0602,"lo":null,"hi":null,"n":100,"of":100},{"category":"az","metric":"aurc","value":0.009,"lo":null,"hi":null,"n":100,"of":100},{"category":"az","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"bn","metric":"accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":100},{"category":"bn","metric":"brier","value":0.1258,"lo":null,"hi":null,"n":100,"of":100},{"category":"bn","metric":"ece","value":0.0573,"lo":null,"hi":null,"n":100,"of":100},{"category":"bn","metric":"aurc","value":0.0133,"lo":null,"hi":null,"n":100,"of":100},{"category":"bn","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"cy","metric":"accuracy","value":0.87,"lo":null,"hi":null,"n":100,"of":100},{"category":"cy","metric":"brier","value":0.1928,"lo":null,"hi":null,"n":100,"of":100},{"category":"cy","metric":"ece","value":0.0866,"lo":null,"hi":null,"n":100,"of":100},{"category":"cy","metric":"aurc","value":0.0221,"lo":null,"hi":null,"n":100,"of":100},{"category":"cy","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"da","metric":"accuracy","value":0.91,"lo":null,"hi":null,"n":100,"of":100},{"category":"da","metric":"brier","value":0.1369,"lo":null,"hi":null,"n":100,"of":100},{"category":"da","metric":"ece","value":0.0532,"lo":null,"hi":null,"n":100,"of":100},{"category":"da","metric":"aurc","value":0.0074,"lo":null,"hi":null,"n":100,"of":100},{"category":"da","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"de","metric":"accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":100},{"category":"de","metric":"brier","value":0.1134,"lo":null,"hi":null,"n":100,"of":100},{"category":"de","metric":"ece","value":0.0514,"lo":null,"hi":null,"n":100,"of":100},{"category":"de","metric":"aurc","value":0.0066,"lo":null,"hi":null,"n":100,"of":100},{"category":"de","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"el","metric":"accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":100},{"category":"el","metric":"brier","value":0.1305,"lo":null,"hi":null,"n":100,"of":100},{"category":"el","metric":"ece","value":0.0823,"lo":null,"hi":null,"n":100,"of":100},{"category":"el","metric":"aurc","value":0.0091,"lo":null,"hi":null,"n":100,"of":100},{"category":"el","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"en","metric":"accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":100},{"category":"en","metric":"brier","value":0.1167,"lo":null,"hi":null,"n":100,"of":100},{"category":"en","metric":"ece","value":0.0592,"lo":null,"hi":null,"n":100,"of":100},{"category":"en","metric":"aurc","value":0.0062,"lo":null,"hi":null,"n":100,"of":100},{"category":"en","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"es","metric":"accuracy","value":0.91,"lo":null,"hi":null,"n":100,"of":100},{"category":"es","metric":"brier","value":0.1331,"lo":null,"hi":null,"n":100,"of":100},{"category":"es","metric":"ece","value":0.0684,"lo":null,"hi":null,"n":100,"of":100},{"category":"es","metric":"aurc","value":0.0117,"lo":null,"hi":null,"n":100,"of":100},{"category":"es","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"fa","metric":"accuracy","value":0.95,"lo":null,"hi":null,"n":100,"of":100},{"category":"fa","metric":"brier","value":0.0896,"lo":null,"hi":null,"n":100,"of":100},{"category":"fa","metric":"ece","value":0.0471,"lo":null,"hi":null,"n":100,"of":100},{"category":"fa","metric":"aurc","value":0.0047,"lo":null,"hi":null,"n":100,"of":100},{"category":"fa","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"fi","metric":"accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":100},{"category":"fi","metric":"brier","value":0.117,"lo":null,"hi":null,"n":100,"of":100},{"category":"fi","metric":"ece","value":0.0567,"lo":null,"hi":null,"n":100,"of":100},{"category":"fi","metric":"aurc","value":0.014,"lo":null,"hi":null,"n":100,"of":100},{"category":"fi","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"fr","metric":"accuracy","value":0.9,"lo":null,"hi":null,"n":100,"of":100},{"category":"fr","metric":"brier","value":0.1444,"lo":null,"hi":null,"n":100,"of":100},{"category":"fr","metric":"ece","value":0.0731,"lo":null,"hi":null,"n":100,"of":100},{"category":"fr","metric":"aurc","value":0.0139,"lo":null,"hi":null,"n":100,"of":100},{"category":"fr","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"he","metric":"accuracy","value":0.91,"lo":null,"hi":null,"n":100,"of":100},{"category":"he","metric":"brier","value":0.1547,"lo":null,"hi":null,"n":100,"of":100},{"category":"he","metric":"ece","value":0.0778,"lo":null,"hi":null,"n":100,"of":100},{"category":"he","metric":"aurc","value":0.0119,"lo":null,"hi":null,"n":100,"of":100},{"category":"he","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"hi","metric":"accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":100},{"category":"hi","metric":"brier","value":0.1033,"lo":null,"hi":null,"n":100,"of":100},{"category":"hi","metric":"ece","value":0.043,"lo":null,"hi":null,"n":100,"of":100},{"category":"hi","metric":"aurc","value":0.0087,"lo":null,"hi":null,"n":100,"of":100},{"category":"hi","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"hu","metric":"accuracy","value":0.91,"lo":null,"hi":null,"n":100,"of":100},{"category":"hu","metric":"brier","value":0.1604,"lo":null,"hi":null,"n":100,"of":100},{"category":"hu","metric":"ece","value":0.083,"lo":null,"hi":null,"n":100,"of":100},{"category":"hu","metric":"aurc","value":0.0158,"lo":null,"hi":null,"n":100,"of":100},{"category":"hu","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"hy","metric":"accuracy","value":0.89,"lo":null,"hi":null,"n":100,"of":100},{"category":"hy","metric":"brier","value":0.1731,"lo":null,"hi":null,"n":100,"of":100},{"category":"hy","metric":"ece","value":0.0587,"lo":null,"hi":null,"n":100,"of":100},{"category":"hy","metric":"aurc","value":0.0145,"lo":null,"hi":null,"n":100,"of":100},{"category":"hy","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"id","metric":"accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":100},{"category":"id","metric":"brier","value":0.0924,"lo":null,"hi":null,"n":100,"of":100},{"category":"id","metric":"ece","value":0.0445,"lo":null,"hi":null,"n":100,"of":100},{"category":"id","metric":"aurc","value":0.0059,"lo":null,"hi":null,"n":100,"of":100},{"category":"id","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"is","metric":"accuracy","value":0.88,"lo":null,"hi":null,"n":100,"of":100},{"category":"is","metric":"brier","value":0.1818,"lo":null,"hi":null,"n":100,"of":100},{"category":"is","metric":"ece","value":0.0849,"lo":null,"hi":null,"n":100,"of":100},{"category":"is","metric":"aurc","value":0.0146,"lo":null,"hi":null,"n":100,"of":100},{"category":"is","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"it","metric":"accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":100},{"category":"it","metric":"brier","value":0.1058,"lo":null,"hi":null,"n":100,"of":100},{"category":"it","metric":"ece","value":0.057,"lo":null,"hi":null,"n":100,"of":100},{"category":"it","metric":"aurc","value":0.0065,"lo":null,"hi":null,"n":100,"of":100},{"category":"it","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"ja","metric":"accuracy","value":0.96,"lo":null,"hi":null,"n":100,"of":100},{"category":"ja","metric":"brier","value":0.0653,"lo":null,"hi":null,"n":100,"of":100},{"category":"ja","metric":"ece","value":0.0348,"lo":null,"hi":null,"n":100,"of":100},{"category":"ja","metric":"aurc","value":0.0022,"lo":null,"hi":null,"n":100,"of":100},{"category":"ja","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"jv","metric":"accuracy","value":0.88,"lo":null,"hi":null,"n":100,"of":100},{"category":"jv","metric":"brier","value":0.1825,"lo":null,"hi":null,"n":100,"of":100},{"category":"jv","metric":"ece","value":0.0686,"lo":null,"hi":null,"n":100,"of":100},{"category":"jv","metric":"aurc","value":0.0146,"lo":null,"hi":null,"n":100,"of":100},{"category":"jv","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"ka","metric":"accuracy","value":0.8,"lo":null,"hi":null,"n":100,"of":100},{"category":"ka","metric":"brier","value":0.2757,"lo":null,"hi":null,"n":100,"of":100},{"category":"ka","metric":"ece","value":0.1308,"lo":null,"hi":null,"n":100,"of":100},{"category":"ka","metric":"aurc","value":0.0425,"lo":null,"hi":null,"n":100,"of":100},{"category":"ka","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"km","metric":"accuracy","value":0.82,"lo":null,"hi":null,"n":100,"of":100},{"category":"km","metric":"brier","value":0.3015,"lo":null,"hi":null,"n":100,"of":100},{"category":"km","metric":"ece","value":0.1456,"lo":null,"hi":null,"n":100,"of":100},{"category":"km","metric":"aurc","value":0.0429,"lo":null,"hi":null,"n":100,"of":100},{"category":"km","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"kn","metric":"accuracy","value":0.9,"lo":null,"hi":null,"n":100,"of":100},{"category":"kn","metric":"brier","value":0.1589,"lo":null,"hi":null,"n":100,"of":100},{"category":"kn","metric":"ece","value":0.0756,"lo":null,"hi":null,"n":100,"of":100},{"category":"kn","metric":"aurc","value":0.0112,"lo":null,"hi":null,"n":100,"of":100},{"category":"kn","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"ko","metric":"accuracy","value":0.9,"lo":null,"hi":null,"n":100,"of":100},{"category":"ko","metric":"brier","value":0.1512,"lo":null,"hi":null,"n":100,"of":100},{"category":"ko","metric":"ece","value":0.0766,"lo":null,"hi":null,"n":100,"of":100},{"category":"ko","metric":"aurc","value":0.014,"lo":null,"hi":null,"n":100,"of":100},{"category":"ko","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"lv","metric":"accuracy","value":0.89,"lo":null,"hi":null,"n":100,"of":100},{"category":"lv","metric":"brier","value":0.1832,"lo":null,"hi":null,"n":100,"of":100},{"category":"lv","metric":"ece","value":0.0946,"lo":null,"hi":null,"n":100,"of":100},{"category":"lv","metric":"aurc","value":0.0157,"lo":null,"hi":null,"n":100,"of":100},{"category":"lv","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"ml","metric":"accuracy","value":0.88,"lo":null,"hi":null,"n":100,"of":100},{"category":"ml","metric":"brier","value":0.1979,"lo":null,"hi":null,"n":100,"of":100},{"category":"ml","metric":"ece","value":0.0721,"lo":null,"hi":null,"n":100,"of":100},{"category":"ml","metric":"aurc","value":0.0256,"lo":null,"hi":null,"n":100,"of":100},{"category":"ml","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"mn","metric":"accuracy","value":0.88,"lo":null,"hi":null,"n":100,"of":100},{"category":"mn","metric":"brier","value":0.1988,"lo":null,"hi":null,"n":100,"of":100},{"category":"mn","metric":"ece","value":0.095,"lo":null,"hi":null,"n":100,"of":100},{"category":"mn","metric":"aurc","value":0.0191,"lo":null,"hi":null,"n":100,"of":100},{"category":"mn","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"ms","metric":"accuracy","value":0.9,"lo":null,"hi":null,"n":100,"of":100},{"category":"ms","metric":"brier","value":0.1438,"lo":null,"hi":null,"n":100,"of":100},{"category":"ms","metric":"ece","value":0.0723,"lo":null,"hi":null,"n":100,"of":100},{"category":"ms","metric":"aurc","value":0.0122,"lo":null,"hi":null,"n":100,"of":100},{"category":"ms","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"my","metric":"accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":100},{"category":"my","metric":"brier","value":0.1027,"lo":null,"hi":null,"n":100,"of":100},{"category":"my","metric":"ece","value":0.0595,"lo":null,"hi":null,"n":100,"of":100},{"category":"my","metric":"aurc","value":0.0057,"lo":null,"hi":null,"n":100,"of":100},{"category":"my","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"nb","metric":"accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":100},{"category":"nb","metric":"brier","value":0.1054,"lo":null,"hi":null,"n":100,"of":100},{"category":"nb","metric":"ece","value":0.0531,"lo":null,"hi":null,"n":100,"of":100},{"category":"nb","metric":"aurc","value":0.0067,"lo":null,"hi":null,"n":100,"of":100},{"category":"nb","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"nl","metric":"accuracy","value":0.88,"lo":null,"hi":null,"n":100,"of":100},{"category":"nl","metric":"brier","value":0.1612,"lo":null,"hi":null,"n":100,"of":100},{"category":"nl","metric":"ece","value":0.0805,"lo":null,"hi":null,"n":100,"of":100},{"category":"nl","metric":"aurc","value":0.0109,"lo":null,"hi":null,"n":100,"of":100},{"category":"nl","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"pl","metric":"accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":100},{"category":"pl","metric":"brier","value":0.0921,"lo":null,"hi":null,"n":100,"of":100},{"category":"pl","metric":"ece","value":0.0453,"lo":null,"hi":null,"n":100,"of":100},{"category":"pl","metric":"aurc","value":0.0072,"lo":null,"hi":null,"n":100,"of":100},{"category":"pl","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"pt","metric":"accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":100},{"category":"pt","metric":"brier","value":0.1267,"lo":null,"hi":null,"n":100,"of":100},{"category":"pt","metric":"ece","value":0.0468,"lo":null,"hi":null,"n":100,"of":100},{"category":"pt","metric":"aurc","value":0.009,"lo":null,"hi":null,"n":100,"of":100},{"category":"pt","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"ro","metric":"accuracy","value":0.96,"lo":null,"hi":null,"n":100,"of":100},{"category":"ro","metric":"brier","value":0.0808,"lo":null,"hi":null,"n":100,"of":100},{"category":"ro","metric":"ece","value":0.0396,"lo":null,"hi":null,"n":100,"of":100},{"category":"ro","metric":"aurc","value":0.0032,"lo":null,"hi":null,"n":100,"of":100},{"category":"ro","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"ru","metric":"accuracy","value":0.95,"lo":null,"hi":null,"n":100,"of":100},{"category":"ru","metric":"brier","value":0.0727,"lo":null,"hi":null,"n":100,"of":100},{"category":"ru","metric":"ece","value":0.0458,"lo":null,"hi":null,"n":100,"of":100},{"category":"ru","metric":"aurc","value":0.002,"lo":null,"hi":null,"n":100,"of":100},{"category":"ru","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"sl","metric":"accuracy","value":0.88,"lo":null,"hi":null,"n":100,"of":100},{"category":"sl","metric":"brier","value":0.1296,"lo":null,"hi":null,"n":100,"of":100},{"category":"sl","metric":"ece","value":0.0613,"lo":null,"hi":null,"n":100,"of":100},{"category":"sl","metric":"aurc","value":0.0084,"lo":null,"hi":null,"n":100,"of":100},{"category":"sl","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"sq","metric":"accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":100},{"category":"sq","metric":"brier","value":0.1116,"lo":null,"hi":null,"n":100,"of":100},{"category":"sq","metric":"ece","value":0.0541,"lo":null,"hi":null,"n":100,"of":100},{"category":"sq","metric":"aurc","value":0.0051,"lo":null,"hi":null,"n":100,"of":100},{"category":"sq","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"sv","metric":"accuracy","value":0.91,"lo":null,"hi":null,"n":100,"of":100},{"category":"sv","metric":"brier","value":0.1299,"lo":null,"hi":null,"n":100,"of":100},{"category":"sv","metric":"ece","value":0.062,"lo":null,"hi":null,"n":100,"of":100},{"category":"sv","metric":"aurc","value":0.0086,"lo":null,"hi":null,"n":100,"of":100},{"category":"sv","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"sw","metric":"accuracy","value":0.85,"lo":null,"hi":null,"n":100,"of":100},{"category":"sw","metric":"brier","value":0.2242,"lo":null,"hi":null,"n":100,"of":100},{"category":"sw","metric":"ece","value":0.0911,"lo":null,"hi":null,"n":100,"of":100},{"category":"sw","metric":"aurc","value":0.0212,"lo":null,"hi":null,"n":100,"of":100},{"category":"sw","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"ta","metric":"accuracy","value":0.89,"lo":null,"hi":null,"n":100,"of":100},{"category":"ta","metric":"brier","value":0.1746,"lo":null,"hi":null,"n":100,"of":100},{"category":"ta","metric":"ece","value":0.0644,"lo":null,"hi":null,"n":100,"of":100},{"category":"ta","metric":"aurc","value":0.0138,"lo":null,"hi":null,"n":100,"of":100},{"category":"ta","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"te","metric":"accuracy","value":0.87,"lo":null,"hi":null,"n":100,"of":100},{"category":"te","metric":"brier","value":0.1936,"lo":null,"hi":null,"n":100,"of":100},{"category":"te","metric":"ece","value":0.0844,"lo":null,"hi":null,"n":100,"of":100},{"category":"te","metric":"aurc","value":0.0195,"lo":null,"hi":null,"n":100,"of":100},{"category":"te","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"th","metric":"accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":100},{"category":"th","metric":"brier","value":0.0966,"lo":null,"hi":null,"n":100,"of":100},{"category":"th","metric":"ece","value":0.0442,"lo":null,"hi":null,"n":100,"of":100},{"category":"th","metric":"aurc","value":0.0042,"lo":null,"hi":null,"n":100,"of":100},{"category":"th","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"tl","metric":"accuracy","value":0.88,"lo":null,"hi":null,"n":100,"of":100},{"category":"tl","metric":"brier","value":0.173,"lo":null,"hi":null,"n":100,"of":100},{"category":"tl","metric":"ece","value":0.0895,"lo":null,"hi":null,"n":100,"of":100},{"category":"tl","metric":"aurc","value":0.0182,"lo":null,"hi":null,"n":100,"of":100},{"category":"tl","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"tr","metric":"accuracy","value":0.87,"lo":null,"hi":null,"n":100,"of":100},{"category":"tr","metric":"brier","value":0.2015,"lo":null,"hi":null,"n":100,"of":100},{"category":"tr","metric":"ece","value":0.1006,"lo":null,"hi":null,"n":100,"of":100},{"category":"tr","metric":"aurc","value":0.0204,"lo":null,"hi":null,"n":100,"of":100},{"category":"tr","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"ur","metric":"accuracy","value":0.91,"lo":null,"hi":null,"n":100,"of":100},{"category":"ur","metric":"brier","value":0.1502,"lo":null,"hi":null,"n":100,"of":100},{"category":"ur","metric":"ece","value":0.0686,"lo":null,"hi":null,"n":100,"of":100},{"category":"ur","metric":"aurc","value":0.012,"lo":null,"hi":null,"n":100,"of":100},{"category":"ur","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"vi","metric":"accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":100},{"category":"vi","metric":"brier","value":0.136,"lo":null,"hi":null,"n":100,"of":100},{"category":"vi","metric":"ece","value":0.0481,"lo":null,"hi":null,"n":100,"of":100},{"category":"vi","metric":"aurc","value":0.0091,"lo":null,"hi":null,"n":100,"of":100},{"category":"vi","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"zh-CN","metric":"accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":100},{"category":"zh-CN","metric":"brier","value":0.0923,"lo":null,"hi":null,"n":100,"of":100},{"category":"zh-CN","metric":"ece","value":0.0505,"lo":null,"hi":null,"n":100,"of":100},{"category":"zh-CN","metric":"aurc","value":0.0043,"lo":null,"hi":null,"n":100,"of":100},{"category":"zh-CN","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"zh-TW","metric":"accuracy","value":0.96,"lo":null,"hi":null,"n":100,"of":100},{"category":"zh-TW","metric":"brier","value":0.0636,"lo":null,"hi":null,"n":100,"of":100},{"category":"zh-TW","metric":"ece","value":0.0342,"lo":null,"hi":null,"n":100,"of":100},{"category":"zh-TW","metric":"aurc","value":0.0033,"lo":null,"hi":null,"n":100,"of":100},{"category":"zh-TW","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100}],"notes":"device metal; dtype bf16; preds sha256:d0a81ccbdc9eb7305fc196d7ad2c6be8c1df1ce5966ac964dabeeea231ae41d6; scores sha256:86b58e1f53447a8fb3b7963ea69f2bee40ffa97b9bda7b74b038064188998f10","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"massive-harness-english-laya","benchmark":"massive","split":"harness","system":"laya","version":"convaiinnovations/laya@55cf4c4ebb4ebe31b2550e8bdf3bd21b99753851","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"735e240323cd7c9f0c880b7d8c62d89f40be2164","questions":5100,"answered":5100,"when":"","ended":"2026-09-25T17:33:56Z","by":"","completion":"complete","measures":[{"category":"all","metric":"macro_accuracy","value":0.2269,"lo":null,"hi":null,"n":5100,"of":5100},{"category":"all","metric":"macro_brier","value":1.2751,"lo":null,"hi":null,"n":5100,"of":5100},{"category":"all","metric":"macro_ece","value":0.5709,"lo":null,"hi":null,"n":5100,"of":5100},{"category":"all","metric":"macro_aurc","value":0.6716,"lo":null,"hi":null,"n":5100,"of":5100},{"category":"all","metric":"macro_zero_prob","value":0.0802,"lo":null,"hi":null,"n":5100,"of":5100},{"category":"af","metric":"accuracy","value":0.29,"lo":null,"hi":null,"n":100,"of":100},{"category":"af","metric":"brier","value":1.1918,"lo":null,"hi":null,"n":100,"of":100},{"category":"af","metric":"ece","value":0.5588,"lo":null,"hi":null,"n":100,"of":100},{"category":"af","metric":"aurc","value":0.5205,"lo":null,"hi":null,"n":100,"of":100},{"category":"af","metric":"zero_prob","value":0.11,"lo":null,"hi":null,"n":100,"of":100},{"category":"am","metric":"accuracy","value":0.12,"lo":null,"hi":null,"n":100,"of":100},{"category":"am","metric":"brier","value":1.461,"lo":null,"hi":null,"n":100,"of":100},{"category":"am","metric":"ece","value":0.6804,"lo":null,"hi":null,"n":100,"of":100},{"category":"am","metric":"aurc","value":0.8853,"lo":null,"hi":null,"n":100,"of":100},{"category":"am","metric":"zero_prob","value":0.07,"lo":null,"hi":null,"n":100,"of":100},{"category":"ar","metric":"accuracy","value":0.11,"lo":null,"hi":null,"n":100,"of":100},{"category":"ar","metric":"brier","value":1.3826,"lo":null,"hi":null,"n":100,"of":100},{"category":"ar","metric":"ece","value":0.5795,"lo":null,"hi":null,"n":100,"of":100},{"category":"ar","metric":"aurc","value":0.8012,"lo":null,"hi":null,"n":100,"of":100},{"category":"ar","metric":"zero_prob","value":0.02,"lo":null,"hi":null,"n":100,"of":100},{"category":"az","metric":"accuracy","value":0.1,"lo":null,"hi":null,"n":100,"of":100},{"category":"az","metric":"brier","value":1.4097,"lo":null,"hi":null,"n":100,"of":100},{"category":"az","metric":"ece","value":0.5983,"lo":null,"hi":null,"n":100,"of":100},{"category":"az","metric":"aurc","value":0.7809,"lo":null,"hi":null,"n":100,"of":100},{"category":"az","metric":"zero_prob","value":0.05,"lo":null,"hi":null,"n":100,"of":100},{"category":"bn","metric":"accuracy","value":0.08,"lo":null,"hi":null,"n":100,"of":100},{"category":"bn","metric":"brier","value":1.4117,"lo":null,"hi":null,"n":100,"of":100},{"category":"bn","metric":"ece","value":0.6267,"lo":null,"hi":null,"n":100,"of":100},{"category":"bn","metric":"aurc","value":0.8268,"lo":null,"hi":null,"n":100,"of":100},{"category":"bn","metric":"zero_prob","value":0.02,"lo":null,"hi":null,"n":100,"of":100},{"category":"cy","metric":"accuracy","value":0.12,"lo":null,"hi":null,"n":100,"of":100},{"category":"cy","metric":"brier","value":1.4675,"lo":null,"hi":null,"n":100,"of":100},{"category":"cy","metric":"ece","value":0.6452,"lo":null,"hi":null,"n":100,"of":100},{"category":"cy","metric":"aurc","value":0.8194,"lo":null,"hi":null,"n":100,"of":100},{"category":"cy","metric":"zero_prob","value":0.06,"lo":null,"hi":null,"n":100,"of":100},{"category":"da","metric":"accuracy","value":0.35,"lo":null,"hi":null,"n":100,"of":100},{"category":"da","metric":"brier","value":1.1294,"lo":null,"hi":null,"n":100,"of":100},{"category":"da","metric":"ece","value":0.526,"lo":null,"hi":null,"n":100,"of":100},{"category":"da","metric":"aurc","value":0.3864,"lo":null,"hi":null,"n":100,"of":100},{"category":"da","metric":"zero_prob","value":0.07,"lo":null,"hi":null,"n":100,"of":100},{"category":"de","metric":"accuracy","value":0.42,"lo":null,"hi":null,"n":100,"of":100},{"category":"de","metric":"brier","value":1.0025,"lo":null,"hi":null,"n":100,"of":100},{"category":"de","metric":"ece","value":0.452,"lo":null,"hi":null,"n":100,"of":100},{"category":"de","metric":"aurc","value":0.3333,"lo":null,"hi":null,"n":100,"of":100},{"category":"de","metric":"zero_prob","value":0.08,"lo":null,"hi":null,"n":100,"of":100},{"category":"el","metric":"accuracy","value":0.13,"lo":null,"hi":null,"n":100,"of":100},{"category":"el","metric":"brier","value":1.4564,"lo":null,"hi":null,"n":100,"of":100},{"category":"el","metric":"ece","value":0.6789,"lo":null,"hi":null,"n":100,"of":100},{"category":"el","metric":"aurc","value":0.7909,"lo":null,"hi":null,"n":100,"of":100},{"category":"el","metric":"zero_prob","value":0.13,"lo":null,"hi":null,"n":100,"of":100},{"category":"en","metric":"accuracy","value":0.82,"lo":null,"hi":null,"n":100,"of":100},{"category":"en","metric":"brier","value":0.2882,"lo":null,"hi":null,"n":100,"of":100},{"category":"en","metric":"ece","value":0.1382,"lo":null,"hi":null,"n":100,"of":100},{"category":"en","metric":"aurc","value":0.0386,"lo":null,"hi":null,"n":100,"of":100},{"category":"en","metric":"zero_prob","value":0.02,"lo":null,"hi":null,"n":100,"of":100},{"category":"es","metric":"accuracy","value":0.51,"lo":null,"hi":null,"n":100,"of":100},{"category":"es","metric":"brier","value":0.8845,"lo":null,"hi":null,"n":100,"of":100},{"category":"es","metric":"ece","value":0.4225,"lo":null,"hi":null,"n":100,"of":100},{"category":"es","metric":"aurc","value":0.3235,"lo":null,"hi":null,"n":100,"of":100},{"category":"es","metric":"zero_prob","value":0.16,"lo":null,"hi":null,"n":100,"of":100},{"category":"fa","metric":"accuracy","value":0.14,"lo":null,"hi":null,"n":100,"of":100},{"category":"fa","metric":"brier","value":1.453,"lo":null,"hi":null,"n":100,"of":100},{"category":"fa","metric":"ece","value":0.6331,"lo":null,"hi":null,"n":100,"of":100},{"category":"fa","metric":"aurc","value":0.8445,"lo":null,"hi":null,"n":100,"of":100},{"category":"fa","metric":"zero_prob","value":0.09,"lo":null,"hi":null,"n":100,"of":100},{"category":"fi","metric":"accuracy","value":0.13,"lo":null,"hi":null,"n":100,"of":100},{"category":"fi","metric":"brier","value":1.5179,"lo":null,"hi":null,"n":100,"of":100},{"category":"fi","metric":"ece","value":0.7189,"lo":null,"hi":null,"n":100,"of":100},{"category":"fi","metric":"aurc","value":0.7984,"lo":null,"hi":null,"n":100,"of":100},{"category":"fi","metric":"zero_prob","value":0.13,"lo":null,"hi":null,"n":100,"of":100},{"category":"fr","metric":"accuracy","value":0.59,"lo":null,"hi":null,"n":100,"of":100},{"category":"fr","metric":"brier","value":0.694,"lo":null,"hi":null,"n":100,"of":100},{"category":"fr","metric":"ece","value":0.2995,"lo":null,"hi":null,"n":100,"of":100},{"category":"fr","metric":"aurc","value":0.2064,"lo":null,"hi":null,"n":100,"of":100},{"category":"fr","metric":"zero_prob","value":0.05,"lo":null,"hi":null,"n":100,"of":100},{"category":"he","metric":"accuracy","value":0.06,"lo":null,"hi":null,"n":100,"of":100},{"category":"he","metric":"brier","value":1.6073,"lo":null,"hi":null,"n":100,"of":100},{"category":"he","metric":"ece","value":0.7379,"lo":null,"hi":null,"n":100,"of":100},{"category":"he","metric":"aurc","value":0.9793,"lo":null,"hi":null,"n":100,"of":100},{"category":"he","metric":"zero_prob","value":0.11,"lo":null,"hi":null,"n":100,"of":100},{"category":"hi","metric":"accuracy","value":0.1,"lo":null,"hi":null,"n":100,"of":100},{"category":"hi","metric":"brier","value":1.4552,"lo":null,"hi":null,"n":100,"of":100},{"category":"hi","metric":"ece","value":0.6416,"lo":null,"hi":null,"n":100,"of":100},{"category":"hi","metric":"aurc","value":0.8837,"lo":null,"hi":null,"n":100,"of":100},{"category":"hi","metric":"zero_prob","value":0.08,"lo":null,"hi":null,"n":100,"of":100},{"category":"hu","metric":"accuracy","value":0.09,"lo":null,"hi":null,"n":100,"of":100},{"category":"hu","metric":"brier","value":1.5325,"lo":null,"hi":null,"n":100,"of":100},{"category":"hu","metric":"ece","value":0.6783,"lo":null,"hi":null,"n":100,"of":100},{"category":"hu","metric":"aurc","value":0.9155,"lo":null,"hi":null,"n":100,"of":100},{"category":"hu","metric":"zero_prob","value":0.14,"lo":null,"hi":null,"n":100,"of":100},{"category":"hy","metric":"accuracy","value":0.05,"lo":null,"hi":null,"n":100,"of":100},{"category":"hy","metric":"brier","value":1.4019,"lo":null,"hi":null,"n":100,"of":100},{"category":"hy","metric":"ece","value":0.5709,"lo":null,"hi":null,"n":100,"of":100},{"category":"hy","metric":"aurc","value":0.9629,"lo":null,"hi":null,"n":100,"of":100},{"category":"hy","metric":"zero_prob","value":0.01,"lo":null,"hi":null,"n":100,"of":100},{"category":"id","metric":"accuracy","value":0.36,"lo":null,"hi":null,"n":100,"of":100},{"category":"id","metric":"brier","value":1.1037,"lo":null,"hi":null,"n":100,"of":100},{"category":"id","metric":"ece","value":0.487,"lo":null,"hi":null,"n":100,"of":100},{"category":"id","metric":"aurc","value":0.4559,"lo":null,"hi":null,"n":100,"of":100},{"category":"id","metric":"zero_prob","value":0.06,"lo":null,"hi":null,"n":100,"of":100},{"category":"is","metric":"accuracy","value":0.11,"lo":null,"hi":null,"n":100,"of":100},{"category":"is","metric":"brier","value":1.4728,"lo":null,"hi":null,"n":100,"of":100},{"category":"is","metric":"ece","value":0.6641,"lo":null,"hi":null,"n":100,"of":100},{"category":"is","metric":"aurc","value":0.8328,"lo":null,"hi":null,"n":100,"of":100},{"category":"is","metric":"zero_prob","value":0.16,"lo":null,"hi":null,"n":100,"of":100},{"category":"it","metric":"accuracy","value":0.34,"lo":null,"hi":null,"n":100,"of":100},{"category":"it","metric":"brier","value":1.1115,"lo":null,"hi":null,"n":100,"of":100},{"category":"it","metric":"ece","value":0.5156,"lo":null,"hi":null,"n":100,"of":100},{"category":"it","metric":"aurc","value":0.4051,"lo":null,"hi":null,"n":100,"of":100},{"category":"it","metric":"zero_prob","value":0.1,"lo":null,"hi":null,"n":100,"of":100},{"category":"ja","metric":"accuracy","value":0.53,"lo":null,"hi":null,"n":100,"of":100},{"category":"ja","metric":"brier","value":0.832,"lo":null,"hi":null,"n":100,"of":100},{"category":"ja","metric":"ece","value":0.3657,"lo":null,"hi":null,"n":100,"of":100},{"category":"ja","metric":"aurc","value":0.2844,"lo":null,"hi":null,"n":100,"of":100},{"category":"ja","metric":"zero_prob","value":0.06,"lo":null,"hi":null,"n":100,"of":100},{"category":"jv","metric":"accuracy","value":0.16,"lo":null,"hi":null,"n":100,"of":100},{"category":"jv","metric":"brier","value":1.456,"lo":null,"hi":null,"n":100,"of":100},{"category":"jv","metric":"ece","value":0.67,"lo":null,"hi":null,"n":100,"of":100},{"category":"jv","metric":"aurc","value":0.7935,"lo":null,"hi":null,"n":100,"of":100},{"category":"jv","metric":"zero_prob","value":0.11,"lo":null,"hi":null,"n":100,"of":100},{"category":"ka","metric":"accuracy","value":0.09,"lo":null,"hi":null,"n":100,"of":100},{"category":"ka","metric":"brier","value":1.4279,"lo":null,"hi":null,"n":100,"of":100},{"category":"ka","metric":"ece","value":0.6102,"lo":null,"hi":null,"n":100,"of":100},{"category":"ka","metric":"aurc","value":0.9134,"lo":null,"hi":null,"n":100,"of":100},{"category":"ka","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"km","metric":"accuracy","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"km","metric":"brier","value":1.5573,"lo":null,"hi":null,"n":100,"of":100},{"category":"km","metric":"ece","value":0.7051,"lo":null,"hi":null,"n":100,"of":100},{"category":"km","metric":"aurc","value":1,"lo":null,"hi":null,"n":100,"of":100},{"category":"km","metric":"zero_prob","value":0.03,"lo":null,"hi":null,"n":100,"of":100},{"category":"kn","metric":"accuracy","value":0.11,"lo":null,"hi":null,"n":100,"of":100},{"category":"kn","metric":"brier","value":1.4055,"lo":null,"hi":null,"n":100,"of":100},{"category":"kn","metric":"ece","value":0.5903,"lo":null,"hi":null,"n":100,"of":100},{"category":"kn","metric":"aurc","value":0.9276,"lo":null,"hi":null,"n":100,"of":100},{"category":"kn","metric":"zero_prob","value":0.01,"lo":null,"hi":null,"n":100,"of":100},{"category":"ko","metric":"accuracy","value":0.11,"lo":null,"hi":null,"n":100,"of":100},{"category":"ko","metric":"brier","value":1.5102,"lo":null,"hi":null,"n":100,"of":100},{"category":"ko","metric":"ece","value":0.6804,"lo":null,"hi":null,"n":100,"of":100},{"category":"ko","metric":"aurc","value":0.8448,"lo":null,"hi":null,"n":100,"of":100},{"category":"ko","metric":"zero_prob","value":0.12,"lo":null,"hi":null,"n":100,"of":100},{"category":"lv","metric":"accuracy","value":0.1,"lo":null,"hi":null,"n":100,"of":100},{"category":"lv","metric":"brier","value":1.4871,"lo":null,"hi":null,"n":100,"of":100},{"category":"lv","metric":"ece","value":0.6659,"lo":null,"hi":null,"n":100,"of":100},{"category":"lv","metric":"aurc","value":0.885,"lo":null,"hi":null,"n":100,"of":100},{"category":"lv","metric":"zero_prob","value":0.08,"lo":null,"hi":null,"n":100,"of":100},{"category":"ml","metric":"accuracy","value":0.07,"lo":null,"hi":null,"n":100,"of":100},{"category":"ml","metric":"brier","value":1.3834,"lo":null,"hi":null,"n":100,"of":100},{"category":"ml","metric":"ece","value":0.601,"lo":null,"hi":null,"n":100,"of":100},{"category":"ml","metric":"aurc","value":0.8316,"lo":null,"hi":null,"n":100,"of":100},{"category":"ml","metric":"zero_prob","value":0.02,"lo":null,"hi":null,"n":100,"of":100},{"category":"mn","metric":"accuracy","value":0.13,"lo":null,"hi":null,"n":100,"of":100},{"category":"mn","metric":"brier","value":1.4925,"lo":null,"hi":null,"n":100,"of":100},{"category":"mn","metric":"ece","value":0.6704,"lo":null,"hi":null,"n":100,"of":100},{"category":"mn","metric":"aurc","value":0.887,"lo":null,"hi":null,"n":100,"of":100},{"category":"mn","metric":"zero_prob","value":0.1,"lo":null,"hi":null,"n":100,"of":100},{"category":"ms","metric":"accuracy","value":0.27,"lo":null,"hi":null,"n":100,"of":100},{"category":"ms","metric":"brier","value":1.1586,"lo":null,"hi":null,"n":100,"of":100},{"category":"ms","metric":"ece","value":0.5064,"lo":null,"hi":null,"n":100,"of":100},{"category":"ms","metric":"aurc","value":0.546,"lo":null,"hi":null,"n":100,"of":100},{"category":"ms","metric":"zero_prob","value":0.06,"lo":null,"hi":null,"n":100,"of":100},{"category":"my","metric":"accuracy","value":0.06,"lo":null,"hi":null,"n":100,"of":100},{"category":"my","metric":"brier","value":1.3653,"lo":null,"hi":null,"n":100,"of":100},{"category":"my","metric":"ece","value":0.5797,"lo":null,"hi":null,"n":100,"of":100},{"category":"my","metric":"aurc","value":0.9128,"lo":null,"hi":null,"n":100,"of":100},{"category":"my","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"nb","metric":"accuracy","value":0.33,"lo":null,"hi":null,"n":100,"of":100},{"category":"nb","metric":"brier","value":1.1373,"lo":null,"hi":null,"n":100,"of":100},{"category":"nb","metric":"ece","value":0.5179,"lo":null,"hi":null,"n":100,"of":100},{"category":"nb","metric":"aurc","value":0.4227,"lo":null,"hi":null,"n":100,"of":100},{"category":"nb","metric":"zero_prob","value":0.09,"lo":null,"hi":null,"n":100,"of":100},{"category":"nl","metric":"accuracy","value":0.39,"lo":null,"hi":null,"n":100,"of":100},{"category":"nl","metric":"brier","value":1.0787,"lo":null,"hi":null,"n":100,"of":100},{"category":"nl","metric":"ece","value":0.5025,"lo":null,"hi":null,"n":100,"of":100},{"category":"nl","metric":"aurc","value":0.4603,"lo":null,"hi":null,"n":100,"of":100},{"category":"nl","metric":"zero_prob","value":0.18,"lo":null,"hi":null,"n":100,"of":100},{"category":"pl","metric":"accuracy","value":0.24,"lo":null,"hi":null,"n":100,"of":100},{"category":"pl","metric":"brier","value":1.2812,"lo":null,"hi":null,"n":100,"of":100},{"category":"pl","metric":"ece","value":0.5695,"lo":null,"hi":null,"n":100,"of":100},{"category":"pl","metric":"aurc","value":0.5929,"lo":null,"hi":null,"n":100,"of":100},{"category":"pl","metric":"zero_prob","value":0.08,"lo":null,"hi":null,"n":100,"of":100},{"category":"pt","metric":"accuracy","value":0.47,"lo":null,"hi":null,"n":100,"of":100},{"category":"pt","metric":"brier","value":0.9253,"lo":null,"hi":null,"n":100,"of":100},{"category":"pt","metric":"ece","value":0.4317,"lo":null,"hi":null,"n":100,"of":100},{"category":"pt","metric":"aurc","value":0.3301,"lo":null,"hi":null,"n":100,"of":100},{"category":"pt","metric":"zero_prob","value":0.15,"lo":null,"hi":null,"n":100,"of":100},{"category":"ro","metric":"accuracy","value":0.33,"lo":null,"hi":null,"n":100,"of":100},{"category":"ro","metric":"brier","value":1.144,"lo":null,"hi":null,"n":100,"of":100},{"category":"ro","metric":"ece","value":0.5499,"lo":null,"hi":null,"n":100,"of":100},{"category":"ro","metric":"aurc","value":0.4988,"lo":null,"hi":null,"n":100,"of":100},{"category":"ro","metric":"zero_prob","value":0.11,"lo":null,"hi":null,"n":100,"of":100},{"category":"ru","metric":"accuracy","value":0.31,"lo":null,"hi":null,"n":100,"of":100},{"category":"ru","metric":"brier","value":1.228,"lo":null,"hi":null,"n":100,"of":100},{"category":"ru","metric":"ece","value":0.5946,"lo":null,"hi":null,"n":100,"of":100},{"category":"ru","metric":"aurc","value":0.5355,"lo":null,"hi":null,"n":100,"of":100},{"category":"ru","metric":"zero_prob","value":0.15,"lo":null,"hi":null,"n":100,"of":100},{"category":"sl","metric":"accuracy","value":0.2,"lo":null,"hi":null,"n":100,"of":100},{"category":"sl","metric":"brier","value":1.3626,"lo":null,"hi":null,"n":100,"of":100},{"category":"sl","metric":"ece","value":0.5994,"lo":null,"hi":null,"n":100,"of":100},{"category":"sl","metric":"aurc","value":0.6777,"lo":null,"hi":null,"n":100,"of":100},{"category":"sl","metric":"zero_prob","value":0.12,"lo":null,"hi":null,"n":100,"of":100},{"category":"sq","metric":"accuracy","value":0.21,"lo":null,"hi":null,"n":100,"of":100},{"category":"sq","metric":"brier","value":1.3494,"lo":null,"hi":null,"n":100,"of":100},{"category":"sq","metric":"ece","value":0.6031,"lo":null,"hi":null,"n":100,"of":100},{"category":"sq","metric":"aurc","value":0.684,"lo":null,"hi":null,"n":100,"of":100},{"category":"sq","metric":"zero_prob","value":0.16,"lo":null,"hi":null,"n":100,"of":100},{"category":"sv","metric":"accuracy","value":0.38,"lo":null,"hi":null,"n":100,"of":100},{"category":"sv","metric":"brier","value":1.0817,"lo":null,"hi":null,"n":100,"of":100},{"category":"sv","metric":"ece","value":0.4874,"lo":null,"hi":null,"n":100,"of":100},{"category":"sv","metric":"aurc","value":0.445,"lo":null,"hi":null,"n":100,"of":100},{"category":"sv","metric":"zero_prob","value":0.09,"lo":null,"hi":null,"n":100,"of":100},{"category":"sw","metric":"accuracy","value":0.13,"lo":null,"hi":null,"n":100,"of":100},{"category":"sw","metric":"brier","value":1.4341,"lo":null,"hi":null,"n":100,"of":100},{"category":"sw","metric":"ece","value":0.6318,"lo":null,"hi":null,"n":100,"of":100},{"category":"sw","metric":"aurc","value":0.842,"lo":null,"hi":null,"n":100,"of":100},{"category":"sw","metric":"zero_prob","value":0.06,"lo":null,"hi":null,"n":100,"of":100},{"category":"ta","metric":"accuracy","value":0.12,"lo":null,"hi":null,"n":100,"of":100},{"category":"ta","metric":"brier","value":1.3544,"lo":null,"hi":null,"n":100,"of":100},{"category":"ta","metric":"ece","value":0.5541,"lo":null,"hi":null,"n":100,"of":100},{"category":"ta","metric":"aurc","value":0.8994,"lo":null,"hi":null,"n":100,"of":100},{"category":"ta","metric":"zero_prob","value":0.02,"lo":null,"hi":null,"n":100,"of":100},{"category":"te","metric":"accuracy","value":0.09,"lo":null,"hi":null,"n":100,"of":100},{"category":"te","metric":"brier","value":1.5148,"lo":null,"hi":null,"n":100,"of":100},{"category":"te","metric":"ece","value":0.6689,"lo":null,"hi":null,"n":100,"of":100},{"category":"te","metric":"aurc","value":0.9438,"lo":null,"hi":null,"n":100,"of":100},{"category":"te","metric":"zero_prob","value":0.02,"lo":null,"hi":null,"n":100,"of":100},{"category":"th","metric":"accuracy","value":0.08,"lo":null,"hi":null,"n":100,"of":100},{"category":"th","metric":"brier","value":1.5336,"lo":null,"hi":null,"n":100,"of":100},{"category":"th","metric":"ece","value":0.7179,"lo":null,"hi":null,"n":100,"of":100},{"category":"th","metric":"aurc","value":0.8998,"lo":null,"hi":null,"n":100,"of":100},{"category":"th","metric":"zero_prob","value":0.11,"lo":null,"hi":null,"n":100,"of":100},{"category":"tl","metric":"accuracy","value":0.29,"lo":null,"hi":null,"n":100,"of":100},{"category":"tl","metric":"brier","value":1.1543,"lo":null,"hi":null,"n":100,"of":100},{"category":"tl","metric":"ece","value":0.5121,"lo":null,"hi":null,"n":100,"of":100},{"category":"tl","metric":"aurc","value":0.4547,"lo":null,"hi":null,"n":100,"of":100},{"category":"tl","metric":"zero_prob","value":0.07,"lo":null,"hi":null,"n":100,"of":100},{"category":"tr","metric":"accuracy","value":0.14,"lo":null,"hi":null,"n":100,"of":100},{"category":"tr","metric":"brier","value":1.3509,"lo":null,"hi":null,"n":100,"of":100},{"category":"tr","metric":"ece","value":0.5874,"lo":null,"hi":null,"n":100,"of":100},{"category":"tr","metric":"aurc","value":0.7108,"lo":null,"hi":null,"n":100,"of":100},{"category":"tr","metric":"zero_prob","value":0.06,"lo":null,"hi":null,"n":100,"of":100},{"category":"ur","metric":"accuracy","value":0.07,"lo":null,"hi":null,"n":100,"of":100},{"category":"ur","metric":"brier","value":1.4593,"lo":null,"hi":null,"n":100,"of":100},{"category":"ur","metric":"ece","value":0.6428,"lo":null,"hi":null,"n":100,"of":100},{"category":"ur","metric":"aurc","value":0.8953,"lo":null,"hi":null,"n":100,"of":100},{"category":"ur","metric":"zero_prob","value":0.05,"lo":null,"hi":null,"n":100,"of":100},{"category":"vi","metric":"accuracy","value":0.06,"lo":null,"hi":null,"n":100,"of":100},{"category":"vi","metric":"brier","value":1.5327,"lo":null,"hi":null,"n":100,"of":100},{"category":"vi","metric":"ece","value":0.69,"lo":null,"hi":null,"n":100,"of":100},{"category":"vi","metric":"aurc","value":0.886,"lo":null,"hi":null,"n":100,"of":100},{"category":"vi","metric":"zero_prob","value":0.06,"lo":null,"hi":null,"n":100,"of":100},{"category":"zh-CN","metric":"accuracy","value":0.62,"lo":null,"hi":null,"n":100,"of":100},{"category":"zh-CN","metric":"brier","value":0.6745,"lo":null,"hi":null,"n":100,"of":100},{"category":"zh-CN","metric":"ece","value":0.3198,"lo":null,"hi":null,"n":100,"of":100},{"category":"zh-CN","metric":"aurc","value":0.1931,"lo":null,"hi":null,"n":100,"of":100},{"category":"zh-CN","metric":"zero_prob","value":0.07,"lo":null,"hi":null,"n":100,"of":100},{"category":"zh-TW","metric":"accuracy","value":0.46,"lo":null,"hi":null,"n":100,"of":100},{"category":"zh-TW","metric":"brier","value":0.9242,"lo":null,"hi":null,"n":100,"of":100},{"category":"zh-TW","metric":"ece","value":0.4344,"lo":null,"hi":null,"n":100,"of":100},{"category":"zh-TW","metric":"aurc","value":0.2625,"lo":null,"hi":null,"n":100,"of":100},{"category":"zh-TW","metric":"zero_prob","value":0.13,"lo":null,"hi":null,"n":100,"of":100}],"notes":"runtime laya 0.3.20; device cpu; dtype f32; preds sha256:15e4ba94d76db187424d43e7823ffbd029214f6552bf91aade0037448ce694c6","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"massive-harness-kai-1.0834a74f-kai","benchmark":"massive","split":"harness","system":"kai","version":"kai-1@0834a74f2d140642a453373da09e5a128e3d2c8d9e8bb1dcaf86d7210a4ecdfc","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"2c74e2d61addc885e847358975fb866b03cf17dc","questions":5100,"answered":5100,"when":"","ended":"2026-10-03T00:06:45Z","by":"","completion":"complete","measures":[{"category":"all","metric":"macro_accuracy","value":0.9096,"lo":null,"hi":null,"n":5100,"of":5100},{"category":"all","metric":"macro_brier","value":0.1358,"lo":null,"hi":null,"n":5100,"of":5100},{"category":"all","metric":"macro_ece","value":0.06,"lo":null,"hi":null,"n":5100,"of":5100},{"category":"all","metric":"macro_aurc","value":0.0112,"lo":null,"hi":null,"n":5100,"of":5100},{"category":"all","metric":"macro_zero_prob","value":0,"lo":null,"hi":null,"n":5100,"of":5100},{"category":"af","metric":"accuracy","value":0.9,"lo":null,"hi":null,"n":100,"of":100},{"category":"af","metric":"brier","value":0.1488,"lo":null,"hi":null,"n":100,"of":100},{"category":"af","metric":"ece","value":0.0555,"lo":null,"hi":null,"n":100,"of":100},{"category":"af","metric":"aurc","value":0.0093,"lo":null,"hi":null,"n":100,"of":100},{"category":"af","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"am","metric":"accuracy","value":0.85,"lo":null,"hi":null,"n":100,"of":100},{"category":"am","metric":"brier","value":0.2193,"lo":null,"hi":null,"n":100,"of":100},{"category":"am","metric":"ece","value":0.0846,"lo":null,"hi":null,"n":100,"of":100},{"category":"am","metric":"aurc","value":0.0217,"lo":null,"hi":null,"n":100,"of":100},{"category":"am","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"ar","metric":"accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":100},{"category":"ar","metric":"brier","value":0.1228,"lo":null,"hi":null,"n":100,"of":100},{"category":"ar","metric":"ece","value":0.0636,"lo":null,"hi":null,"n":100,"of":100},{"category":"ar","metric":"aurc","value":0.0065,"lo":null,"hi":null,"n":100,"of":100},{"category":"ar","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"az","metric":"accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":100},{"category":"az","metric":"brier","value":0.075,"lo":null,"hi":null,"n":100,"of":100},{"category":"az","metric":"ece","value":0.0451,"lo":null,"hi":null,"n":100,"of":100},{"category":"az","metric":"aurc","value":0.0042,"lo":null,"hi":null,"n":100,"of":100},{"category":"az","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"bn","metric":"accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":100},{"category":"bn","metric":"brier","value":0.1423,"lo":null,"hi":null,"n":100,"of":100},{"category":"bn","metric":"ece","value":0.0619,"lo":null,"hi":null,"n":100,"of":100},{"category":"bn","metric":"aurc","value":0.0126,"lo":null,"hi":null,"n":100,"of":100},{"category":"bn","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"cy","metric":"accuracy","value":0.85,"lo":null,"hi":null,"n":100,"of":100},{"category":"cy","metric":"brier","value":0.1932,"lo":null,"hi":null,"n":100,"of":100},{"category":"cy","metric":"ece","value":0.0744,"lo":null,"hi":null,"n":100,"of":100},{"category":"cy","metric":"aurc","value":0.023,"lo":null,"hi":null,"n":100,"of":100},{"category":"cy","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"da","metric":"accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":100},{"category":"da","metric":"brier","value":0.1216,"lo":null,"hi":null,"n":100,"of":100},{"category":"da","metric":"ece","value":0.051,"lo":null,"hi":null,"n":100,"of":100},{"category":"da","metric":"aurc","value":0.0065,"lo":null,"hi":null,"n":100,"of":100},{"category":"da","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"de","metric":"accuracy","value":0.91,"lo":null,"hi":null,"n":100,"of":100},{"category":"de","metric":"brier","value":0.1143,"lo":null,"hi":null,"n":100,"of":100},{"category":"de","metric":"ece","value":0.0466,"lo":null,"hi":null,"n":100,"of":100},{"category":"de","metric":"aurc","value":0.0062,"lo":null,"hi":null,"n":100,"of":100},{"category":"de","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"el","metric":"accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":100},{"category":"el","metric":"brier","value":0.0984,"lo":null,"hi":null,"n":100,"of":100},{"category":"el","metric":"ece","value":0.0412,"lo":null,"hi":null,"n":100,"of":100},{"category":"el","metric":"aurc","value":0.0048,"lo":null,"hi":null,"n":100,"of":100},{"category":"el","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"en","metric":"accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":100},{"category":"en","metric":"brier","value":0.1035,"lo":null,"hi":null,"n":100,"of":100},{"category":"en","metric":"ece","value":0.044,"lo":null,"hi":null,"n":100,"of":100},{"category":"en","metric":"aurc","value":0.0046,"lo":null,"hi":null,"n":100,"of":100},{"category":"en","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"es","metric":"accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":100},{"category":"es","metric":"brier","value":0.1293,"lo":null,"hi":null,"n":100,"of":100},{"category":"es","metric":"ece","value":0.049,"lo":null,"hi":null,"n":100,"of":100},{"category":"es","metric":"aurc","value":0.0124,"lo":null,"hi":null,"n":100,"of":100},{"category":"es","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"fa","metric":"accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":100},{"category":"fa","metric":"brier","value":0.0889,"lo":null,"hi":null,"n":100,"of":100},{"category":"fa","metric":"ece","value":0.0318,"lo":null,"hi":null,"n":100,"of":100},{"category":"fa","metric":"aurc","value":0.0037,"lo":null,"hi":null,"n":100,"of":100},{"category":"fa","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"fi","metric":"accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":100},{"category":"fi","metric":"brier","value":0.1092,"lo":null,"hi":null,"n":100,"of":100},{"category":"fi","metric":"ece","value":0.0513,"lo":null,"hi":null,"n":100,"of":100},{"category":"fi","metric":"aurc","value":0.0192,"lo":null,"hi":null,"n":100,"of":100},{"category":"fi","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"fr","metric":"accuracy","value":0.91,"lo":null,"hi":null,"n":100,"of":100},{"category":"fr","metric":"brier","value":0.1374,"lo":null,"hi":null,"n":100,"of":100},{"category":"fr","metric":"ece","value":0.0738,"lo":null,"hi":null,"n":100,"of":100},{"category":"fr","metric":"aurc","value":0.011,"lo":null,"hi":null,"n":100,"of":100},{"category":"fr","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"he","metric":"accuracy","value":0.91,"lo":null,"hi":null,"n":100,"of":100},{"category":"he","metric":"brier","value":0.1455,"lo":null,"hi":null,"n":100,"of":100},{"category":"he","metric":"ece","value":0.063,"lo":null,"hi":null,"n":100,"of":100},{"category":"he","metric":"aurc","value":0.0124,"lo":null,"hi":null,"n":100,"of":100},{"category":"he","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"hi","metric":"accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":100},{"category":"hi","metric":"brier","value":0.1248,"lo":null,"hi":null,"n":100,"of":100},{"category":"hi","metric":"ece","value":0.0588,"lo":null,"hi":null,"n":100,"of":100},{"category":"hi","metric":"aurc","value":0.0066,"lo":null,"hi":null,"n":100,"of":100},{"category":"hi","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"hu","metric":"accuracy","value":0.9,"lo":null,"hi":null,"n":100,"of":100},{"category":"hu","metric":"brier","value":0.1636,"lo":null,"hi":null,"n":100,"of":100},{"category":"hu","metric":"ece","value":0.0788,"lo":null,"hi":null,"n":100,"of":100},{"category":"hu","metric":"aurc","value":0.015,"lo":null,"hi":null,"n":100,"of":100},{"category":"hu","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"hy","metric":"accuracy","value":0.9,"lo":null,"hi":null,"n":100,"of":100},{"category":"hy","metric":"brier","value":0.1602,"lo":null,"hi":null,"n":100,"of":100},{"category":"hy","metric":"ece","value":0.0619,"lo":null,"hi":null,"n":100,"of":100},{"category":"hy","metric":"aurc","value":0.0116,"lo":null,"hi":null,"n":100,"of":100},{"category":"hy","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"id","metric":"accuracy","value":0.96,"lo":null,"hi":null,"n":100,"of":100},{"category":"id","metric":"brier","value":0.0753,"lo":null,"hi":null,"n":100,"of":100},{"category":"id","metric":"ece","value":0.0517,"lo":null,"hi":null,"n":100,"of":100},{"category":"id","metric":"aurc","value":0.0055,"lo":null,"hi":null,"n":100,"of":100},{"category":"id","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"is","metric":"accuracy","value":0.85,"lo":null,"hi":null,"n":100,"of":100},{"category":"is","metric":"brier","value":0.1946,"lo":null,"hi":null,"n":100,"of":100},{"category":"is","metric":"ece","value":0.0808,"lo":null,"hi":null,"n":100,"of":100},{"category":"is","metric":"aurc","value":0.017,"lo":null,"hi":null,"n":100,"of":100},{"category":"is","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"it","metric":"accuracy","value":0.91,"lo":null,"hi":null,"n":100,"of":100},{"category":"it","metric":"brier","value":0.1255,"lo":null,"hi":null,"n":100,"of":100},{"category":"it","metric":"ece","value":0.0541,"lo":null,"hi":null,"n":100,"of":100},{"category":"it","metric":"aurc","value":0.0107,"lo":null,"hi":null,"n":100,"of":100},{"category":"it","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"ja","metric":"accuracy","value":0.97,"lo":null,"hi":null,"n":100,"of":100},{"category":"ja","metric":"brier","value":0.069,"lo":null,"hi":null,"n":100,"of":100},{"category":"ja","metric":"ece","value":0.0368,"lo":null,"hi":null,"n":100,"of":100},{"category":"ja","metric":"aurc","value":0.0022,"lo":null,"hi":null,"n":100,"of":100},{"category":"ja","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"jv","metric":"accuracy","value":0.89,"lo":null,"hi":null,"n":100,"of":100},{"category":"jv","metric":"brier","value":0.1866,"lo":null,"hi":null,"n":100,"of":100},{"category":"jv","metric":"ece","value":0.0811,"lo":null,"hi":null,"n":100,"of":100},{"category":"jv","metric":"aurc","value":0.0149,"lo":null,"hi":null,"n":100,"of":100},{"category":"jv","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"ka","metric":"accuracy","value":0.83,"lo":null,"hi":null,"n":100,"of":100},{"category":"ka","metric":"brier","value":0.2399,"lo":null,"hi":null,"n":100,"of":100},{"category":"ka","metric":"ece","value":0.0852,"lo":null,"hi":null,"n":100,"of":100},{"category":"ka","metric":"aurc","value":0.0362,"lo":null,"hi":null,"n":100,"of":100},{"category":"ka","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"km","metric":"accuracy","value":0.84,"lo":null,"hi":null,"n":100,"of":100},{"category":"km","metric":"brier","value":0.2351,"lo":null,"hi":null,"n":100,"of":100},{"category":"km","metric":"ece","value":0.0837,"lo":null,"hi":null,"n":100,"of":100},{"category":"km","metric":"aurc","value":0.0316,"lo":null,"hi":null,"n":100,"of":100},{"category":"km","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"kn","metric":"accuracy","value":0.91,"lo":null,"hi":null,"n":100,"of":100},{"category":"kn","metric":"brier","value":0.1591,"lo":null,"hi":null,"n":100,"of":100},{"category":"kn","metric":"ece","value":0.0757,"lo":null,"hi":null,"n":100,"of":100},{"category":"kn","metric":"aurc","value":0.0133,"lo":null,"hi":null,"n":100,"of":100},{"category":"kn","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"ko","metric":"accuracy","value":0.91,"lo":null,"hi":null,"n":100,"of":100},{"category":"ko","metric":"brier","value":0.1229,"lo":null,"hi":null,"n":100,"of":100},{"category":"ko","metric":"ece","value":0.0512,"lo":null,"hi":null,"n":100,"of":100},{"category":"ko","metric":"aurc","value":0.0075,"lo":null,"hi":null,"n":100,"of":100},{"category":"ko","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"lv","metric":"accuracy","value":0.89,"lo":null,"hi":null,"n":100,"of":100},{"category":"lv","metric":"brier","value":0.1669,"lo":null,"hi":null,"n":100,"of":100},{"category":"lv","metric":"ece","value":0.0783,"lo":null,"hi":null,"n":100,"of":100},{"category":"lv","metric":"aurc","value":0.0119,"lo":null,"hi":null,"n":100,"of":100},{"category":"lv","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"ml","metric":"accuracy","value":0.87,"lo":null,"hi":null,"n":100,"of":100},{"category":"ml","metric":"brier","value":0.2184,"lo":null,"hi":null,"n":100,"of":100},{"category":"ml","metric":"ece","value":0.1029,"lo":null,"hi":null,"n":100,"of":100},{"category":"ml","metric":"aurc","value":0.0254,"lo":null,"hi":null,"n":100,"of":100},{"category":"ml","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"mn","metric":"accuracy","value":0.88,"lo":null,"hi":null,"n":100,"of":100},{"category":"mn","metric":"brier","value":0.1975,"lo":null,"hi":null,"n":100,"of":100},{"category":"mn","metric":"ece","value":0.0968,"lo":null,"hi":null,"n":100,"of":100},{"category":"mn","metric":"aurc","value":0.0193,"lo":null,"hi":null,"n":100,"of":100},{"category":"mn","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"ms","metric":"accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":100},{"category":"ms","metric":"brier","value":0.1111,"lo":null,"hi":null,"n":100,"of":100},{"category":"ms","metric":"ece","value":0.0452,"lo":null,"hi":null,"n":100,"of":100},{"category":"ms","metric":"aurc","value":0.0117,"lo":null,"hi":null,"n":100,"of":100},{"category":"ms","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"my","metric":"accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":100},{"category":"my","metric":"brier","value":0.1123,"lo":null,"hi":null,"n":100,"of":100},{"category":"my","metric":"ece","value":0.0696,"lo":null,"hi":null,"n":100,"of":100},{"category":"my","metric":"aurc","value":0.0099,"lo":null,"hi":null,"n":100,"of":100},{"category":"my","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"nb","metric":"accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":100},{"category":"nb","metric":"brier","value":0.0879,"lo":null,"hi":null,"n":100,"of":100},{"category":"nb","metric":"ece","value":0.0369,"lo":null,"hi":null,"n":100,"of":100},{"category":"nb","metric":"aurc","value":0.0044,"lo":null,"hi":null,"n":100,"of":100},{"category":"nb","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"nl","metric":"accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":100},{"category":"nl","metric":"brier","value":0.1198,"lo":null,"hi":null,"n":100,"of":100},{"category":"nl","metric":"ece","value":0.0492,"lo":null,"hi":null,"n":100,"of":100},{"category":"nl","metric":"aurc","value":0.006,"lo":null,"hi":null,"n":100,"of":100},{"category":"nl","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"pl","metric":"accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":100},{"category":"pl","metric":"brier","value":0.1045,"lo":null,"hi":null,"n":100,"of":100},{"category":"pl","metric":"ece","value":0.0411,"lo":null,"hi":null,"n":100,"of":100},{"category":"pl","metric":"aurc","value":0.0054,"lo":null,"hi":null,"n":100,"of":100},{"category":"pl","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"pt","metric":"accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":100},{"category":"pt","metric":"brier","value":0.1049,"lo":null,"hi":null,"n":100,"of":100},{"category":"pt","metric":"ece","value":0.0365,"lo":null,"hi":null,"n":100,"of":100},{"category":"pt","metric":"aurc","value":0.0069,"lo":null,"hi":null,"n":100,"of":100},{"category":"pt","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"ro","metric":"accuracy","value":0.91,"lo":null,"hi":null,"n":100,"of":100},{"category":"ro","metric":"brier","value":0.0938,"lo":null,"hi":null,"n":100,"of":100},{"category":"ro","metric":"ece","value":0.0505,"lo":null,"hi":null,"n":100,"of":100},{"category":"ro","metric":"aurc","value":0.005,"lo":null,"hi":null,"n":100,"of":100},{"category":"ro","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"ru","metric":"accuracy","value":0.97,"lo":null,"hi":null,"n":100,"of":100},{"category":"ru","metric":"brier","value":0.0417,"lo":null,"hi":null,"n":100,"of":100},{"category":"ru","metric":"ece","value":0.0302,"lo":null,"hi":null,"n":100,"of":100},{"category":"ru","metric":"aurc","value":0.0008,"lo":null,"hi":null,"n":100,"of":100},{"category":"ru","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"sl","metric":"accuracy","value":0.91,"lo":null,"hi":null,"n":100,"of":100},{"category":"sl","metric":"brier","value":0.1254,"lo":null,"hi":null,"n":100,"of":100},{"category":"sl","metric":"ece","value":0.0483,"lo":null,"hi":null,"n":100,"of":100},{"category":"sl","metric":"aurc","value":0.0075,"lo":null,"hi":null,"n":100,"of":100},{"category":"sl","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"sq","metric":"accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":100},{"category":"sq","metric":"brier","value":0.1093,"lo":null,"hi":null,"n":100,"of":100},{"category":"sq","metric":"ece","value":0.0584,"lo":null,"hi":null,"n":100,"of":100},{"category":"sq","metric":"aurc","value":0.0047,"lo":null,"hi":null,"n":100,"of":100},{"category":"sq","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"sv","metric":"accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":100},{"category":"sv","metric":"brier","value":0.0977,"lo":null,"hi":null,"n":100,"of":100},{"category":"sv","metric":"ece","value":0.0366,"lo":null,"hi":null,"n":100,"of":100},{"category":"sv","metric":"aurc","value":0.0042,"lo":null,"hi":null,"n":100,"of":100},{"category":"sv","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"sw","metric":"accuracy","value":0.84,"lo":null,"hi":null,"n":100,"of":100},{"category":"sw","metric":"brier","value":0.221,"lo":null,"hi":null,"n":100,"of":100},{"category":"sw","metric":"ece","value":0.0871,"lo":null,"hi":null,"n":100,"of":100},{"category":"sw","metric":"aurc","value":0.0191,"lo":null,"hi":null,"n":100,"of":100},{"category":"sw","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"ta","metric":"accuracy","value":0.89,"lo":null,"hi":null,"n":100,"of":100},{"category":"ta","metric":"brier","value":0.1493,"lo":null,"hi":null,"n":100,"of":100},{"category":"ta","metric":"ece","value":0.0731,"lo":null,"hi":null,"n":100,"of":100},{"category":"ta","metric":"aurc","value":0.0095,"lo":null,"hi":null,"n":100,"of":100},{"category":"ta","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"te","metric":"accuracy","value":0.87,"lo":null,"hi":null,"n":100,"of":100},{"category":"te","metric":"brier","value":0.194,"lo":null,"hi":null,"n":100,"of":100},{"category":"te","metric":"ece","value":0.0825,"lo":null,"hi":null,"n":100,"of":100},{"category":"te","metric":"aurc","value":0.0222,"lo":null,"hi":null,"n":100,"of":100},{"category":"te","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"th","metric":"accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":100},{"category":"th","metric":"brier","value":0.0999,"lo":null,"hi":null,"n":100,"of":100},{"category":"th","metric":"ece","value":0.039,"lo":null,"hi":null,"n":100,"of":100},{"category":"th","metric":"aurc","value":0.0056,"lo":null,"hi":null,"n":100,"of":100},{"category":"th","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"tl","metric":"accuracy","value":0.91,"lo":null,"hi":null,"n":100,"of":100},{"category":"tl","metric":"brier","value":0.1482,"lo":null,"hi":null,"n":100,"of":100},{"category":"tl","metric":"ece","value":0.0666,"lo":null,"hi":null,"n":100,"of":100},{"category":"tl","metric":"aurc","value":0.0197,"lo":null,"hi":null,"n":100,"of":100},{"category":"tl","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"tr","metric":"accuracy","value":0.9,"lo":null,"hi":null,"n":100,"of":100},{"category":"tr","metric":"brier","value":0.1743,"lo":null,"hi":null,"n":100,"of":100},{"category":"tr","metric":"ece","value":0.0748,"lo":null,"hi":null,"n":100,"of":100},{"category":"tr","metric":"aurc","value":0.0191,"lo":null,"hi":null,"n":100,"of":100},{"category":"tr","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"ur","metric":"accuracy","value":0.9,"lo":null,"hi":null,"n":100,"of":100},{"category":"ur","metric":"brier","value":0.1465,"lo":null,"hi":null,"n":100,"of":100},{"category":"ur","metric":"ece","value":0.0592,"lo":null,"hi":null,"n":100,"of":100},{"category":"ur","metric":"aurc","value":0.0095,"lo":null,"hi":null,"n":100,"of":100},{"category":"ur","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"vi","metric":"accuracy","value":0.91,"lo":null,"hi":null,"n":100,"of":100},{"category":"vi","metric":"brier","value":0.1386,"lo":null,"hi":null,"n":100,"of":100},{"category":"vi","metric":"ece","value":0.0697,"lo":null,"hi":null,"n":100,"of":100},{"category":"vi","metric":"aurc","value":0.0088,"lo":null,"hi":null,"n":100,"of":100},{"category":"vi","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"zh-CN","metric":"accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":100},{"category":"zh-CN","metric":"brier","value":0.0895,"lo":null,"hi":null,"n":100,"of":100},{"category":"zh-CN","metric":"ece","value":0.0587,"lo":null,"hi":null,"n":100,"of":100},{"category":"zh-CN","metric":"aurc","value":0.0032,"lo":null,"hi":null,"n":100,"of":100},{"category":"zh-CN","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"zh-TW","metric":"accuracy","value":0.95,"lo":null,"hi":null,"n":100,"of":100},{"category":"zh-TW","metric":"brier","value":0.0689,"lo":null,"hi":null,"n":100,"of":100},{"category":"zh-TW","metric":"ece","value":0.0325,"lo":null,"hi":null,"n":100,"of":100},{"category":"zh-TW","metric":"aurc","value":0.0029,"lo":null,"hi":null,"n":100,"of":100},{"category":"zh-TW","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100}],"notes":"device cuda; dtype bf16; preds sha256:368a5f76afa5171bd80e36a827debfd93803318a4e78d92ba6c4af493b8bf23d; scores sha256:1ec811cfb085ac209c7b094b408b00ef95d3d3c73b7ae7bdf6c52fc314d112e0","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"massive-harness-multilingual-laya","benchmark":"massive","split":"harness","system":"laya","version":"convaiinnovations/laya-multilingual@e4e9ddf21a7b1903b7acffd8814ad4307bf63a67","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"735e240323cd7c9f0c880b7d8c62d89f40be2164","questions":5100,"answered":5100,"when":"","ended":"2026-09-25T17:33:56Z","by":"","completion":"complete","measures":[{"category":"all","metric":"macro_accuracy","value":0.4008,"lo":null,"hi":null,"n":5100,"of":5100},{"category":"all","metric":"macro_brier","value":0.9344,"lo":null,"hi":null,"n":5100,"of":5100},{"category":"all","metric":"macro_ece","value":0.3911,"lo":null,"hi":null,"n":5100,"of":5100},{"category":"all","metric":"macro_aurc","value":0.449,"lo":null,"hi":null,"n":5100,"of":5100},{"category":"all","metric":"macro_zero_prob","value":0.0073,"lo":null,"hi":null,"n":5100,"of":5100},{"category":"af","metric":"accuracy","value":0.35,"lo":null,"hi":null,"n":100,"of":100},{"category":"af","metric":"brier","value":0.9812,"lo":null,"hi":null,"n":100,"of":100},{"category":"af","metric":"ece","value":0.4394,"lo":null,"hi":null,"n":100,"of":100},{"category":"af","metric":"aurc","value":0.4303,"lo":null,"hi":null,"n":100,"of":100},{"category":"af","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"am","metric":"accuracy","value":0.15,"lo":null,"hi":null,"n":100,"of":100},{"category":"am","metric":"brier","value":1.2188,"lo":null,"hi":null,"n":100,"of":100},{"category":"am","metric":"ece","value":0.4839,"lo":null,"hi":null,"n":100,"of":100},{"category":"am","metric":"aurc","value":0.8162,"lo":null,"hi":null,"n":100,"of":100},{"category":"am","metric":"zero_prob","value":0.01,"lo":null,"hi":null,"n":100,"of":100},{"category":"ar","metric":"accuracy","value":0.46,"lo":null,"hi":null,"n":100,"of":100},{"category":"ar","metric":"brier","value":0.8464,"lo":null,"hi":null,"n":100,"of":100},{"category":"ar","metric":"ece","value":0.315,"lo":null,"hi":null,"n":100,"of":100},{"category":"ar","metric":"aurc","value":0.3901,"lo":null,"hi":null,"n":100,"of":100},{"category":"ar","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"az","metric":"accuracy","value":0.36,"lo":null,"hi":null,"n":100,"of":100},{"category":"az","metric":"brier","value":0.9578,"lo":null,"hi":null,"n":100,"of":100},{"category":"az","metric":"ece","value":0.381,"lo":null,"hi":null,"n":100,"of":100},{"category":"az","metric":"aurc","value":0.4695,"lo":null,"hi":null,"n":100,"of":100},{"category":"az","metric":"zero_prob","value":0.01,"lo":null,"hi":null,"n":100,"of":100},{"category":"bn","metric":"accuracy","value":0.45,"lo":null,"hi":null,"n":100,"of":100},{"category":"bn","metric":"brier","value":0.8976,"lo":null,"hi":null,"n":100,"of":100},{"category":"bn","metric":"ece","value":0.354,"lo":null,"hi":null,"n":100,"of":100},{"category":"bn","metric":"aurc","value":0.4341,"lo":null,"hi":null,"n":100,"of":100},{"category":"bn","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"cy","metric":"accuracy","value":0.13,"lo":null,"hi":null,"n":100,"of":100},{"category":"cy","metric":"brier","value":1.3123,"lo":null,"hi":null,"n":100,"of":100},{"category":"cy","metric":"ece","value":0.5873,"lo":null,"hi":null,"n":100,"of":100},{"category":"cy","metric":"aurc","value":0.7637,"lo":null,"hi":null,"n":100,"of":100},{"category":"cy","metric":"zero_prob","value":0.03,"lo":null,"hi":null,"n":100,"of":100},{"category":"da","metric":"accuracy","value":0.52,"lo":null,"hi":null,"n":100,"of":100},{"category":"da","metric":"brier","value":0.7303,"lo":null,"hi":null,"n":100,"of":100},{"category":"da","metric":"ece","value":0.315,"lo":null,"hi":null,"n":100,"of":100},{"category":"da","metric":"aurc","value":0.269,"lo":null,"hi":null,"n":100,"of":100},{"category":"da","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"de","metric":"accuracy","value":0.5,"lo":null,"hi":null,"n":100,"of":100},{"category":"de","metric":"brier","value":0.7588,"lo":null,"hi":null,"n":100,"of":100},{"category":"de","metric":"ece","value":0.3358,"lo":null,"hi":null,"n":100,"of":100},{"category":"de","metric":"aurc","value":0.2531,"lo":null,"hi":null,"n":100,"of":100},{"category":"de","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"el","metric":"accuracy","value":0.44,"lo":null,"hi":null,"n":100,"of":100},{"category":"el","metric":"brier","value":0.9225,"lo":null,"hi":null,"n":100,"of":100},{"category":"el","metric":"ece","value":0.4297,"lo":null,"hi":null,"n":100,"of":100},{"category":"el","metric":"aurc","value":0.4247,"lo":null,"hi":null,"n":100,"of":100},{"category":"el","metric":"zero_prob","value":0.01,"lo":null,"hi":null,"n":100,"of":100},{"category":"en","metric":"accuracy","value":0.71,"lo":null,"hi":null,"n":100,"of":100},{"category":"en","metric":"brier","value":0.5062,"lo":null,"hi":null,"n":100,"of":100},{"category":"en","metric":"ece","value":0.2333,"lo":null,"hi":null,"n":100,"of":100},{"category":"en","metric":"aurc","value":0.1252,"lo":null,"hi":null,"n":100,"of":100},{"category":"en","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"es","metric":"accuracy","value":0.58,"lo":null,"hi":null,"n":100,"of":100},{"category":"es","metric":"brier","value":0.6722,"lo":null,"hi":null,"n":100,"of":100},{"category":"es","metric":"ece","value":0.2577,"lo":null,"hi":null,"n":100,"of":100},{"category":"es","metric":"aurc","value":0.2623,"lo":null,"hi":null,"n":100,"of":100},{"category":"es","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"fa","metric":"accuracy","value":0.51,"lo":null,"hi":null,"n":100,"of":100},{"category":"fa","metric":"brier","value":0.7935,"lo":null,"hi":null,"n":100,"of":100},{"category":"fa","metric":"ece","value":0.2985,"lo":null,"hi":null,"n":100,"of":100},{"category":"fa","metric":"aurc","value":0.3877,"lo":null,"hi":null,"n":100,"of":100},{"category":"fa","metric":"zero_prob","value":0.01,"lo":null,"hi":null,"n":100,"of":100},{"category":"fi","metric":"accuracy","value":0.34,"lo":null,"hi":null,"n":100,"of":100},{"category":"fi","metric":"brier","value":1.0128,"lo":null,"hi":null,"n":100,"of":100},{"category":"fi","metric":"ece","value":0.3918,"lo":null,"hi":null,"n":100,"of":100},{"category":"fi","metric":"aurc","value":0.5655,"lo":null,"hi":null,"n":100,"of":100},{"category":"fi","metric":"zero_prob","value":0.01,"lo":null,"hi":null,"n":100,"of":100},{"category":"fr","metric":"accuracy","value":0.6,"lo":null,"hi":null,"n":100,"of":100},{"category":"fr","metric":"brier","value":0.6184,"lo":null,"hi":null,"n":100,"of":100},{"category":"fr","metric":"ece","value":0.2484,"lo":null,"hi":null,"n":100,"of":100},{"category":"fr","metric":"aurc","value":0.2022,"lo":null,"hi":null,"n":100,"of":100},{"category":"fr","metric":"zero_prob","value":0.01,"lo":null,"hi":null,"n":100,"of":100},{"category":"he","metric":"accuracy","value":0.37,"lo":null,"hi":null,"n":100,"of":100},{"category":"he","metric":"brier","value":0.9688,"lo":null,"hi":null,"n":100,"of":100},{"category":"he","metric":"ece","value":0.4186,"lo":null,"hi":null,"n":100,"of":100},{"category":"he","metric":"aurc","value":0.4009,"lo":null,"hi":null,"n":100,"of":100},{"category":"he","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"hi","metric":"accuracy","value":0.46,"lo":null,"hi":null,"n":100,"of":100},{"category":"hi","metric":"brier","value":0.8783,"lo":null,"hi":null,"n":100,"of":100},{"category":"hi","metric":"ece","value":0.3685,"lo":null,"hi":null,"n":100,"of":100},{"category":"hi","metric":"aurc","value":0.3879,"lo":null,"hi":null,"n":100,"of":100},{"category":"hi","metric":"zero_prob","value":0.01,"lo":null,"hi":null,"n":100,"of":100},{"category":"hu","metric":"accuracy","value":0.36,"lo":null,"hi":null,"n":100,"of":100},{"category":"hu","metric":"brier","value":1.0048,"lo":null,"hi":null,"n":100,"of":100},{"category":"hu","metric":"ece","value":0.4005,"lo":null,"hi":null,"n":100,"of":100},{"category":"hu","metric":"aurc","value":0.4871,"lo":null,"hi":null,"n":100,"of":100},{"category":"hu","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"hy","metric":"accuracy","value":0.25,"lo":null,"hi":null,"n":100,"of":100},{"category":"hy","metric":"brier","value":1.1723,"lo":null,"hi":null,"n":100,"of":100},{"category":"hy","metric":"ece","value":0.5127,"lo":null,"hi":null,"n":100,"of":100},{"category":"hy","metric":"aurc","value":0.7133,"lo":null,"hi":null,"n":100,"of":100},{"category":"hy","metric":"zero_prob","value":0.01,"lo":null,"hi":null,"n":100,"of":100},{"category":"id","metric":"accuracy","value":0.51,"lo":null,"hi":null,"n":100,"of":100},{"category":"id","metric":"brier","value":0.7524,"lo":null,"hi":null,"n":100,"of":100},{"category":"id","metric":"ece","value":0.3225,"lo":null,"hi":null,"n":100,"of":100},{"category":"id","metric":"aurc","value":0.239,"lo":null,"hi":null,"n":100,"of":100},{"category":"id","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"is","metric":"accuracy","value":0.35,"lo":null,"hi":null,"n":100,"of":100},{"category":"is","metric":"brier","value":1.0317,"lo":null,"hi":null,"n":100,"of":100},{"category":"is","metric":"ece","value":0.4518,"lo":null,"hi":null,"n":100,"of":100},{"category":"is","metric":"aurc","value":0.4926,"lo":null,"hi":null,"n":100,"of":100},{"category":"is","metric":"zero_prob","value":0.04,"lo":null,"hi":null,"n":100,"of":100},{"category":"it","metric":"accuracy","value":0.52,"lo":null,"hi":null,"n":100,"of":100},{"category":"it","metric":"brier","value":0.7809,"lo":null,"hi":null,"n":100,"of":100},{"category":"it","metric":"ece","value":0.3428,"lo":null,"hi":null,"n":100,"of":100},{"category":"it","metric":"aurc","value":0.3414,"lo":null,"hi":null,"n":100,"of":100},{"category":"it","metric":"zero_prob","value":0.02,"lo":null,"hi":null,"n":100,"of":100},{"category":"ja","metric":"accuracy","value":0.64,"lo":null,"hi":null,"n":100,"of":100},{"category":"ja","metric":"brier","value":0.5645,"lo":null,"hi":null,"n":100,"of":100},{"category":"ja","metric":"ece","value":0.2337,"lo":null,"hi":null,"n":100,"of":100},{"category":"ja","metric":"aurc","value":0.1879,"lo":null,"hi":null,"n":100,"of":100},{"category":"ja","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"jv","metric":"accuracy","value":0.3,"lo":null,"hi":null,"n":100,"of":100},{"category":"jv","metric":"brier","value":1.1318,"lo":null,"hi":null,"n":100,"of":100},{"category":"jv","metric":"ece","value":0.4842,"lo":null,"hi":null,"n":100,"of":100},{"category":"jv","metric":"aurc","value":0.6118,"lo":null,"hi":null,"n":100,"of":100},{"category":"jv","metric":"zero_prob","value":0.03,"lo":null,"hi":null,"n":100,"of":100},{"category":"ka","metric":"accuracy","value":0.15,"lo":null,"hi":null,"n":100,"of":100},{"category":"ka","metric":"brier","value":1.2966,"lo":null,"hi":null,"n":100,"of":100},{"category":"ka","metric":"ece","value":0.5422,"lo":null,"hi":null,"n":100,"of":100},{"category":"ka","metric":"aurc","value":0.8097,"lo":null,"hi":null,"n":100,"of":100},{"category":"ka","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"km","metric":"accuracy","value":0.2,"lo":null,"hi":null,"n":100,"of":100},{"category":"km","metric":"brier","value":1.1586,"lo":null,"hi":null,"n":100,"of":100},{"category":"km","metric":"ece","value":0.4454,"lo":null,"hi":null,"n":100,"of":100},{"category":"km","metric":"aurc","value":0.6855,"lo":null,"hi":null,"n":100,"of":100},{"category":"km","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"kn","metric":"accuracy","value":0.3,"lo":null,"hi":null,"n":100,"of":100},{"category":"kn","metric":"brier","value":1.028,"lo":null,"hi":null,"n":100,"of":100},{"category":"kn","metric":"ece","value":0.388,"lo":null,"hi":null,"n":100,"of":100},{"category":"kn","metric":"aurc","value":0.6162,"lo":null,"hi":null,"n":100,"of":100},{"category":"kn","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"ko","metric":"accuracy","value":0.47,"lo":null,"hi":null,"n":100,"of":100},{"category":"ko","metric":"brier","value":0.7771,"lo":null,"hi":null,"n":100,"of":100},{"category":"ko","metric":"ece","value":0.3108,"lo":null,"hi":null,"n":100,"of":100},{"category":"ko","metric":"aurc","value":0.2997,"lo":null,"hi":null,"n":100,"of":100},{"category":"ko","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"lv","metric":"accuracy","value":0.31,"lo":null,"hi":null,"n":100,"of":100},{"category":"lv","metric":"brier","value":1.0767,"lo":null,"hi":null,"n":100,"of":100},{"category":"lv","metric":"ece","value":0.472,"lo":null,"hi":null,"n":100,"of":100},{"category":"lv","metric":"aurc","value":0.5248,"lo":null,"hi":null,"n":100,"of":100},{"category":"lv","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"ml","metric":"accuracy","value":0.28,"lo":null,"hi":null,"n":100,"of":100},{"category":"ml","metric":"brier","value":1.0956,"lo":null,"hi":null,"n":100,"of":100},{"category":"ml","metric":"ece","value":0.4433,"lo":null,"hi":null,"n":100,"of":100},{"category":"ml","metric":"aurc","value":0.6375,"lo":null,"hi":null,"n":100,"of":100},{"category":"ml","metric":"zero_prob","value":0.01,"lo":null,"hi":null,"n":100,"of":100},{"category":"mn","metric":"accuracy","value":0.16,"lo":null,"hi":null,"n":100,"of":100},{"category":"mn","metric":"brier","value":1.3206,"lo":null,"hi":null,"n":100,"of":100},{"category":"mn","metric":"ece","value":0.577,"lo":null,"hi":null,"n":100,"of":100},{"category":"mn","metric":"aurc","value":0.7486,"lo":null,"hi":null,"n":100,"of":100},{"category":"mn","metric":"zero_prob","value":0.03,"lo":null,"hi":null,"n":100,"of":100},{"category":"ms","metric":"accuracy","value":0.41,"lo":null,"hi":null,"n":100,"of":100},{"category":"ms","metric":"brier","value":0.9581,"lo":null,"hi":null,"n":100,"of":100},{"category":"ms","metric":"ece","value":0.4364,"lo":null,"hi":null,"n":100,"of":100},{"category":"ms","metric":"aurc","value":0.4225,"lo":null,"hi":null,"n":100,"of":100},{"category":"ms","metric":"zero_prob","value":0.01,"lo":null,"hi":null,"n":100,"of":100},{"category":"my","metric":"accuracy","value":0.16,"lo":null,"hi":null,"n":100,"of":100},{"category":"my","metric":"brier","value":1.2443,"lo":null,"hi":null,"n":100,"of":100},{"category":"my","metric":"ece","value":0.4766,"lo":null,"hi":null,"n":100,"of":100},{"category":"my","metric":"aurc","value":0.8866,"lo":null,"hi":null,"n":100,"of":100},{"category":"my","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"nb","metric":"accuracy","value":0.56,"lo":null,"hi":null,"n":100,"of":100},{"category":"nb","metric":"brier","value":0.7025,"lo":null,"hi":null,"n":100,"of":100},{"category":"nb","metric":"ece","value":0.2937,"lo":null,"hi":null,"n":100,"of":100},{"category":"nb","metric":"aurc","value":0.2423,"lo":null,"hi":null,"n":100,"of":100},{"category":"nb","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"nl","metric":"accuracy","value":0.47,"lo":null,"hi":null,"n":100,"of":100},{"category":"nl","metric":"brier","value":0.8739,"lo":null,"hi":null,"n":100,"of":100},{"category":"nl","metric":"ece","value":0.397,"lo":null,"hi":null,"n":100,"of":100},{"category":"nl","metric":"aurc","value":0.3565,"lo":null,"hi":null,"n":100,"of":100},{"category":"nl","metric":"zero_prob","value":0.03,"lo":null,"hi":null,"n":100,"of":100},{"category":"pl","metric":"accuracy","value":0.5,"lo":null,"hi":null,"n":100,"of":100},{"category":"pl","metric":"brier","value":0.7994,"lo":null,"hi":null,"n":100,"of":100},{"category":"pl","metric":"ece","value":0.3529,"lo":null,"hi":null,"n":100,"of":100},{"category":"pl","metric":"aurc","value":0.2589,"lo":null,"hi":null,"n":100,"of":100},{"category":"pl","metric":"zero_prob","value":0.01,"lo":null,"hi":null,"n":100,"of":100},{"category":"pt","metric":"accuracy","value":0.5,"lo":null,"hi":null,"n":100,"of":100},{"category":"pt","metric":"brier","value":0.7884,"lo":null,"hi":null,"n":100,"of":100},{"category":"pt","metric":"ece","value":0.3405,"lo":null,"hi":null,"n":100,"of":100},{"category":"pt","metric":"aurc","value":0.2881,"lo":null,"hi":null,"n":100,"of":100},{"category":"pt","metric":"zero_prob","value":0.01,"lo":null,"hi":null,"n":100,"of":100},{"category":"ro","metric":"accuracy","value":0.37,"lo":null,"hi":null,"n":100,"of":100},{"category":"ro","metric":"brier","value":0.96,"lo":null,"hi":null,"n":100,"of":100},{"category":"ro","metric":"ece","value":0.3974,"lo":null,"hi":null,"n":100,"of":100},{"category":"ro","metric":"aurc","value":0.3993,"lo":null,"hi":null,"n":100,"of":100},{"category":"ro","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"ru","metric":"accuracy","value":0.57,"lo":null,"hi":null,"n":100,"of":100},{"category":"ru","metric":"brier","value":0.7319,"lo":null,"hi":null,"n":100,"of":100},{"category":"ru","metric":"ece","value":0.2959,"lo":null,"hi":null,"n":100,"of":100},{"category":"ru","metric":"aurc","value":0.3078,"lo":null,"hi":null,"n":100,"of":100},{"category":"ru","metric":"zero_prob","value":0.01,"lo":null,"hi":null,"n":100,"of":100},{"category":"sl","metric":"accuracy","value":0.37,"lo":null,"hi":null,"n":100,"of":100},{"category":"sl","metric":"brier","value":1.0103,"lo":null,"hi":null,"n":100,"of":100},{"category":"sl","metric":"ece","value":0.4338,"lo":null,"hi":null,"n":100,"of":100},{"category":"sl","metric":"aurc","value":0.4403,"lo":null,"hi":null,"n":100,"of":100},{"category":"sl","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"sq","metric":"accuracy","value":0.3,"lo":null,"hi":null,"n":100,"of":100},{"category":"sq","metric":"brier","value":1.1324,"lo":null,"hi":null,"n":100,"of":100},{"category":"sq","metric":"ece","value":0.4869,"lo":null,"hi":null,"n":100,"of":100},{"category":"sq","metric":"aurc","value":0.5115,"lo":null,"hi":null,"n":100,"of":100},{"category":"sq","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"sv","metric":"accuracy","value":0.49,"lo":null,"hi":null,"n":100,"of":100},{"category":"sv","metric":"brier","value":0.7772,"lo":null,"hi":null,"n":100,"of":100},{"category":"sv","metric":"ece","value":0.341,"lo":null,"hi":null,"n":100,"of":100},{"category":"sv","metric":"aurc","value":0.3089,"lo":null,"hi":null,"n":100,"of":100},{"category":"sv","metric":"zero_prob","value":0.01,"lo":null,"hi":null,"n":100,"of":100},{"category":"sw","metric":"accuracy","value":0.23,"lo":null,"hi":null,"n":100,"of":100},{"category":"sw","metric":"brier","value":1.1812,"lo":null,"hi":null,"n":100,"of":100},{"category":"sw","metric":"ece","value":0.4922,"lo":null,"hi":null,"n":100,"of":100},{"category":"sw","metric":"aurc","value":0.5978,"lo":null,"hi":null,"n":100,"of":100},{"category":"sw","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"ta","metric":"accuracy","value":0.31,"lo":null,"hi":null,"n":100,"of":100},{"category":"ta","metric":"brier","value":1.1179,"lo":null,"hi":null,"n":100,"of":100},{"category":"ta","metric":"ece","value":0.4479,"lo":null,"hi":null,"n":100,"of":100},{"category":"ta","metric":"aurc","value":0.6719,"lo":null,"hi":null,"n":100,"of":100},{"category":"ta","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"te","metric":"accuracy","value":0.22,"lo":null,"hi":null,"n":100,"of":100},{"category":"te","metric":"brier","value":1.1789,"lo":null,"hi":null,"n":100,"of":100},{"category":"te","metric":"ece","value":0.4969,"lo":null,"hi":null,"n":100,"of":100},{"category":"te","metric":"aurc","value":0.635,"lo":null,"hi":null,"n":100,"of":100},{"category":"te","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"th","metric":"accuracy","value":0.48,"lo":null,"hi":null,"n":100,"of":100},{"category":"th","metric":"brier","value":0.8366,"lo":null,"hi":null,"n":100,"of":100},{"category":"th","metric":"ece","value":0.3563,"lo":null,"hi":null,"n":100,"of":100},{"category":"th","metric":"aurc","value":0.3897,"lo":null,"hi":null,"n":100,"of":100},{"category":"th","metric":"zero_prob","value":0.01,"lo":null,"hi":null,"n":100,"of":100},{"category":"tl","metric":"accuracy","value":0.35,"lo":null,"hi":null,"n":100,"of":100},{"category":"tl","metric":"brier","value":1.0109,"lo":null,"hi":null,"n":100,"of":100},{"category":"tl","metric":"ece","value":0.4297,"lo":null,"hi":null,"n":100,"of":100},{"category":"tl","metric":"aurc","value":0.3897,"lo":null,"hi":null,"n":100,"of":100},{"category":"tl","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"tr","metric":"accuracy","value":0.4,"lo":null,"hi":null,"n":100,"of":100},{"category":"tr","metric":"brier","value":0.953,"lo":null,"hi":null,"n":100,"of":100},{"category":"tr","metric":"ece","value":0.4004,"lo":null,"hi":null,"n":100,"of":100},{"category":"tr","metric":"aurc","value":0.3915,"lo":null,"hi":null,"n":100,"of":100},{"category":"tr","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"ur","metric":"accuracy","value":0.42,"lo":null,"hi":null,"n":100,"of":100},{"category":"ur","metric":"brier","value":0.8645,"lo":null,"hi":null,"n":100,"of":100},{"category":"ur","metric":"ece","value":0.3355,"lo":null,"hi":null,"n":100,"of":100},{"category":"ur","metric":"aurc","value":0.4357,"lo":null,"hi":null,"n":100,"of":100},{"category":"ur","metric":"zero_prob","value":0.01,"lo":null,"hi":null,"n":100,"of":100},{"category":"vi","metric":"accuracy","value":0.34,"lo":null,"hi":null,"n":100,"of":100},{"category":"vi","metric":"brier","value":1.0617,"lo":null,"hi":null,"n":100,"of":100},{"category":"vi","metric":"ece","value":0.4636,"lo":null,"hi":null,"n":100,"of":100},{"category":"vi","metric":"aurc","value":0.5502,"lo":null,"hi":null,"n":100,"of":100},{"category":"vi","metric":"zero_prob","value":0.02,"lo":null,"hi":null,"n":100,"of":100},{"category":"zh-CN","metric":"accuracy","value":0.65,"lo":null,"hi":null,"n":100,"of":100},{"category":"zh-CN","metric":"brier","value":0.5484,"lo":null,"hi":null,"n":100,"of":100},{"category":"zh-CN","metric":"ece","value":0.2192,"lo":null,"hi":null,"n":100,"of":100},{"category":"zh-CN","metric":"aurc","value":0.1685,"lo":null,"hi":null,"n":100,"of":100},{"category":"zh-CN","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"zh-TW","metric":"accuracy","value":0.61,"lo":null,"hi":null,"n":100,"of":100},{"category":"zh-TW","metric":"brier","value":0.6563,"lo":null,"hi":null,"n":100,"of":100},{"category":"zh-TW","metric":"ece","value":0.2662,"lo":null,"hi":null,"n":100,"of":100},{"category":"zh-TW","metric":"aurc","value":0.2685,"lo":null,"hi":null,"n":100,"of":100},{"category":"zh-TW","metric":"zero_prob","value":0.01,"lo":null,"hi":null,"n":100,"of":100}],"notes":"runtime laya 0.3.20; device cpu; dtype f32; preds sha256:15e4ba94d76db187424d43e7823ffbd029214f6552bf91aade0037448ce694c6","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"massive-harness-routed-laya","benchmark":"massive","split":"harness","system":"laya","version":"convaiinnovations/laya@55cf4c4ebb4ebe31b2550e8bdf3bd21b99753851+convaiinnovations/laya-multilingual@e4e9ddf21a7b1903b7acffd8814ad4307bf63a67","baseline":true,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"735e240323cd7c9f0c880b7d8c62d89f40be2164","questions":5100,"answered":5100,"when":"","ended":"2026-09-25T17:33:56Z","by":"","completion":"complete","measures":[{"category":"all","metric":"macro_accuracy","value":0.3822,"lo":null,"hi":null,"n":5100,"of":5100},{"category":"all","metric":"macro_brier","value":0.987,"lo":null,"hi":null,"n":5100,"of":5100},{"category":"all","metric":"macro_ece","value":0.4184,"lo":null,"hi":null,"n":5100,"of":5100},{"category":"all","metric":"macro_aurc","value":0.4804,"lo":null,"hi":null,"n":5100,"of":5100},{"category":"all","metric":"macro_zero_prob","value":0.0298,"lo":null,"hi":null,"n":5100,"of":5100},{"category":"af","metric":"accuracy","value":0.34,"lo":null,"hi":null,"n":100,"of":100},{"category":"af","metric":"brier","value":1.0683,"lo":null,"hi":null,"n":100,"of":100},{"category":"af","metric":"ece","value":0.4845,"lo":null,"hi":null,"n":100,"of":100},{"category":"af","metric":"aurc","value":0.4721,"lo":null,"hi":null,"n":100,"of":100},{"category":"af","metric":"zero_prob","value":0.09,"lo":null,"hi":null,"n":100,"of":100},{"category":"am","metric":"accuracy","value":0.15,"lo":null,"hi":null,"n":100,"of":100},{"category":"am","metric":"brier","value":1.2188,"lo":null,"hi":null,"n":100,"of":100},{"category":"am","metric":"ece","value":0.4839,"lo":null,"hi":null,"n":100,"of":100},{"category":"am","metric":"aurc","value":0.8162,"lo":null,"hi":null,"n":100,"of":100},{"category":"am","metric":"zero_prob","value":0.01,"lo":null,"hi":null,"n":100,"of":100},{"category":"ar","metric":"accuracy","value":0.46,"lo":null,"hi":null,"n":100,"of":100},{"category":"ar","metric":"brier","value":0.8464,"lo":null,"hi":null,"n":100,"of":100},{"category":"ar","metric":"ece","value":0.315,"lo":null,"hi":null,"n":100,"of":100},{"category":"ar","metric":"aurc","value":0.3768,"lo":null,"hi":null,"n":100,"of":100},{"category":"ar","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"az","metric":"accuracy","value":0.36,"lo":null,"hi":null,"n":100,"of":100},{"category":"az","metric":"brier","value":0.9729,"lo":null,"hi":null,"n":100,"of":100},{"category":"az","metric":"ece","value":0.3929,"lo":null,"hi":null,"n":100,"of":100},{"category":"az","metric":"aurc","value":0.4833,"lo":null,"hi":null,"n":100,"of":100},{"category":"az","metric":"zero_prob","value":0.02,"lo":null,"hi":null,"n":100,"of":100},{"category":"bn","metric":"accuracy","value":0.45,"lo":null,"hi":null,"n":100,"of":100},{"category":"bn","metric":"brier","value":0.8976,"lo":null,"hi":null,"n":100,"of":100},{"category":"bn","metric":"ece","value":0.354,"lo":null,"hi":null,"n":100,"of":100},{"category":"bn","metric":"aurc","value":0.4341,"lo":null,"hi":null,"n":100,"of":100},{"category":"bn","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"cy","metric":"accuracy","value":0.16,"lo":null,"hi":null,"n":100,"of":100},{"category":"cy","metric":"brier","value":1.3894,"lo":null,"hi":null,"n":100,"of":100},{"category":"cy","metric":"ece","value":0.6107,"lo":null,"hi":null,"n":100,"of":100},{"category":"cy","metric":"aurc","value":0.7641,"lo":null,"hi":null,"n":100,"of":100},{"category":"cy","metric":"zero_prob","value":0.06,"lo":null,"hi":null,"n":100,"of":100},{"category":"da","metric":"accuracy","value":0.46,"lo":null,"hi":null,"n":100,"of":100},{"category":"da","metric":"brier","value":0.934,"lo":null,"hi":null,"n":100,"of":100},{"category":"da","metric":"ece","value":0.4518,"lo":null,"hi":null,"n":100,"of":100},{"category":"da","metric":"aurc","value":0.3267,"lo":null,"hi":null,"n":100,"of":100},{"category":"da","metric":"zero_prob","value":0.05,"lo":null,"hi":null,"n":100,"of":100},{"category":"de","metric":"accuracy","value":0.53,"lo":null,"hi":null,"n":100,"of":100},{"category":"de","metric":"brier","value":0.7303,"lo":null,"hi":null,"n":100,"of":100},{"category":"de","metric":"ece","value":0.3037,"lo":null,"hi":null,"n":100,"of":100},{"category":"de","metric":"aurc","value":0.2334,"lo":null,"hi":null,"n":100,"of":100},{"category":"de","metric":"zero_prob","value":0.03,"lo":null,"hi":null,"n":100,"of":100},{"category":"el","metric":"accuracy","value":0.44,"lo":null,"hi":null,"n":100,"of":100},{"category":"el","metric":"brier","value":0.9225,"lo":null,"hi":null,"n":100,"of":100},{"category":"el","metric":"ece","value":0.4297,"lo":null,"hi":null,"n":100,"of":100},{"category":"el","metric":"aurc","value":0.4247,"lo":null,"hi":null,"n":100,"of":100},{"category":"el","metric":"zero_prob","value":0.01,"lo":null,"hi":null,"n":100,"of":100},{"category":"en","metric":"accuracy","value":0.82,"lo":null,"hi":null,"n":100,"of":100},{"category":"en","metric":"brier","value":0.2882,"lo":null,"hi":null,"n":100,"of":100},{"category":"en","metric":"ece","value":0.1382,"lo":null,"hi":null,"n":100,"of":100},{"category":"en","metric":"aurc","value":0.0386,"lo":null,"hi":null,"n":100,"of":100},{"category":"en","metric":"zero_prob","value":0.02,"lo":null,"hi":null,"n":100,"of":100},{"category":"es","metric":"accuracy","value":0.53,"lo":null,"hi":null,"n":100,"of":100},{"category":"es","metric":"brier","value":0.8103,"lo":null,"hi":null,"n":100,"of":100},{"category":"es","metric":"ece","value":0.3476,"lo":null,"hi":null,"n":100,"of":100},{"category":"es","metric":"aurc","value":0.3585,"lo":null,"hi":null,"n":100,"of":100},{"category":"es","metric":"zero_prob","value":0.09,"lo":null,"hi":null,"n":100,"of":100},{"category":"fa","metric":"accuracy","value":0.51,"lo":null,"hi":null,"n":100,"of":100},{"category":"fa","metric":"brier","value":0.7935,"lo":null,"hi":null,"n":100,"of":100},{"category":"fa","metric":"ece","value":0.2985,"lo":null,"hi":null,"n":100,"of":100},{"category":"fa","metric":"aurc","value":0.3877,"lo":null,"hi":null,"n":100,"of":100},{"category":"fa","metric":"zero_prob","value":0.01,"lo":null,"hi":null,"n":100,"of":100},{"category":"fi","metric":"accuracy","value":0.3,"lo":null,"hi":null,"n":100,"of":100},{"category":"fi","metric":"brier","value":1.1197,"lo":null,"hi":null,"n":100,"of":100},{"category":"fi","metric":"ece","value":0.4618,"lo":null,"hi":null,"n":100,"of":100},{"category":"fi","metric":"aurc","value":0.5732,"lo":null,"hi":null,"n":100,"of":100},{"category":"fi","metric":"zero_prob","value":0.02,"lo":null,"hi":null,"n":100,"of":100},{"category":"fr","metric":"accuracy","value":0.6,"lo":null,"hi":null,"n":100,"of":100},{"category":"fr","metric":"brier","value":0.6627,"lo":null,"hi":null,"n":100,"of":100},{"category":"fr","metric":"ece","value":0.2683,"lo":null,"hi":null,"n":100,"of":100},{"category":"fr","metric":"aurc","value":0.2329,"lo":null,"hi":null,"n":100,"of":100},{"category":"fr","metric":"zero_prob","value":0.01,"lo":null,"hi":null,"n":100,"of":100},{"category":"he","metric":"accuracy","value":0.37,"lo":null,"hi":null,"n":100,"of":100},{"category":"he","metric":"brier","value":0.9688,"lo":null,"hi":null,"n":100,"of":100},{"category":"he","metric":"ece","value":0.4186,"lo":null,"hi":null,"n":100,"of":100},{"category":"he","metric":"aurc","value":0.4009,"lo":null,"hi":null,"n":100,"of":100},{"category":"he","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"hi","metric":"accuracy","value":0.46,"lo":null,"hi":null,"n":100,"of":100},{"category":"hi","metric":"brier","value":0.8783,"lo":null,"hi":null,"n":100,"of":100},{"category":"hi","metric":"ece","value":0.3685,"lo":null,"hi":null,"n":100,"of":100},{"category":"hi","metric":"aurc","value":0.3879,"lo":null,"hi":null,"n":100,"of":100},{"category":"hi","metric":"zero_prob","value":0.01,"lo":null,"hi":null,"n":100,"of":100},{"category":"hu","metric":"accuracy","value":0.34,"lo":null,"hi":null,"n":100,"of":100},{"category":"hu","metric":"brier","value":1.0516,"lo":null,"hi":null,"n":100,"of":100},{"category":"hu","metric":"ece","value":0.4374,"lo":null,"hi":null,"n":100,"of":100},{"category":"hu","metric":"aurc","value":0.5046,"lo":null,"hi":null,"n":100,"of":100},{"category":"hu","metric":"zero_prob","value":0.01,"lo":null,"hi":null,"n":100,"of":100},{"category":"hy","metric":"accuracy","value":0.25,"lo":null,"hi":null,"n":100,"of":100},{"category":"hy","metric":"brier","value":1.1723,"lo":null,"hi":null,"n":100,"of":100},{"category":"hy","metric":"ece","value":0.5127,"lo":null,"hi":null,"n":100,"of":100},{"category":"hy","metric":"aurc","value":0.7133,"lo":null,"hi":null,"n":100,"of":100},{"category":"hy","metric":"zero_prob","value":0.01,"lo":null,"hi":null,"n":100,"of":100},{"category":"id","metric":"accuracy","value":0.36,"lo":null,"hi":null,"n":100,"of":100},{"category":"id","metric":"brier","value":1.1008,"lo":null,"hi":null,"n":100,"of":100},{"category":"id","metric":"ece","value":0.4856,"lo":null,"hi":null,"n":100,"of":100},{"category":"id","metric":"aurc","value":0.4555,"lo":null,"hi":null,"n":100,"of":100},{"category":"id","metric":"zero_prob","value":0.06,"lo":null,"hi":null,"n":100,"of":100},{"category":"is","metric":"accuracy","value":0.33,"lo":null,"hi":null,"n":100,"of":100},{"category":"is","metric":"brier","value":1.0847,"lo":null,"hi":null,"n":100,"of":100},{"category":"is","metric":"ece","value":0.486,"lo":null,"hi":null,"n":100,"of":100},{"category":"is","metric":"aurc","value":0.5699,"lo":null,"hi":null,"n":100,"of":100},{"category":"is","metric":"zero_prob","value":0.08,"lo":null,"hi":null,"n":100,"of":100},{"category":"it","metric":"accuracy","value":0.47,"lo":null,"hi":null,"n":100,"of":100},{"category":"it","metric":"brier","value":0.8862,"lo":null,"hi":null,"n":100,"of":100},{"category":"it","metric":"ece","value":0.38,"lo":null,"hi":null,"n":100,"of":100},{"category":"it","metric":"aurc","value":0.3937,"lo":null,"hi":null,"n":100,"of":100},{"category":"it","metric":"zero_prob","value":0.07,"lo":null,"hi":null,"n":100,"of":100},{"category":"ja","metric":"accuracy","value":0.64,"lo":null,"hi":null,"n":100,"of":100},{"category":"ja","metric":"brier","value":0.5645,"lo":null,"hi":null,"n":100,"of":100},{"category":"ja","metric":"ece","value":0.2337,"lo":null,"hi":null,"n":100,"of":100},{"category":"ja","metric":"aurc","value":0.1879,"lo":null,"hi":null,"n":100,"of":100},{"category":"ja","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"jv","metric":"accuracy","value":0.16,"lo":null,"hi":null,"n":100,"of":100},{"category":"jv","metric":"brier","value":1.4425,"lo":null,"hi":null,"n":100,"of":100},{"category":"jv","metric":"ece","value":0.6634,"lo":null,"hi":null,"n":100,"of":100},{"category":"jv","metric":"aurc","value":0.7915,"lo":null,"hi":null,"n":100,"of":100},{"category":"jv","metric":"zero_prob","value":0.11,"lo":null,"hi":null,"n":100,"of":100},{"category":"ka","metric":"accuracy","value":0.15,"lo":null,"hi":null,"n":100,"of":100},{"category":"ka","metric":"brier","value":1.2966,"lo":null,"hi":null,"n":100,"of":100},{"category":"ka","metric":"ece","value":0.5422,"lo":null,"hi":null,"n":100,"of":100},{"category":"ka","metric":"aurc","value":0.8097,"lo":null,"hi":null,"n":100,"of":100},{"category":"ka","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"km","metric":"accuracy","value":0.2,"lo":null,"hi":null,"n":100,"of":100},{"category":"km","metric":"brier","value":1.1586,"lo":null,"hi":null,"n":100,"of":100},{"category":"km","metric":"ece","value":0.4454,"lo":null,"hi":null,"n":100,"of":100},{"category":"km","metric":"aurc","value":0.6855,"lo":null,"hi":null,"n":100,"of":100},{"category":"km","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"kn","metric":"accuracy","value":0.3,"lo":null,"hi":null,"n":100,"of":100},{"category":"kn","metric":"brier","value":1.028,"lo":null,"hi":null,"n":100,"of":100},{"category":"kn","metric":"ece","value":0.388,"lo":null,"hi":null,"n":100,"of":100},{"category":"kn","metric":"aurc","value":0.6162,"lo":null,"hi":null,"n":100,"of":100},{"category":"kn","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"ko","metric":"accuracy","value":0.47,"lo":null,"hi":null,"n":100,"of":100},{"category":"ko","metric":"brier","value":0.7771,"lo":null,"hi":null,"n":100,"of":100},{"category":"ko","metric":"ece","value":0.3108,"lo":null,"hi":null,"n":100,"of":100},{"category":"ko","metric":"aurc","value":0.2997,"lo":null,"hi":null,"n":100,"of":100},{"category":"ko","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"lv","metric":"accuracy","value":0.3,"lo":null,"hi":null,"n":100,"of":100},{"category":"lv","metric":"brier","value":1.1049,"lo":null,"hi":null,"n":100,"of":100},{"category":"lv","metric":"ece","value":0.486,"lo":null,"hi":null,"n":100,"of":100},{"category":"lv","metric":"aurc","value":0.5448,"lo":null,"hi":null,"n":100,"of":100},{"category":"lv","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"ml","metric":"accuracy","value":0.28,"lo":null,"hi":null,"n":100,"of":100},{"category":"ml","metric":"brier","value":1.0956,"lo":null,"hi":null,"n":100,"of":100},{"category":"ml","metric":"ece","value":0.4433,"lo":null,"hi":null,"n":100,"of":100},{"category":"ml","metric":"aurc","value":0.6375,"lo":null,"hi":null,"n":100,"of":100},{"category":"ml","metric":"zero_prob","value":0.01,"lo":null,"hi":null,"n":100,"of":100},{"category":"mn","metric":"accuracy","value":0.16,"lo":null,"hi":null,"n":100,"of":100},{"category":"mn","metric":"brier","value":1.3206,"lo":null,"hi":null,"n":100,"of":100},{"category":"mn","metric":"ece","value":0.577,"lo":null,"hi":null,"n":100,"of":100},{"category":"mn","metric":"aurc","value":0.7486,"lo":null,"hi":null,"n":100,"of":100},{"category":"mn","metric":"zero_prob","value":0.03,"lo":null,"hi":null,"n":100,"of":100},{"category":"ms","metric":"accuracy","value":0.27,"lo":null,"hi":null,"n":100,"of":100},{"category":"ms","metric":"brier","value":1.1586,"lo":null,"hi":null,"n":100,"of":100},{"category":"ms","metric":"ece","value":0.5064,"lo":null,"hi":null,"n":100,"of":100},{"category":"ms","metric":"aurc","value":0.546,"lo":null,"hi":null,"n":100,"of":100},{"category":"ms","metric":"zero_prob","value":0.06,"lo":null,"hi":null,"n":100,"of":100},{"category":"my","metric":"accuracy","value":0.16,"lo":null,"hi":null,"n":100,"of":100},{"category":"my","metric":"brier","value":1.2443,"lo":null,"hi":null,"n":100,"of":100},{"category":"my","metric":"ece","value":0.4766,"lo":null,"hi":null,"n":100,"of":100},{"category":"my","metric":"aurc","value":0.8866,"lo":null,"hi":null,"n":100,"of":100},{"category":"my","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"nb","metric":"accuracy","value":0.5,"lo":null,"hi":null,"n":100,"of":100},{"category":"nb","metric":"brier","value":0.8543,"lo":null,"hi":null,"n":100,"of":100},{"category":"nb","metric":"ece","value":0.3679,"lo":null,"hi":null,"n":100,"of":100},{"category":"nb","metric":"aurc","value":0.2972,"lo":null,"hi":null,"n":100,"of":100},{"category":"nb","metric":"zero_prob","value":0.04,"lo":null,"hi":null,"n":100,"of":100},{"category":"nl","metric":"accuracy","value":0.4,"lo":null,"hi":null,"n":100,"of":100},{"category":"nl","metric":"brier","value":1.0453,"lo":null,"hi":null,"n":100,"of":100},{"category":"nl","metric":"ece","value":0.4792,"lo":null,"hi":null,"n":100,"of":100},{"category":"nl","metric":"aurc","value":0.445,"lo":null,"hi":null,"n":100,"of":100},{"category":"nl","metric":"zero_prob","value":0.16,"lo":null,"hi":null,"n":100,"of":100},{"category":"pl","metric":"accuracy","value":0.47,"lo":null,"hi":null,"n":100,"of":100},{"category":"pl","metric":"brier","value":0.8546,"lo":null,"hi":null,"n":100,"of":100},{"category":"pl","metric":"ece","value":0.3737,"lo":null,"hi":null,"n":100,"of":100},{"category":"pl","metric":"aurc","value":0.2628,"lo":null,"hi":null,"n":100,"of":100},{"category":"pl","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"pt","metric":"accuracy","value":0.44,"lo":null,"hi":null,"n":100,"of":100},{"category":"pt","metric":"brier","value":0.9349,"lo":null,"hi":null,"n":100,"of":100},{"category":"pt","metric":"ece","value":0.3956,"lo":null,"hi":null,"n":100,"of":100},{"category":"pt","metric":"aurc","value":0.385,"lo":null,"hi":null,"n":100,"of":100},{"category":"pt","metric":"zero_prob","value":0.08,"lo":null,"hi":null,"n":100,"of":100},{"category":"ro","metric":"accuracy","value":0.37,"lo":null,"hi":null,"n":100,"of":100},{"category":"ro","metric":"brier","value":1.0551,"lo":null,"hi":null,"n":100,"of":100},{"category":"ro","metric":"ece","value":0.473,"lo":null,"hi":null,"n":100,"of":100},{"category":"ro","metric":"aurc","value":0.4941,"lo":null,"hi":null,"n":100,"of":100},{"category":"ro","metric":"zero_prob","value":0.07,"lo":null,"hi":null,"n":100,"of":100},{"category":"ru","metric":"accuracy","value":0.57,"lo":null,"hi":null,"n":100,"of":100},{"category":"ru","metric":"brier","value":0.7319,"lo":null,"hi":null,"n":100,"of":100},{"category":"ru","metric":"ece","value":0.2959,"lo":null,"hi":null,"n":100,"of":100},{"category":"ru","metric":"aurc","value":0.3078,"lo":null,"hi":null,"n":100,"of":100},{"category":"ru","metric":"zero_prob","value":0.01,"lo":null,"hi":null,"n":100,"of":100},{"category":"sl","metric":"accuracy","value":0.33,"lo":null,"hi":null,"n":100,"of":100},{"category":"sl","metric":"brier","value":1.1102,"lo":null,"hi":null,"n":100,"of":100},{"category":"sl","metric":"ece","value":0.4871,"lo":null,"hi":null,"n":100,"of":100},{"category":"sl","metric":"aurc","value":0.5472,"lo":null,"hi":null,"n":100,"of":100},{"category":"sl","metric":"zero_prob","value":0.06,"lo":null,"hi":null,"n":100,"of":100},{"category":"sq","metric":"accuracy","value":0.29,"lo":null,"hi":null,"n":100,"of":100},{"category":"sq","metric":"brier","value":1.1614,"lo":null,"hi":null,"n":100,"of":100},{"category":"sq","metric":"ece","value":0.4996,"lo":null,"hi":null,"n":100,"of":100},{"category":"sq","metric":"aurc","value":0.5334,"lo":null,"hi":null,"n":100,"of":100},{"category":"sq","metric":"zero_prob","value":0.02,"lo":null,"hi":null,"n":100,"of":100},{"category":"sv","metric":"accuracy","value":0.48,"lo":null,"hi":null,"n":100,"of":100},{"category":"sv","metric":"brier","value":0.8409,"lo":null,"hi":null,"n":100,"of":100},{"category":"sv","metric":"ece","value":0.3699,"lo":null,"hi":null,"n":100,"of":100},{"category":"sv","metric":"aurc","value":0.3547,"lo":null,"hi":null,"n":100,"of":100},{"category":"sv","metric":"zero_prob","value":0.04,"lo":null,"hi":null,"n":100,"of":100},{"category":"sw","metric":"accuracy","value":0.13,"lo":null,"hi":null,"n":100,"of":100},{"category":"sw","metric":"brier","value":1.4219,"lo":null,"hi":null,"n":100,"of":100},{"category":"sw","metric":"ece","value":0.6279,"lo":null,"hi":null,"n":100,"of":100},{"category":"sw","metric":"aurc","value":0.8262,"lo":null,"hi":null,"n":100,"of":100},{"category":"sw","metric":"zero_prob","value":0.06,"lo":null,"hi":null,"n":100,"of":100},{"category":"ta","metric":"accuracy","value":0.31,"lo":null,"hi":null,"n":100,"of":100},{"category":"ta","metric":"brier","value":1.1179,"lo":null,"hi":null,"n":100,"of":100},{"category":"ta","metric":"ece","value":0.4479,"lo":null,"hi":null,"n":100,"of":100},{"category":"ta","metric":"aurc","value":0.6719,"lo":null,"hi":null,"n":100,"of":100},{"category":"ta","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"te","metric":"accuracy","value":0.22,"lo":null,"hi":null,"n":100,"of":100},{"category":"te","metric":"brier","value":1.1789,"lo":null,"hi":null,"n":100,"of":100},{"category":"te","metric":"ece","value":0.4969,"lo":null,"hi":null,"n":100,"of":100},{"category":"te","metric":"aurc","value":0.635,"lo":null,"hi":null,"n":100,"of":100},{"category":"te","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"th","metric":"accuracy","value":0.48,"lo":null,"hi":null,"n":100,"of":100},{"category":"th","metric":"brier","value":0.8366,"lo":null,"hi":null,"n":100,"of":100},{"category":"th","metric":"ece","value":0.3563,"lo":null,"hi":null,"n":100,"of":100},{"category":"th","metric":"aurc","value":0.3897,"lo":null,"hi":null,"n":100,"of":100},{"category":"th","metric":"zero_prob","value":0.01,"lo":null,"hi":null,"n":100,"of":100},{"category":"tl","metric":"accuracy","value":0.3,"lo":null,"hi":null,"n":100,"of":100},{"category":"tl","metric":"brier","value":1.1279,"lo":null,"hi":null,"n":100,"of":100},{"category":"tl","metric":"ece","value":0.4922,"lo":null,"hi":null,"n":100,"of":100},{"category":"tl","metric":"aurc","value":0.447,"lo":null,"hi":null,"n":100,"of":100},{"category":"tl","metric":"zero_prob","value":0.06,"lo":null,"hi":null,"n":100,"of":100},{"category":"tr","metric":"accuracy","value":0.41,"lo":null,"hi":null,"n":100,"of":100},{"category":"tr","metric":"brier","value":0.937,"lo":null,"hi":null,"n":100,"of":100},{"category":"tr","metric":"ece","value":0.3854,"lo":null,"hi":null,"n":100,"of":100},{"category":"tr","metric":"aurc","value":0.3757,"lo":null,"hi":null,"n":100,"of":100},{"category":"tr","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"ur","metric":"accuracy","value":0.42,"lo":null,"hi":null,"n":100,"of":100},{"category":"ur","metric":"brier","value":0.8645,"lo":null,"hi":null,"n":100,"of":100},{"category":"ur","metric":"ece","value":0.3355,"lo":null,"hi":null,"n":100,"of":100},{"category":"ur","metric":"aurc","value":0.4357,"lo":null,"hi":null,"n":100,"of":100},{"category":"ur","metric":"zero_prob","value":0.01,"lo":null,"hi":null,"n":100,"of":100},{"category":"vi","metric":"accuracy","value":0.33,"lo":null,"hi":null,"n":100,"of":100},{"category":"vi","metric":"brier","value":1.0723,"lo":null,"hi":null,"n":100,"of":100},{"category":"vi","metric":"ece","value":0.467,"lo":null,"hi":null,"n":100,"of":100},{"category":"vi","metric":"aurc","value":0.5551,"lo":null,"hi":null,"n":100,"of":100},{"category":"vi","metric":"zero_prob","value":0.02,"lo":null,"hi":null,"n":100,"of":100},{"category":"zh-CN","metric":"accuracy","value":0.65,"lo":null,"hi":null,"n":100,"of":100},{"category":"zh-CN","metric":"brier","value":0.5484,"lo":null,"hi":null,"n":100,"of":100},{"category":"zh-CN","metric":"ece","value":0.2192,"lo":null,"hi":null,"n":100,"of":100},{"category":"zh-CN","metric":"aurc","value":0.1685,"lo":null,"hi":null,"n":100,"of":100},{"category":"zh-CN","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"zh-TW","metric":"accuracy","value":0.61,"lo":null,"hi":null,"n":100,"of":100},{"category":"zh-TW","metric":"brier","value":0.6563,"lo":null,"hi":null,"n":100,"of":100},{"category":"zh-TW","metric":"ece","value":0.2662,"lo":null,"hi":null,"n":100,"of":100},{"category":"zh-TW","metric":"aurc","value":0.2685,"lo":null,"hi":null,"n":100,"of":100},{"category":"zh-TW","metric":"zero_prob","value":0.01,"lo":null,"hi":null,"n":100,"of":100}],"notes":"runtime laya 0.3.20; device cpu; dtype f32; preds sha256:15e4ba94d76db187424d43e7823ffbd029214f6552bf91aade0037448ce694c6","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"massive-harness-typed-decisions-laya","benchmark":"massive","split":"harness","system":"laya","version":"convaiinnovations/laya-typed-decisions@1a793eb568e6718f15941d08f85432581df534e3","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"735e240323cd7c9f0c880b7d8c62d89f40be2164","questions":5100,"answered":5100,"when":"","ended":"2026-09-25T17:33:56Z","by":"","completion":"complete","measures":[{"category":"all","metric":"macro_accuracy","value":0.2224,"lo":null,"hi":null,"n":5100,"of":5100},{"category":"all","metric":"macro_brier","value":1.0785,"lo":null,"hi":null,"n":5100,"of":5100},{"category":"all","metric":"macro_ece","value":0.3989,"lo":null,"hi":null,"n":5100,"of":5100},{"category":"all","metric":"macro_aurc","value":0.6682,"lo":null,"hi":null,"n":5100,"of":5100},{"category":"all","metric":"macro_zero_prob","value":0,"lo":null,"hi":null,"n":5100,"of":5100},{"category":"af","metric":"accuracy","value":0.32,"lo":null,"hi":null,"n":100,"of":100},{"category":"af","metric":"brier","value":1.0065,"lo":null,"hi":null,"n":100,"of":100},{"category":"af","metric":"ece","value":0.4265,"lo":null,"hi":null,"n":100,"of":100},{"category":"af","metric":"aurc","value":0.5088,"lo":null,"hi":null,"n":100,"of":100},{"category":"af","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"am","metric":"accuracy","value":0.16,"lo":null,"hi":null,"n":100,"of":100},{"category":"am","metric":"brier","value":1.2868,"lo":null,"hi":null,"n":100,"of":100},{"category":"am","metric":"ece","value":0.4891,"lo":null,"hi":null,"n":100,"of":100},{"category":"am","metric":"aurc","value":0.8667,"lo":null,"hi":null,"n":100,"of":100},{"category":"am","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"ar","metric":"accuracy","value":0.08,"lo":null,"hi":null,"n":100,"of":100},{"category":"ar","metric":"brier","value":1.1943,"lo":null,"hi":null,"n":100,"of":100},{"category":"ar","metric":"ece","value":0.4425,"lo":null,"hi":null,"n":100,"of":100},{"category":"ar","metric":"aurc","value":0.8303,"lo":null,"hi":null,"n":100,"of":100},{"category":"ar","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"az","metric":"accuracy","value":0.07,"lo":null,"hi":null,"n":100,"of":100},{"category":"az","metric":"brier","value":1.2345,"lo":null,"hi":null,"n":100,"of":100},{"category":"az","metric":"ece","value":0.4652,"lo":null,"hi":null,"n":100,"of":100},{"category":"az","metric":"aurc","value":0.8464,"lo":null,"hi":null,"n":100,"of":100},{"category":"az","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"bn","metric":"accuracy","value":0.12,"lo":null,"hi":null,"n":100,"of":100},{"category":"bn","metric":"brier","value":1.2058,"lo":null,"hi":null,"n":100,"of":100},{"category":"bn","metric":"ece","value":0.4204,"lo":null,"hi":null,"n":100,"of":100},{"category":"bn","metric":"aurc","value":0.8005,"lo":null,"hi":null,"n":100,"of":100},{"category":"bn","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"cy","metric":"accuracy","value":0.17,"lo":null,"hi":null,"n":100,"of":100},{"category":"cy","metric":"brier","value":1.2037,"lo":null,"hi":null,"n":100,"of":100},{"category":"cy","metric":"ece","value":0.4455,"lo":null,"hi":null,"n":100,"of":100},{"category":"cy","metric":"aurc","value":0.6951,"lo":null,"hi":null,"n":100,"of":100},{"category":"cy","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"da","metric":"accuracy","value":0.34,"lo":null,"hi":null,"n":100,"of":100},{"category":"da","metric":"brier","value":0.9178,"lo":null,"hi":null,"n":100,"of":100},{"category":"da","metric":"ece","value":0.3517,"lo":null,"hi":null,"n":100,"of":100},{"category":"da","metric":"aurc","value":0.3578,"lo":null,"hi":null,"n":100,"of":100},{"category":"da","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"de","metric":"accuracy","value":0.39,"lo":null,"hi":null,"n":100,"of":100},{"category":"de","metric":"brier","value":0.8496,"lo":null,"hi":null,"n":100,"of":100},{"category":"de","metric":"ece","value":0.3393,"lo":null,"hi":null,"n":100,"of":100},{"category":"de","metric":"aurc","value":0.362,"lo":null,"hi":null,"n":100,"of":100},{"category":"de","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"el","metric":"accuracy","value":0.13,"lo":null,"hi":null,"n":100,"of":100},{"category":"el","metric":"brier","value":1.2128,"lo":null,"hi":null,"n":100,"of":100},{"category":"el","metric":"ece","value":0.4772,"lo":null,"hi":null,"n":100,"of":100},{"category":"el","metric":"aurc","value":0.7676,"lo":null,"hi":null,"n":100,"of":100},{"category":"el","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"en","metric":"accuracy","value":0.82,"lo":null,"hi":null,"n":100,"of":100},{"category":"en","metric":"brier","value":0.2756,"lo":null,"hi":null,"n":100,"of":100},{"category":"en","metric":"ece","value":0.0932,"lo":null,"hi":null,"n":100,"of":100},{"category":"en","metric":"aurc","value":0.0405,"lo":null,"hi":null,"n":100,"of":100},{"category":"en","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"es","metric":"accuracy","value":0.45,"lo":null,"hi":null,"n":100,"of":100},{"category":"es","metric":"brier","value":0.7533,"lo":null,"hi":null,"n":100,"of":100},{"category":"es","metric":"ece","value":0.2835,"lo":null,"hi":null,"n":100,"of":100},{"category":"es","metric":"aurc","value":0.2888,"lo":null,"hi":null,"n":100,"of":100},{"category":"es","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"fa","metric":"accuracy","value":0.09,"lo":null,"hi":null,"n":100,"of":100},{"category":"fa","metric":"brier","value":1.2247,"lo":null,"hi":null,"n":100,"of":100},{"category":"fa","metric":"ece","value":0.4465,"lo":null,"hi":null,"n":100,"of":100},{"category":"fa","metric":"aurc","value":0.9094,"lo":null,"hi":null,"n":100,"of":100},{"category":"fa","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"fi","metric":"accuracy","value":0.12,"lo":null,"hi":null,"n":100,"of":100},{"category":"fi","metric":"brier","value":1.2553,"lo":null,"hi":null,"n":100,"of":100},{"category":"fi","metric":"ece","value":0.515,"lo":null,"hi":null,"n":100,"of":100},{"category":"fi","metric":"aurc","value":0.7582,"lo":null,"hi":null,"n":100,"of":100},{"category":"fi","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"fr","metric":"accuracy","value":0.6,"lo":null,"hi":null,"n":100,"of":100},{"category":"fr","metric":"brier","value":0.638,"lo":null,"hi":null,"n":100,"of":100},{"category":"fr","metric":"ece","value":0.228,"lo":null,"hi":null,"n":100,"of":100},{"category":"fr","metric":"aurc","value":0.1975,"lo":null,"hi":null,"n":100,"of":100},{"category":"fr","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"he","metric":"accuracy","value":0.03,"lo":null,"hi":null,"n":100,"of":100},{"category":"he","metric":"brier","value":1.2889,"lo":null,"hi":null,"n":100,"of":100},{"category":"he","metric":"ece","value":0.5053,"lo":null,"hi":null,"n":100,"of":100},{"category":"he","metric":"aurc","value":0.9626,"lo":null,"hi":null,"n":100,"of":100},{"category":"he","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"hi","metric":"accuracy","value":0.11,"lo":null,"hi":null,"n":100,"of":100},{"category":"hi","metric":"brier","value":1.2697,"lo":null,"hi":null,"n":100,"of":100},{"category":"hi","metric":"ece","value":0.46,"lo":null,"hi":null,"n":100,"of":100},{"category":"hi","metric":"aurc","value":0.8761,"lo":null,"hi":null,"n":100,"of":100},{"category":"hi","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"hu","metric":"accuracy","value":0.07,"lo":null,"hi":null,"n":100,"of":100},{"category":"hu","metric":"brier","value":1.3306,"lo":null,"hi":null,"n":100,"of":100},{"category":"hu","metric":"ece","value":0.5381,"lo":null,"hi":null,"n":100,"of":100},{"category":"hu","metric":"aurc","value":0.8788,"lo":null,"hi":null,"n":100,"of":100},{"category":"hu","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"hy","metric":"accuracy","value":0.04,"lo":null,"hi":null,"n":100,"of":100},{"category":"hy","metric":"brier","value":1.2273,"lo":null,"hi":null,"n":100,"of":100},{"category":"hy","metric":"ece","value":0.419,"lo":null,"hi":null,"n":100,"of":100},{"category":"hy","metric":"aurc","value":0.9777,"lo":null,"hi":null,"n":100,"of":100},{"category":"hy","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"id","metric":"accuracy","value":0.35,"lo":null,"hi":null,"n":100,"of":100},{"category":"id","metric":"brier","value":0.9377,"lo":null,"hi":null,"n":100,"of":100},{"category":"id","metric":"ece","value":0.3176,"lo":null,"hi":null,"n":100,"of":100},{"category":"id","metric":"aurc","value":0.4618,"lo":null,"hi":null,"n":100,"of":100},{"category":"id","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"is","metric":"accuracy","value":0.1,"lo":null,"hi":null,"n":100,"of":100},{"category":"is","metric":"brier","value":1.3122,"lo":null,"hi":null,"n":100,"of":100},{"category":"is","metric":"ece","value":0.5395,"lo":null,"hi":null,"n":100,"of":100},{"category":"is","metric":"aurc","value":0.8442,"lo":null,"hi":null,"n":100,"of":100},{"category":"is","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"it","metric":"accuracy","value":0.41,"lo":null,"hi":null,"n":100,"of":100},{"category":"it","metric":"brier","value":0.8979,"lo":null,"hi":null,"n":100,"of":100},{"category":"it","metric":"ece","value":0.3423,"lo":null,"hi":null,"n":100,"of":100},{"category":"it","metric":"aurc","value":0.4054,"lo":null,"hi":null,"n":100,"of":100},{"category":"it","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"ja","metric":"accuracy","value":0.56,"lo":null,"hi":null,"n":100,"of":100},{"category":"ja","metric":"brier","value":0.6878,"lo":null,"hi":null,"n":100,"of":100},{"category":"ja","metric":"ece","value":0.2391,"lo":null,"hi":null,"n":100,"of":100},{"category":"ja","metric":"aurc","value":0.2428,"lo":null,"hi":null,"n":100,"of":100},{"category":"ja","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"jv","metric":"accuracy","value":0.11,"lo":null,"hi":null,"n":100,"of":100},{"category":"jv","metric":"brier","value":1.2828,"lo":null,"hi":null,"n":100,"of":100},{"category":"jv","metric":"ece","value":0.5344,"lo":null,"hi":null,"n":100,"of":100},{"category":"jv","metric":"aurc","value":0.792,"lo":null,"hi":null,"n":100,"of":100},{"category":"jv","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"ka","metric":"accuracy","value":0.04,"lo":null,"hi":null,"n":100,"of":100},{"category":"ka","metric":"brier","value":1.2406,"lo":null,"hi":null,"n":100,"of":100},{"category":"ka","metric":"ece","value":0.4568,"lo":null,"hi":null,"n":100,"of":100},{"category":"ka","metric":"aurc","value":0.9532,"lo":null,"hi":null,"n":100,"of":100},{"category":"ka","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"km","metric":"accuracy","value":0.06,"lo":null,"hi":null,"n":100,"of":100},{"category":"km","metric":"brier","value":1.1745,"lo":null,"hi":null,"n":100,"of":100},{"category":"km","metric":"ece","value":0.3552,"lo":null,"hi":null,"n":100,"of":100},{"category":"km","metric":"aurc","value":0.9687,"lo":null,"hi":null,"n":100,"of":100},{"category":"km","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"kn","metric":"accuracy","value":0.12,"lo":null,"hi":null,"n":100,"of":100},{"category":"kn","metric":"brier","value":1.1718,"lo":null,"hi":null,"n":100,"of":100},{"category":"kn","metric":"ece","value":0.3606,"lo":null,"hi":null,"n":100,"of":100},{"category":"kn","metric":"aurc","value":0.8859,"lo":null,"hi":null,"n":100,"of":100},{"category":"kn","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"ko","metric":"accuracy","value":0.06,"lo":null,"hi":null,"n":100,"of":100},{"category":"ko","metric":"brier","value":1.2604,"lo":null,"hi":null,"n":100,"of":100},{"category":"ko","metric":"ece","value":0.5059,"lo":null,"hi":null,"n":100,"of":100},{"category":"ko","metric":"aurc","value":0.8776,"lo":null,"hi":null,"n":100,"of":100},{"category":"ko","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"lv","metric":"accuracy","value":0.09,"lo":null,"hi":null,"n":100,"of":100},{"category":"lv","metric":"brier","value":1.2831,"lo":null,"hi":null,"n":100,"of":100},{"category":"lv","metric":"ece","value":0.4966,"lo":null,"hi":null,"n":100,"of":100},{"category":"lv","metric":"aurc","value":0.8759,"lo":null,"hi":null,"n":100,"of":100},{"category":"lv","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"ml","metric":"accuracy","value":0.1,"lo":null,"hi":null,"n":100,"of":100},{"category":"ml","metric":"brier","value":1.1473,"lo":null,"hi":null,"n":100,"of":100},{"category":"ml","metric":"ece","value":0.3686,"lo":null,"hi":null,"n":100,"of":100},{"category":"ml","metric":"aurc","value":0.8208,"lo":null,"hi":null,"n":100,"of":100},{"category":"ml","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"mn","metric":"accuracy","value":0.07,"lo":null,"hi":null,"n":100,"of":100},{"category":"mn","metric":"brier","value":1.3665,"lo":null,"hi":null,"n":100,"of":100},{"category":"mn","metric":"ece","value":0.569,"lo":null,"hi":null,"n":100,"of":100},{"category":"mn","metric":"aurc","value":0.8957,"lo":null,"hi":null,"n":100,"of":100},{"category":"mn","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"ms","metric":"accuracy","value":0.24,"lo":null,"hi":null,"n":100,"of":100},{"category":"ms","metric":"brier","value":1.0249,"lo":null,"hi":null,"n":100,"of":100},{"category":"ms","metric":"ece","value":0.3277,"lo":null,"hi":null,"n":100,"of":100},{"category":"ms","metric":"aurc","value":0.6168,"lo":null,"hi":null,"n":100,"of":100},{"category":"ms","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"my","metric":"accuracy","value":0.08,"lo":null,"hi":null,"n":100,"of":100},{"category":"my","metric":"brier","value":1.1473,"lo":null,"hi":null,"n":100,"of":100},{"category":"my","metric":"ece","value":0.3454,"lo":null,"hi":null,"n":100,"of":100},{"category":"my","metric":"aurc","value":0.9293,"lo":null,"hi":null,"n":100,"of":100},{"category":"my","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"nb","metric":"accuracy","value":0.36,"lo":null,"hi":null,"n":100,"of":100},{"category":"nb","metric":"brier","value":0.8917,"lo":null,"hi":null,"n":100,"of":100},{"category":"nb","metric":"ece","value":0.3212,"lo":null,"hi":null,"n":100,"of":100},{"category":"nb","metric":"aurc","value":0.3869,"lo":null,"hi":null,"n":100,"of":100},{"category":"nb","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"nl","metric":"accuracy","value":0.37,"lo":null,"hi":null,"n":100,"of":100},{"category":"nl","metric":"brier","value":0.924,"lo":null,"hi":null,"n":100,"of":100},{"category":"nl","metric":"ece","value":0.3445,"lo":null,"hi":null,"n":100,"of":100},{"category":"nl","metric":"aurc","value":0.4568,"lo":null,"hi":null,"n":100,"of":100},{"category":"nl","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"pl","metric":"accuracy","value":0.25,"lo":null,"hi":null,"n":100,"of":100},{"category":"pl","metric":"brier","value":1.0012,"lo":null,"hi":null,"n":100,"of":100},{"category":"pl","metric":"ece","value":0.3442,"lo":null,"hi":null,"n":100,"of":100},{"category":"pl","metric":"aurc","value":0.5573,"lo":null,"hi":null,"n":100,"of":100},{"category":"pl","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"pt","metric":"accuracy","value":0.5,"lo":null,"hi":null,"n":100,"of":100},{"category":"pt","metric":"brier","value":0.7686,"lo":null,"hi":null,"n":100,"of":100},{"category":"pt","metric":"ece","value":0.2695,"lo":null,"hi":null,"n":100,"of":100},{"category":"pt","metric":"aurc","value":0.3309,"lo":null,"hi":null,"n":100,"of":100},{"category":"pt","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"ro","metric":"accuracy","value":0.31,"lo":null,"hi":null,"n":100,"of":100},{"category":"ro","metric":"brier","value":1.0408,"lo":null,"hi":null,"n":100,"of":100},{"category":"ro","metric":"ece","value":0.399,"lo":null,"hi":null,"n":100,"of":100},{"category":"ro","metric":"aurc","value":0.5802,"lo":null,"hi":null,"n":100,"of":100},{"category":"ro","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"ru","metric":"accuracy","value":0.39,"lo":null,"hi":null,"n":100,"of":100},{"category":"ru","metric":"brier","value":0.9005,"lo":null,"hi":null,"n":100,"of":100},{"category":"ru","metric":"ece","value":0.2835,"lo":null,"hi":null,"n":100,"of":100},{"category":"ru","metric":"aurc","value":0.4751,"lo":null,"hi":null,"n":100,"of":100},{"category":"ru","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"sl","metric":"accuracy","value":0.16,"lo":null,"hi":null,"n":100,"of":100},{"category":"sl","metric":"brier","value":1.1234,"lo":null,"hi":null,"n":100,"of":100},{"category":"sl","metric":"ece","value":0.3987,"lo":null,"hi":null,"n":100,"of":100},{"category":"sl","metric":"aurc","value":0.6819,"lo":null,"hi":null,"n":100,"of":100},{"category":"sl","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"sq","metric":"accuracy","value":0.19,"lo":null,"hi":null,"n":100,"of":100},{"category":"sq","metric":"brier","value":1.1237,"lo":null,"hi":null,"n":100,"of":100},{"category":"sq","metric":"ece","value":0.4217,"lo":null,"hi":null,"n":100,"of":100},{"category":"sq","metric":"aurc","value":0.6277,"lo":null,"hi":null,"n":100,"of":100},{"category":"sq","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"sv","metric":"accuracy","value":0.24,"lo":null,"hi":null,"n":100,"of":100},{"category":"sv","metric":"brier","value":1.0703,"lo":null,"hi":null,"n":100,"of":100},{"category":"sv","metric":"ece","value":0.4338,"lo":null,"hi":null,"n":100,"of":100},{"category":"sv","metric":"aurc","value":0.5683,"lo":null,"hi":null,"n":100,"of":100},{"category":"sv","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"sw","metric":"accuracy","value":0.1,"lo":null,"hi":null,"n":100,"of":100},{"category":"sw","metric":"brier","value":1.2274,"lo":null,"hi":null,"n":100,"of":100},{"category":"sw","metric":"ece","value":0.4629,"lo":null,"hi":null,"n":100,"of":100},{"category":"sw","metric":"aurc","value":0.8354,"lo":null,"hi":null,"n":100,"of":100},{"category":"sw","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"ta","metric":"accuracy","value":0.08,"lo":null,"hi":null,"n":100,"of":100},{"category":"ta","metric":"brier","value":1.14,"lo":null,"hi":null,"n":100,"of":100},{"category":"ta","metric":"ece","value":0.3566,"lo":null,"hi":null,"n":100,"of":100},{"category":"ta","metric":"aurc","value":0.8933,"lo":null,"hi":null,"n":100,"of":100},{"category":"ta","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"te","metric":"accuracy","value":0.09,"lo":null,"hi":null,"n":100,"of":100},{"category":"te","metric":"brier","value":1.2271,"lo":null,"hi":null,"n":100,"of":100},{"category":"te","metric":"ece","value":0.4389,"lo":null,"hi":null,"n":100,"of":100},{"category":"te","metric":"aurc","value":0.9119,"lo":null,"hi":null,"n":100,"of":100},{"category":"te","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"th","metric":"accuracy","value":0.1,"lo":null,"hi":null,"n":100,"of":100},{"category":"th","metric":"brier","value":1.2942,"lo":null,"hi":null,"n":100,"of":100},{"category":"th","metric":"ece","value":0.54,"lo":null,"hi":null,"n":100,"of":100},{"category":"th","metric":"aurc","value":0.8633,"lo":null,"hi":null,"n":100,"of":100},{"category":"th","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"tl","metric":"accuracy","value":0.32,"lo":null,"hi":null,"n":100,"of":100},{"category":"tl","metric":"brier","value":0.9696,"lo":null,"hi":null,"n":100,"of":100},{"category":"tl","metric":"ece","value":0.349,"lo":null,"hi":null,"n":100,"of":100},{"category":"tl","metric":"aurc","value":0.4553,"lo":null,"hi":null,"n":100,"of":100},{"category":"tl","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"tr","metric":"accuracy","value":0.14,"lo":null,"hi":null,"n":100,"of":100},{"category":"tr","metric":"brier","value":1.1428,"lo":null,"hi":null,"n":100,"of":100},{"category":"tr","metric":"ece","value":0.4196,"lo":null,"hi":null,"n":100,"of":100},{"category":"tr","metric":"aurc","value":0.7145,"lo":null,"hi":null,"n":100,"of":100},{"category":"tr","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"ur","metric":"accuracy","value":0.11,"lo":null,"hi":null,"n":100,"of":100},{"category":"ur","metric":"brier","value":1.1741,"lo":null,"hi":null,"n":100,"of":100},{"category":"ur","metric":"ece","value":0.4079,"lo":null,"hi":null,"n":100,"of":100},{"category":"ur","metric":"aurc","value":0.8238,"lo":null,"hi":null,"n":100,"of":100},{"category":"ur","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"vi","metric":"accuracy","value":0.06,"lo":null,"hi":null,"n":100,"of":100},{"category":"vi","metric":"brier","value":1.284,"lo":null,"hi":null,"n":100,"of":100},{"category":"vi","metric":"ece","value":0.5037,"lo":null,"hi":null,"n":100,"of":100},{"category":"vi","metric":"aurc","value":0.9104,"lo":null,"hi":null,"n":100,"of":100},{"category":"vi","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"zh-CN","metric":"accuracy","value":0.6,"lo":null,"hi":null,"n":100,"of":100},{"category":"zh-CN","metric":"brier","value":0.6039,"lo":null,"hi":null,"n":100,"of":100},{"category":"zh-CN","metric":"ece","value":0.2483,"lo":null,"hi":null,"n":100,"of":100},{"category":"zh-CN","metric":"aurc","value":0.1983,"lo":null,"hi":null,"n":100,"of":100},{"category":"zh-CN","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100},{"category":"zh-TW","metric":"accuracy","value":0.47,"lo":null,"hi":null,"n":100,"of":100},{"category":"zh-TW","metric":"brier","value":0.7866,"lo":null,"hi":null,"n":100,"of":100},{"category":"zh-TW","metric":"ece","value":0.2984,"lo":null,"hi":null,"n":100,"of":100},{"category":"zh-TW","metric":"aurc","value":0.3109,"lo":null,"hi":null,"n":100,"of":100},{"category":"zh-TW","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":100,"of":100}],"notes":"runtime laya 0.3.20; device cpu; dtype f32; preds sha256:15e4ba94d76db187424d43e7823ffbd029214f6552bf91aade0037448ce694c6","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"typed_decisions-harness-20260917-jev","benchmark":"typed_decisions","split":"harness","system":"jev","version":"typesafe/jev-1.13-20260917","baseline":true,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"e95c6ee6362fd66d6d80c711095e03f30bbeeffd","questions":2000,"answered":2000,"when":"","ended":"2026-09-25T02:55:39Z","by":"","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.7355,"lo":null,"hi":null,"n":2000,"of":2000},{"category":"all","metric":"brier","value":0.3658,"lo":null,"hi":null,"n":2000,"of":2000},{"category":"all","metric":"ece","value":0.0472,"lo":null,"hi":null,"n":2000,"of":2000},{"category":"all","metric":"aurc","value":0.1277,"lo":null,"hi":null,"n":2000,"of":2000},{"category":"all","metric":"zero_prob","value":0.003,"lo":null,"hi":null,"n":2000,"of":2000}],"notes":"preds sha256:506b7c6c1de83a990601073493071c287f89cba9c455ccdf34d26575edc1a7a2","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"typed_decisions-harness-a5.df1c16a2-kai","benchmark":"typed_decisions","split":"harness","system":"kai","version":"a5@df1c16a2fef7ec93f2b34a6a194f6dc3c2eee70e936473b802151543c5d46e83","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"611059b60f64ab686d6475cd76778b3fac52cd98","questions":2000,"answered":2000,"when":"","ended":"2026-09-27T21:09:54Z","by":"","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.705,"lo":null,"hi":null,"n":2000,"of":2000},{"category":"all","metric":"brier","value":0.4488,"lo":null,"hi":null,"n":2000,"of":2000},{"category":"all","metric":"ece","value":0.1706,"lo":null,"hi":null,"n":2000,"of":2000},{"category":"all","metric":"aurc","value":0.152,"lo":null,"hi":null,"n":2000,"of":2000},{"category":"all","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":2000,"of":2000}],"notes":"device metal; dtype bf16; preds sha256:f28954731ec991ecbed3c1fbfc45980eeffef7744d8c12bcfc4cb160cb6f3eac; scores sha256:071b556a8eafd2c19d4f9a948b4f10251c9b253107c6afb21ad959ef72af4a77","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"typed_decisions-harness-a6.a211cc70-kai","benchmark":"typed_decisions","split":"harness","system":"kai","version":"a6@a211cc70103825bfbc9bfb0fbe970f8bf08da2efe0953cd8045a3cd07397f4bf","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"a1a4a73d524cecf8038e30c408ad30b66e71d490","questions":2000,"answered":2000,"when":"","ended":"2026-09-28T03:18:42Z","by":"","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.75,"lo":null,"hi":null,"n":2000,"of":2000},{"category":"all","metric":"brier","value":0.4274,"lo":null,"hi":null,"n":2000,"of":2000},{"category":"all","metric":"ece","value":0.2086,"lo":null,"hi":null,"n":2000,"of":2000},{"category":"all","metric":"aurc","value":0.1301,"lo":null,"hi":null,"n":2000,"of":2000},{"category":"all","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":2000,"of":2000}],"notes":"device metal; dtype bf16; preds sha256:43b1bd476a72a536ee96fbc934a065a808fdb35395e694d9af873e462b3d8866; scores sha256:3760df4b2c4cdcccb225bbb446b2ddcff998e09ab723e36ee466f771dc355a7e","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"typed_decisions-harness-a7.0834a74f-kai","benchmark":"typed_decisions","split":"harness","system":"kai","version":"a7@0834a74f2d140642a453373da09e5a128e3d2c8d9e8bb1dcaf86d7210a4ecdfc","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"a1a4a73d524cecf8038e30c408ad30b66e71d490","questions":2000,"answered":2000,"when":"","ended":"2026-09-28T12:24:45Z","by":"","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.759,"lo":null,"hi":null,"n":2000,"of":2000},{"category":"all","metric":"brier","value":0.3884,"lo":null,"hi":null,"n":2000,"of":2000},{"category":"all","metric":"ece","value":0.1716,"lo":null,"hi":null,"n":2000,"of":2000},{"category":"all","metric":"aurc","value":0.1188,"lo":null,"hi":null,"n":2000,"of":2000},{"category":"all","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":2000,"of":2000}],"notes":"device metal; dtype bf16; preds sha256:25bfec2f5ad49bc050cc9015161c7ef2dbf07f599f35e9dad153b4a5b6f7b543; scores sha256:ddca48bcf9816e7df2a187251d9f0f00de3b2fc24f22805b34b7bd650a048c81","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"typed_decisions-harness-a8.0cfeef05-kai","benchmark":"typed_decisions","split":"harness","system":"kai","version":"a8@0cfeef05c879d7f24bbde275a6c23e85e80b6baf06cf9311d529010688d4a697","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"a1a4a73d524cecf8038e30c408ad30b66e71d490","questions":2000,"answered":2000,"when":"","ended":"2026-09-28T16:33:11Z","by":"","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.7615,"lo":null,"hi":null,"n":2000,"of":2000},{"category":"all","metric":"brier","value":0.3842,"lo":null,"hi":null,"n":2000,"of":2000},{"category":"all","metric":"ece","value":0.1638,"lo":null,"hi":null,"n":2000,"of":2000},{"category":"all","metric":"aurc","value":0.1184,"lo":null,"hi":null,"n":2000,"of":2000},{"category":"all","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":2000,"of":2000}],"notes":"device metal; dtype bf16; preds sha256:d0a81ccbdc9eb7305fc196d7ad2c6be8c1df1ce5966ac964dabeeea231ae41d6; scores sha256:86b58e1f53447a8fb3b7963ea69f2bee40ffa97b9bda7b74b038064188998f10","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"typed_decisions-harness-english-laya","benchmark":"typed_decisions","split":"harness","system":"laya","version":"convaiinnovations/laya@55cf4c4ebb4ebe31b2550e8bdf3bd21b99753851","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"735e240323cd7c9f0c880b7d8c62d89f40be2164","questions":2000,"answered":2000,"when":"","ended":"2026-09-25T17:33:56Z","by":"","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.3615,"lo":null,"hi":null,"n":2000,"of":2000},{"category":"all","metric":"brier","value":0.7497,"lo":null,"hi":null,"n":2000,"of":2000},{"category":"all","metric":"ece","value":0.1747,"lo":null,"hi":null,"n":2000,"of":2000},{"category":"all","metric":"aurc","value":0.5929,"lo":null,"hi":null,"n":2000,"of":2000},{"category":"all","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":2000,"of":2000}],"notes":"runtime laya 0.3.20; device cpu; dtype f32; preds sha256:50466f49ea7681cf049e8ed1e0554fd975386b35af04871926767b75b5c67894","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"typed_decisions-harness-kai-1.0834a74f-kai","benchmark":"typed_decisions","split":"harness","system":"kai","version":"kai-1@0834a74f2d140642a453373da09e5a128e3d2c8d9e8bb1dcaf86d7210a4ecdfc","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"2c74e2d61addc885e847358975fb866b03cf17dc","questions":2000,"answered":2000,"when":"","ended":"2026-10-03T00:06:45Z","by":"","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.759,"lo":null,"hi":null,"n":2000,"of":2000},{"category":"all","metric":"brier","value":0.3884,"lo":null,"hi":null,"n":2000,"of":2000},{"category":"all","metric":"ece","value":0.1708,"lo":null,"hi":null,"n":2000,"of":2000},{"category":"all","metric":"aurc","value":0.1187,"lo":null,"hi":null,"n":2000,"of":2000},{"category":"all","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":2000,"of":2000}],"notes":"device cuda; dtype bf16; preds sha256:be8a08a52469da5defd4bb4a4ef597f94f9508d1ae2c048234062a1cf9a713be; scores sha256:1ec811cfb085ac209c7b094b408b00ef95d3d3c73b7ae7bdf6c52fc314d112e0","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"typed_decisions-harness-multilingual-laya","benchmark":"typed_decisions","split":"harness","system":"laya","version":"convaiinnovations/laya-multilingual@e4e9ddf21a7b1903b7acffd8814ad4307bf63a67","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"735e240323cd7c9f0c880b7d8c62d89f40be2164","questions":2000,"answered":2000,"when":"","ended":"2026-09-25T17:33:56Z","by":"","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.352,"lo":null,"hi":null,"n":2000,"of":2000},{"category":"all","metric":"brier","value":0.8904,"lo":null,"hi":null,"n":2000,"of":2000},{"category":"all","metric":"ece","value":0.3139,"lo":null,"hi":null,"n":2000,"of":2000},{"category":"all","metric":"aurc","value":0.5563,"lo":null,"hi":null,"n":2000,"of":2000},{"category":"all","metric":"zero_prob","value":0.0005,"lo":null,"hi":null,"n":2000,"of":2000}],"notes":"runtime laya 0.3.20; device cpu; dtype f32; preds sha256:50466f49ea7681cf049e8ed1e0554fd975386b35af04871926767b75b5c67894","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"typed_decisions-harness-routed-laya","benchmark":"typed_decisions","split":"harness","system":"laya","version":"convaiinnovations/laya@55cf4c4ebb4ebe31b2550e8bdf3bd21b99753851+convaiinnovations/laya-multilingual@e4e9ddf21a7b1903b7acffd8814ad4307bf63a67","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"735e240323cd7c9f0c880b7d8c62d89f40be2164","questions":2000,"answered":2000,"when":"","ended":"2026-09-25T17:33:56Z","by":"","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.3615,"lo":null,"hi":null,"n":2000,"of":2000},{"category":"all","metric":"brier","value":0.7497,"lo":null,"hi":null,"n":2000,"of":2000},{"category":"all","metric":"ece","value":0.1747,"lo":null,"hi":null,"n":2000,"of":2000},{"category":"all","metric":"aurc","value":0.5929,"lo":null,"hi":null,"n":2000,"of":2000},{"category":"all","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":2000,"of":2000}],"notes":"runtime laya 0.3.20; device cpu; dtype f32; preds sha256:50466f49ea7681cf049e8ed1e0554fd975386b35af04871926767b75b5c67894","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false},{"project":"default","id":"typed_decisions-harness-typed-decisions-laya","benchmark":"typed_decisions","split":"harness","system":"laya","version":"convaiinnovations/laya-typed-decisions@1a793eb568e6718f15941d08f85432581df534e3","baseline":true,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"dde140a63fc527cbeb8f7ada29088bf17064d4bcf30c9680d6423696f0378673","store_digest":"","facts_digest":"","commit":"735e240323cd7c9f0c880b7d8c62d89f40be2164","questions":2000,"answered":2000,"when":"","ended":"2026-09-25T17:33:56Z","by":"","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.766,"lo":null,"hi":null,"n":2000,"of":2000},{"category":"all","metric":"brier","value":0.3996,"lo":null,"hi":null,"n":2000,"of":2000},{"category":"all","metric":"ece","value":0.2132,"lo":null,"hi":null,"n":2000,"of":2000},{"category":"all","metric":"aurc","value":0.0988,"lo":null,"hi":null,"n":2000,"of":2000},{"category":"all","metric":"zero_prob","value":0,"lo":null,"hi":null,"n":2000,"of":2000}],"notes":"runtime laya 0.3.20; device cpu; dtype f32; preds sha256:50466f49ea7681cf049e8ed1e0554fd975386b35af04871926767b75b5c67894","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false}],"total":160,"completion":{"complete":128,"partial":1,"inconsistent":0,"unknown":31},"gaps":[{"field":"embedder","runs":160},{"field":"facts_digest","runs":160},{"field":"k","runs":160},{"field":"max_tokens","runs":160},{"field":"prompt","runs":160},{"field":"prompt_digest","runs":160},{"field":"reader","runs":160},{"field":"store_digest","runs":160},{"field":"temperature","runs":160},{"field":"when","runs":151},{"field":"by","runs":128},{"field":"answered","runs":31},{"field":"questions","runs":31},{"field":"dataset_digest","runs":23},{"field":"commit","runs":9}]}