{"data":[{"project":"default","id":"gate.jev-drawn-a8.0cfeef05-kai","benchmark":"gate.jev","split":"drawn","system":"kai","version":"a8@0cfeef05c879d7f24bbde275a6c23e85e80b6baf06cf9311d529010688d4a697","baseline":false,"study":"","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"11474208ce49e6834bad21db4af8bb7ddf33b0ada6138b3661938522cafc815f","store_digest":"","facts_digest":"","commit":"a1a4a73d524cecf8038e30c408ad30b66e71d490","questions":null,"answered":null,"when":"","ended":"2026-09-28T16:33:11Z","by":"","completion":"unknown","measures":[{"category":"all","metric":"pass","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"suites","value":62,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"candidate","value":0.89321440208537,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"baseline","value":0.8669257086999022,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"delta","value":0.0262886933854678,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"regressions","value":14,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"cases","value":100,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/jev.ag_news","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/jev.ag_news","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/jev.ag_news","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/jev.ag_news","metric":"candidate_accuracy","value":0.96,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/jev.ag_news","metric":"candidate_ece","value":0.03911538153886794,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/jev.ag_news","metric":"baseline_accuracy","value":0.83,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/jev.ag_news","metric":"baseline_ece","value":0.12050000071525568,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/jev.emotion","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/jev.emotion","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/jev.emotion","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/jev.emotion","metric":"candidate_accuracy","value":0.95,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/jev.emotion","metric":"candidate_ece","value":0.042020905315876,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/jev.emotion","metric":"baseline_accuracy","value":0.64,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/jev.emotion","metric":"baseline_ece","value":0.24150000125169754,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/jev.banking77_full","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/jev.banking77_full","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/jev.banking77_full","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/jev.banking77_full","metric":"candidate_accuracy","value":0.89,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/jev.banking77_full","metric":"candidate_ece","value":0.09766138911247257,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/jev.banking77_full","metric":"baseline_accuracy","value":0.85,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/jev.banking77_full","metric":"baseline_ece","value":0.08564848810434346,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.support_triage","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.support_triage","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.support_triage","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.support_triage","metric":"candidate_accuracy","value":0.33,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.support_triage","metric":"candidate_ece","value":0.4952091199159622,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.support_triage","metric":"baseline_accuracy","value":0.29,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.support_triage","metric":"baseline_ece","value":0.561000002026558,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.email_spam","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.email_spam","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.email_spam","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.email_spam","metric":"candidate_accuracy","value":0.99,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.email_spam","metric":"candidate_ece","value":0.009769555330276458,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.email_spam","metric":"baseline_accuracy","value":0.96,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.email_spam","metric":"baseline_ece","value":0.07149999797344209,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.phishing","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.phishing","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.phishing","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.phishing","metric":"candidate_accuracy","value":1,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.phishing","metric":"candidate_ece","value":0.0000613886117935003,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.phishing","metric":"baseline_accuracy","value":0.89,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.phishing","metric":"baseline_ece","value":0.08049999296665189,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.guardrails_jailbreak","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.guardrails_jailbreak","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.guardrails_jailbreak","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.guardrails_jailbreak","metric":"candidate_accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.guardrails_jailbreak","metric":"candidate_ece","value":0.05118291318416589,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.guardrails_jailbreak","metric":"baseline_accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.guardrails_jailbreak","metric":"baseline_ece","value":0.0655999952554703,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.moderation_toxicity","metric":"regressed","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.moderation_toxicity","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.moderation_toxicity","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.moderation_toxicity","metric":"candidate_accuracy","value":0.81,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.moderation_toxicity","metric":"candidate_ece","value":0.16926764965057373,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.moderation_toxicity","metric":"baseline_accuracy","value":0.73,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.moderation_toxicity","metric":"baseline_ece","value":0.1372000062465668,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.rag_relevance","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.rag_relevance","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.rag_relevance","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.rag_relevance","metric":"candidate_accuracy","value":0.57,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.rag_relevance","metric":"candidate_ece","value":0.12333745658397677,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.rag_relevance","metric":"baseline_accuracy","value":0.51,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.rag_relevance","metric":"baseline_ece","value":0.39620000362396246,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.model_routing_domain","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.model_routing_domain","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.model_routing_domain","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/app.model_routing_domain","metric":"candidate_accuracy","value":1,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.model_routing_domain","metric":"candidate_ece","value":0.003844951391220098,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.model_routing_domain","metric":"baseline_accuracy","value":0.98,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/app.model_routing_domain","metric":"baseline_ece","value":0.026600003242492693,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/typed_decisions","metric":"regressed","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/typed_decisions","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/typed_decisions","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/typed_decisions","metric":"candidate_accuracy","value":0.75,"lo":null,"hi":null,"n":500,"of":null},{"category":"harness/typed_decisions","metric":"candidate_ece","value":0.1543838354349136,"lo":null,"hi":null,"n":500,"of":null},{"category":"harness/typed_decisions","metric":"baseline_accuracy","value":0.73,"lo":null,"hi":null,"n":500,"of":null},{"category":"harness/typed_decisions","metric":"baseline_ece","value":0.07618141478300096,"lo":null,"hi":null,"n":500,"of":null},{"category":"harness/massive.af","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.af","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.af","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.af","metric":"candidate_accuracy","value":0.89,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.af","metric":"candidate_ece","value":0.05268498986959454,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.af","metric":"baseline_accuracy","value":0.87,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.af","metric":"baseline_ece","value":0.08726868644356728,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.am","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.am","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.am","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.am","metric":"candidate_accuracy","value":0.85,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.am","metric":"candidate_ece","value":0.11598066598176962,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.am","metric":"baseline_accuracy","value":0.85,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.am","metric":"baseline_ece","value":0.09598384097218512,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ar","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ar","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ar","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ar","metric":"candidate_accuracy","value":0.89,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ar","metric":"candidate_ece","value":0.07288791090250017,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ar","metric":"baseline_accuracy","value":0.87,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ar","metric":"baseline_ece","value":0.08222424164414406,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.az","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.az","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.az","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.az","metric":"candidate_accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.az","metric":"candidate_ece","value":0.06024763107299812,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.az","metric":"baseline_accuracy","value":0.88,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.az","metric":"baseline_ece","value":0.04820606261491777,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.bn","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.bn","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.bn","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.bn","metric":"candidate_accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.bn","metric":"candidate_ece","value":0.05731592342257498,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.bn","metric":"baseline_accuracy","value":0.86,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.bn","metric":"baseline_ece","value":0.0710626265406609,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.cy","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.cy","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.cy","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.cy","metric":"candidate_accuracy","value":0.87,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.cy","metric":"candidate_ece","value":0.08664740681648261,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.cy","metric":"baseline_accuracy","value":0.76,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.cy","metric":"baseline_ece","value":0.09916767939925195,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.da","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.da","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.da","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.da","metric":"candidate_accuracy","value":0.91,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.da","metric":"candidate_ece","value":0.053155619204044345,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.da","metric":"baseline_accuracy","value":0.88,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.da","metric":"baseline_ece","value":0.038734343349933635,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.de","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.de","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.de","metric":"unanswered","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.de","metric":"candidate_accuracy","value":0.9292929292929293,"lo":null,"hi":null,"n":99,"of":null},{"category":"harness/massive.de","metric":"candidate_ece","value":0.05187850498189824,"lo":null,"hi":null,"n":99,"of":null},{"category":"harness/massive.de","metric":"baseline_accuracy","value":0.9393939393939394,"lo":null,"hi":null,"n":99,"of":null},{"category":"harness/massive.de","metric":"baseline_ece","value":0.057024792589322484,"lo":null,"hi":null,"n":99,"of":null},{"category":"harness/massive.el","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.el","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.el","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.el","metric":"candidate_accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.el","metric":"candidate_ece","value":0.0822948729991913,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.el","metric":"baseline_accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.el","metric":"baseline_ece","value":0.057093940079212174,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.en","metric":"regressed","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.en","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.en","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.en","metric":"candidate_accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.en","metric":"candidate_ece","value":0.05915291845798494,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.en","metric":"baseline_accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.en","metric":"baseline_ece","value":0.02530000150203709,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.es","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.es","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.es","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.es","metric":"candidate_accuracy","value":0.91,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.es","metric":"candidate_ece","value":0.06837773680686952,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.es","metric":"baseline_accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.es","metric":"baseline_ece","value":0.06282828330993653,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.fa","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.fa","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.fa","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.fa","metric":"candidate_accuracy","value":0.95,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.fa","metric":"candidate_ece","value":0.04708155661821374,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.fa","metric":"baseline_accuracy","value":0.95,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.fa","metric":"baseline_ece","value":0.04546464592218401,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.fi","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.fi","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.fi","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.fi","metric":"candidate_accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.fi","metric":"candidate_ece","value":0.056733679771423406,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.fi","metric":"baseline_accuracy","value":0.89,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.fi","metric":"baseline_ece","value":0.0383111131191253,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.fr","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.fr","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.fr","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.fr","metric":"candidate_accuracy","value":0.9,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.fr","metric":"candidate_ece","value":0.07309922486543659,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.fr","metric":"baseline_accuracy","value":0.9,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.fr","metric":"baseline_ece","value":0.0651949521899224,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.he","metric":"regressed","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.he","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.he","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.he","metric":"candidate_accuracy","value":0.91,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.he","metric":"candidate_ece","value":0.07781283855438227,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.he","metric":"baseline_accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.he","metric":"baseline_ece","value":0.042724243700504254,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.hi","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.hi","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.hi","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.hi","metric":"candidate_accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.hi","metric":"candidate_ece","value":0.043011419475078645,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.hi","metric":"baseline_accuracy","value":0.95,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.hi","metric":"baseline_ece","value":0.029023234546184544,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.hu","metric":"regressed","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.hu","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.hu","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.hu","metric":"candidate_accuracy","value":0.91,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.hu","metric":"candidate_ece","value":0.08298780947923662,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.hu","metric":"baseline_accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.hu","metric":"baseline_ece","value":0.03867272794246674,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.hy","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.hy","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.hy","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.hy","metric":"candidate_accuracy","value":0.89,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.hy","metric":"candidate_ece","value":0.05867982119321828,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.hy","metric":"baseline_accuracy","value":0.84,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.hy","metric":"baseline_ece","value":0.07812121435999876,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.id","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.id","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.id","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.id","metric":"candidate_accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.id","metric":"candidate_ece","value":0.04446434766054152,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.id","metric":"baseline_accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.id","metric":"baseline_ece","value":0.05200909078121181,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.is","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.is","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.is","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.is","metric":"candidate_accuracy","value":0.88,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.is","metric":"candidate_ece","value":0.08488803684711461,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.is","metric":"baseline_accuracy","value":0.84,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.is","metric":"baseline_ece","value":0.07296262651681899,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.it","metric":"regressed","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.it","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.it","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.it","metric":"candidate_accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.it","metric":"candidate_ece","value":0.05702402949333189,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.it","metric":"baseline_accuracy","value":0.96,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.it","metric":"baseline_ece","value":0.05021818310022354,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ja","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ja","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ja","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ja","metric":"candidate_accuracy","value":0.96,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ja","metric":"candidate_ece","value":0.034759365916252095,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ja","metric":"baseline_accuracy","value":0.97,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ja","metric":"baseline_ece","value":0.05874444544315334,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.jv","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.jv","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.jv","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.jv","metric":"candidate_accuracy","value":0.88,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.jv","metric":"candidate_ece","value":0.06856640487909312,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.jv","metric":"baseline_accuracy","value":0.75,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.jv","metric":"baseline_ece","value":0.07757171720266345,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ka","metric":"regressed","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ka","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ka","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ka","metric":"candidate_accuracy","value":0.8,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ka","metric":"candidate_ece","value":0.1308203849196434,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ka","metric":"baseline_accuracy","value":0.88,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ka","metric":"baseline_ece","value":0.09805454522371292,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.km","metric":"regressed","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.km","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.km","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.km","metric":"candidate_accuracy","value":0.82,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.km","metric":"candidate_ece","value":0.14559533908963204,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.km","metric":"baseline_accuracy","value":0.81,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.km","metric":"baseline_ece","value":0.07332727491855619,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.kn","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.kn","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.kn","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.kn","metric":"candidate_accuracy","value":0.9,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.kn","metric":"candidate_ece","value":0.07564685180783273,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.kn","metric":"baseline_accuracy","value":0.87,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.kn","metric":"baseline_ece","value":0.062040405273437446,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ko","metric":"regressed","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ko","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ko","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ko","metric":"candidate_accuracy","value":0.9,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ko","metric":"candidate_ece","value":0.07660935476422309,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ko","metric":"baseline_accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ko","metric":"baseline_ece","value":0.04226868838071823,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.lv","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.lv","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.lv","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.lv","metric":"candidate_accuracy","value":0.89,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.lv","metric":"candidate_ece","value":0.09463837064802645,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.lv","metric":"baseline_accuracy","value":0.85,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.lv","metric":"baseline_ece","value":0.06903636515140534,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ml","metric":"regressed","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ml","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ml","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ml","metric":"candidate_accuracy","value":0.88,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ml","metric":"candidate_ece","value":0.07211435198783879,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ml","metric":"baseline_accuracy","value":0.91,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ml","metric":"baseline_ece","value":0.049515153467655115,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.mn","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.mn","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.mn","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.mn","metric":"candidate_accuracy","value":0.88,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.mn","metric":"candidate_ece","value":0.09503076672554014,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.mn","metric":"baseline_accuracy","value":0.84,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.mn","metric":"baseline_ece","value":0.08651313126087185,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ms","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ms","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ms","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ms","metric":"candidate_accuracy","value":0.9,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ms","metric":"candidate_ece","value":0.07233261793851858,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ms","metric":"baseline_accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ms","metric":"baseline_ece","value":0.06841818481683726,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.my","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.my","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.my","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.my","metric":"candidate_accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.my","metric":"candidate_ece","value":0.059497409313917074,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.my","metric":"baseline_accuracy","value":0.89,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.my","metric":"baseline_ece","value":0.07051919519901269,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.nb","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.nb","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.nb","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.nb","metric":"candidate_accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.nb","metric":"candidate_ece","value":0.05309729158878326,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.nb","metric":"baseline_accuracy","value":0.86,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.nb","metric":"baseline_ece","value":0.06732828453183172,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.nl","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.nl","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.nl","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.nl","metric":"candidate_accuracy","value":0.88,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.nl","metric":"candidate_ece","value":0.08047885030508038,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.nl","metric":"baseline_accuracy","value":0.9,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.nl","metric":"baseline_ece","value":0.055070708692073866,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.pl","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.pl","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.pl","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.pl","metric":"candidate_accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.pl","metric":"candidate_ece","value":0.04532576993107795,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.pl","metric":"baseline_accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.pl","metric":"baseline_ece","value":0.04303939431905744,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.pt","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.pt","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.pt","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.pt","metric":"candidate_accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.pt","metric":"candidate_ece","value":0.04678081497550015,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.pt","metric":"baseline_accuracy","value":0.9,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.pt","metric":"baseline_ece","value":0.09853636592626569,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ro","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ro","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ro","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ro","metric":"candidate_accuracy","value":0.96,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ro","metric":"candidate_ece","value":0.03957485139369964,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ro","metric":"baseline_accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ro","metric":"baseline_ece","value":0.08220606148242947,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ru","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ru","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ru","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ru","metric":"candidate_accuracy","value":0.95,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ru","metric":"candidate_ece","value":0.04580723613500593,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ru","metric":"baseline_accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ru","metric":"baseline_ece","value":0.0405292943120003,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sl","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.sl","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.sl","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.sl","metric":"candidate_accuracy","value":0.88,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sl","metric":"candidate_ece","value":0.06133252531290058,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sl","metric":"baseline_accuracy","value":0.89,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sl","metric":"baseline_ece","value":0.057861616015434277,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sq","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.sq","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.sq","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.sq","metric":"candidate_accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sq","metric":"candidate_ece","value":0.054145977795124055,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sq","metric":"baseline_accuracy","value":0.76,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sq","metric":"baseline_ece","value":0.0900494961440563,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sv","metric":"regressed","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.sv","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.sv","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.sv","metric":"candidate_accuracy","value":0.91,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sv","metric":"candidate_ece","value":0.06203487917780875,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sv","metric":"baseline_accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sv","metric":"baseline_ece","value":0.07373636573553077,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sw","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.sw","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.sw","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.sw","metric":"candidate_accuracy","value":0.85,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sw","metric":"candidate_ece","value":0.0911237014085054,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sw","metric":"baseline_accuracy","value":0.7,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.sw","metric":"baseline_ece","value":0.11177171409130097,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ta","metric":"regressed","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ta","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ta","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ta","metric":"candidate_accuracy","value":0.89,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ta","metric":"candidate_ece","value":0.06441951379179953,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ta","metric":"baseline_accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ta","metric":"baseline_ece","value":0.09244343683123586,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.te","metric":"regressed","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.te","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.te","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.te","metric":"candidate_accuracy","value":0.87,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.te","metric":"candidate_ece","value":0.08435814142227176,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.te","metric":"baseline_accuracy","value":0.91,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.te","metric":"baseline_ece","value":0.04149899095296861,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.th","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.th","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.th","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.th","metric":"candidate_accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.th","metric":"candidate_ece","value":0.044177895784378036,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.th","metric":"baseline_accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.th","metric":"baseline_ece","value":0.04299999922513963,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.tl","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.tl","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.tl","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.tl","metric":"candidate_accuracy","value":0.88,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.tl","metric":"candidate_ece","value":0.08951716601848603,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.tl","metric":"baseline_accuracy","value":0.85,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.tl","metric":"baseline_ece","value":0.08445252478122714,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.tr","metric":"regressed","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.tr","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.tr","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.tr","metric":"candidate_accuracy","value":0.87,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.tr","metric":"candidate_ece","value":0.10064284861087798,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.tr","metric":"baseline_accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.tr","metric":"baseline_ece","value":0.0609899020195007,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ur","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ur","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ur","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.ur","metric":"candidate_accuracy","value":0.91,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ur","metric":"candidate_ece","value":0.06861960574984545,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ur","metric":"baseline_accuracy","value":0.9,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.ur","metric":"baseline_ece","value":0.06538687020540238,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.vi","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.vi","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.vi","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.vi","metric":"candidate_accuracy","value":0.92,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.vi","metric":"candidate_ece","value":0.04812568008899687,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.vi","metric":"baseline_accuracy","value":0.89,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.vi","metric":"baseline_ece","value":0.04620000123977666,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.zh-CN","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.zh-CN","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.zh-CN","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.zh-CN","metric":"candidate_accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.zh-CN","metric":"candidate_ece","value":0.05050742074847214,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.zh-CN","metric":"baseline_accuracy","value":0.94,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.zh-CN","metric":"baseline_ece","value":0.03629999935626985,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.zh-TW","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.zh-TW","metric":"gated","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.zh-TW","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"harness/massive.zh-TW","metric":"candidate_accuracy","value":0.96,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.zh-TW","metric":"candidate_ece","value":0.03419531375169753,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.zh-TW","metric":"baseline_accuracy","value":0.93,"lo":null,"hi":null,"n":100,"of":null},{"category":"harness/massive.zh-TW","metric":"baseline_ece","value":0.06207878708839416,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/clinc150","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/clinc150","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/clinc150","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/clinc150","metric":"candidate_accuracy","value":0.78,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/clinc150","metric":"candidate_ece","value":0.15651181817054743,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/sst5","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/sst5","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/sst5","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/sst5","metric":"candidate_accuracy","value":0.5,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/sst5","metric":"candidate_ece","value":0.1442073956131935,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.ar","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.ar","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.ar","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.ar","metric":"candidate_accuracy","value":0.62,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.ar","metric":"candidate_ece","value":0.13826530635356904,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.bg","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.bg","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.bg","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.bg","metric":"candidate_accuracy","value":0.76,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.bg","metric":"candidate_ece","value":0.12432977795600889,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.de","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.de","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.de","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.de","metric":"candidate_accuracy","value":0.76,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.de","metric":"candidate_ece","value":0.12723101794719702,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.el","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.el","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.el","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.el","metric":"candidate_accuracy","value":0.73,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.el","metric":"candidate_ece","value":0.11346075385808947,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.en","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.en","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.en","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.en","metric":"candidate_accuracy","value":0.76,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.en","metric":"candidate_ece","value":0.12963819324970244,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.es","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.es","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.es","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.es","metric":"candidate_accuracy","value":0.77,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.es","metric":"candidate_ece","value":0.15921563208103184,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.fr","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.fr","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.fr","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.fr","metric":"candidate_accuracy","value":0.79,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.fr","metric":"candidate_ece","value":0.14418751090764997,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.hi","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.hi","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.hi","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.hi","metric":"candidate_accuracy","value":0.59,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.hi","metric":"candidate_ece","value":0.05843946486711503,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.ru","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.ru","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.ru","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.ru","metric":"candidate_accuracy","value":0.72,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.ru","metric":"candidate_ece","value":0.11137971103191374,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.sw","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.sw","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.sw","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.sw","metric":"candidate_accuracy","value":0.58,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.sw","metric":"candidate_ece","value":0.12170465260744094,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.th","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.th","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.th","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.th","metric":"candidate_accuracy","value":0.61,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.th","metric":"candidate_ece","value":0.1322586926817894,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.tr","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.tr","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.tr","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.tr","metric":"candidate_accuracy","value":0.72,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.tr","metric":"candidate_ece","value":0.12287913411855698,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.ur","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.ur","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.ur","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.ur","metric":"candidate_accuracy","value":0.5,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.ur","metric":"candidate_ece","value":0.1103538781404495,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.vi","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.vi","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.vi","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.vi","metric":"candidate_accuracy","value":0.71,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.vi","metric":"candidate_ece","value":0.11963065892457966,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.zh","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.zh","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.zh","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"capability/xnli.zh","metric":"candidate_accuracy","value":0.67,"lo":null,"hi":null,"n":100,"of":null},{"category":"capability/xnli.zh","metric":"candidate_ece","value":0.10528100371360781,"lo":null,"hi":null,"n":100,"of":null},{"category":"robustness/shuffle","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/shuffle","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/shuffle","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/shuffle","metric":"candidate_accuracy","value":0.8932730749174344,"lo":null,"hi":null,"n":5753,"of":null},{"category":"robustness/shuffle","metric":"candidate_ece","value":0.04835260770055868,"lo":null,"hi":null,"n":5753,"of":null},{"category":"robustness/alias","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/alias","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/alias","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/alias","metric":"candidate_accuracy","value":0.848531684698609,"lo":null,"hi":null,"n":647,"of":null},{"category":"robustness/alias","metric":"candidate_ece","value":0.04920363361721247,"lo":null,"hi":null,"n":647,"of":null},{"category":"robustness/swap","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/swap","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/swap","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/swap","metric":"candidate_accuracy","value":0.8500772797527048,"lo":null,"hi":null,"n":647,"of":null},{"category":"robustness/swap","metric":"candidate_ece","value":0.05356464936891703,"lo":null,"hi":null,"n":647,"of":null},{"category":"robustness/subset","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/subset","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/subset","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/subset","metric":"candidate_accuracy","value":0.9507547169811321,"lo":null,"hi":null,"n":5300,"of":null},{"category":"robustness/subset","metric":"candidate_ece","value":0.018969453463576848,"lo":null,"hi":null,"n":5300,"of":null},{"category":"robustness/near","metric":"regressed","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/near","metric":"gated","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/near","metric":"unanswered","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"robustness/near","metric":"candidate_accuracy","value":0.9222641509433962,"lo":null,"hi":null,"n":5300,"of":null},{"category":"robustness/near","metric":"candidate_ece","value":0.029289239268257973,"lo":null,"hi":null,"n":5300,"of":null}],"notes":"tolerance accuracy 0.02, ece 0.03; gate sha256:866b219e0c9e383173d0a82a031b669933c58dc69183ca974987aea7f7be9f3e","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":false}],"total":1,"completion":{"complete":0,"partial":0,"inconsistent":0,"unknown":1},"gaps":[{"field":"answered","runs":1},{"field":"by","runs":1},{"field":"embedder","runs":1},{"field":"facts_digest","runs":1},{"field":"k","runs":1},{"field":"max_tokens","runs":1},{"field":"prompt","runs":1},{"field":"prompt_digest","runs":1},{"field":"questions","runs":1},{"field":"reader","runs":1},{"field":"store_digest","runs":1},{"field":"temperature","runs":1},{"field":"when","runs":1}]}