{"object":"benchmark.report","report_date":"July 2026","evaluation_date":"2026-07-11","api_version":"v1","generated_at":"2026-07-13T01:22:22.276Z","panels":[{"id":"emotion","title":"Seven-class emotion, 32 systems","metric":"Accuracy","unit":"%","direction":"higher","sample":"speech-emotion-bench","confidence_interval":null,"confidence_interval_note":"Not reported in this release because per-example bootstrap artifacts are not available for every system.","results":[{"model":"oruk Spectra","model_id":"oruk-spectra-1","model_version":"Measured on the full 64,384-clip set, 2026-07-11","value":77.6,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"emotion2vec+ seed","model_id":null,"model_version":"Measured on the full 64,384-clip set, 2026-07-11","value":68.7,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"emotion2vec+ large","model_id":null,"model_version":"Measured on the full 64,384-clip set, 2026-07-11","value":68.6,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"emotion2vec+ base","model_id":null,"model_version":"Measured on the full 64,384-clip set, 2026-07-11","value":68.5,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"emotion2vec finetuned","model_id":null,"model_version":"Measured on the full 64,384-clip set, 2026-07-11","value":63.6,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"EmotionThinker","model_id":null,"model_version":"Measured on the fixed 5,000-clip stratified subsample, 2026-07-11","value":60.5,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"SenseVoice Small","model_id":null,"model_version":"Measured on the full 64,384-clip set, 2026-07-11","value":55.7,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"Qwen2.5-Omni 7B","model_id":null,"model_version":"Measured on the fixed 5,000-clip stratified subsample, 2026-07-11","value":51.4,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"Gemini 3 Flash Preview","model_id":null,"model_version":"Measured on the fixed 5,000-clip stratified subsample, 2026-07-11","value":46,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"Gemini 2.5 Flash","model_id":null,"model_version":"Measured on the fixed 5,000-clip stratified subsample, 2026-07-11","value":45.4,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"WavLM Vox-Profile","model_id":null,"model_version":"Measured on the full 64,384-clip set, 2026-07-11","value":45.3,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"WavLM Odyssey MSP","model_id":null,"model_version":"Measured on the full 64,384-clip set, 2026-07-11","value":44.6,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"XLS-R SER (hughlan1214)","model_id":null,"model_version":"Measured on the full 64,384-clip set, 2026-07-11","value":44.4,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"Gemini 2.5 Pro","model_id":null,"model_version":"Measured on the fixed 5,000-clip stratified subsample, 2026-07-11","value":44,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"GPT-Audio 1.5","model_id":null,"model_version":"Measured on the fixed 5,000-clip stratified subsample, 2026-07-11","value":43.3,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"HuBERT-large SUPERB","model_id":null,"model_version":"Measured on the full 64,384-clip set, 2026-07-11","value":40.9,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"MERaLiON-2 10B","model_id":null,"model_version":"Measured on the fixed 5,000-clip stratified subsample, 2026-07-11","value":40.9,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"Claude Opus 4.8 (transcript)","model_id":null,"model_version":"Measured on the fixed 5,000-clip stratified subsample, 2026-07-11","value":40.3,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"XLS-R SER (Hatman)","model_id":null,"model_version":"Measured on the full 64,384-clip set, 2026-07-11","value":40.2,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"Gemini 2.5 Flash-Lite","model_id":null,"model_version":"Measured on the fixed 5,000-clip stratified subsample, 2026-07-11","value":40,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"GPT-Audio mini","model_id":null,"model_version":"Measured on the fixed 5,000-clip stratified subsample, 2026-07-11","value":39.7,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"Claude Sonnet 4.6 (transcript)","model_id":null,"model_version":"Measured on the fixed 5,000-clip stratified subsample, 2026-07-11","value":39.4,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"Claude Haiku 4.5 (transcript)","model_id":null,"model_version":"Measured on the fixed 5,000-clip stratified subsample, 2026-07-11","value":39.3,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"SpeechBrain IEMOCAP","model_id":null,"model_version":"Measured on the full 64,384-clip set, 2026-07-11","value":39.2,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"wav2vec2 SER (Dpngtm)","model_id":null,"model_version":"Measured on the full 64,384-clip set, 2026-07-11","value":38.2,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"Voxtral-Mini 3B","model_id":null,"model_version":"Measured on the fixed 5,000-clip stratified subsample, 2026-07-11","value":36.6,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"Whisper SER (firdhokk)","model_id":null,"model_version":"Measured on the full 64,384-clip set, 2026-07-11","value":32.5,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"wav2vec2-base SUPERB","model_id":null,"model_version":"Measured on the full 64,384-clip set, 2026-07-11","value":31.4,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"XLS-R SER (firdhokk)","model_id":null,"model_version":"Measured on the full 64,384-clip set, 2026-07-11","value":26,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"XLS-R RAVDESS","model_id":null,"model_version":"Measured on the full 64,384-clip set, 2026-07-11","value":23.8,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"wav2vec2 CTC SER (r-f)","model_id":null,"model_version":"Measured on the full 64,384-clip set, 2026-07-11","value":16,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"XLS-R Russian (Aniemore)","model_id":null,"model_version":"Measured on the full 64,384-clip set, 2026-07-11","value":9.1,"measured_in_oruk_harness":true,"vendor_published_reference":false}]},{"id":"clean-asr","title":"English public four-set average","metric":"WER","unit":"%","direction":"lower","sample":"LibriSpeech clean/other · Common Voice · TED-LIUM","confidence_interval":null,"confidence_interval_note":"Not reported in this release because per-example bootstrap artifacts are not available for every system.","results":[{"model":"oruk Spectra","model_id":"oruk-spectra-1","model_version":"2026-07 evaluation release","value":4.44,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"Open ASR A","model_id":null,"model_version":"Internal checkpoint registry, pinned 2026-07-11","value":4.47,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"Open ASR B","model_id":null,"model_version":"Internal checkpoint registry, pinned 2026-07-11","value":6.53,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"Picovoice Leopard","model_id":null,"model_version":"Vendor-published value accessed 2026-07-11","value":9.7,"measured_in_oruk_harness":false,"vendor_published_reference":true},{"model":"Picovoice Cheetah","model_id":null,"model_version":"Vendor-published value accessed 2026-07-11","value":10.1,"measured_in_oruk_harness":false,"vendor_published_reference":true}]},{"id":"device-asr","title":"Real wearable recordings","metric":"WER","unit":"%","direction":"lower","sample":"oruk pendant recordings","confidence_interval":null,"confidence_interval_note":"Not reported in this release because per-example bootstrap artifacts are not available for every system.","results":[{"model":"oruk Spectra 2","model_id":"oruk-spectra-2","model_version":"2026-07 evaluation release","value":12.4,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"Open ASR A","model_id":null,"model_version":"Internal checkpoint registry, pinned 2026-07-11","value":12.7,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"Open ASR B","model_id":null,"model_version":"Internal checkpoint registry, pinned 2026-07-11","value":13.4,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"Open ASR C","model_id":null,"model_version":"Internal checkpoint registry, pinned 2026-07-11","value":19.86,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"Open ASR D","model_id":null,"model_version":"Internal checkpoint registry, pinned 2026-07-11","value":31,"measured_in_oruk_harness":true,"vendor_published_reference":false}]}],"open_emotion_benchmark":{"name":"speech-emotion-bench","clips":64384,"emotionClasses":["anger","happiness","sadness","fear","disgust","surprise","neutral"],"languages":20,"systems":32,"subsampleClips":5000,"description":"A held-out speech emotion recognition benchmark: 64,384 clips across ~20 languages and 7 emotion classes, scored with one identical pipeline for 32 systems. Closed/API and audio-LLM models are scored on a fixed 5,000-clip stratified subsample; rescoring open models on the same subsample shifts results by less than two points.","evaluation_date":"2026-07-11","dataset_manifest":"/benchmarks/emotion-eval-manifest.json","fairness_note":"The oruk entry is trained in-distribution; other listed systems are evaluated zero-shot across corpora.","results":[{"name":"oruk Spectra","short":"Spectra","tag":"ours","acc":77.6,"mf1":0.81,"ours":true,"model_id":"oruk-spectra-1","model_version":"2026-07 evaluation release"},{"name":"emotion2vec+ seed","short":"e2v+ seed","tag":"open","acc":68.7,"mf1":0.68,"model_id":null,"model_version":"emotion2vec+ seed"},{"name":"emotion2vec+ large","short":"e2v+ large","tag":"open","acc":68.6,"mf1":0.677,"model_id":null,"model_version":"emotion2vec+ large"},{"name":"emotion2vec+ base","short":"e2v+ base","tag":"open","acc":68.5,"mf1":0.683,"model_id":null,"model_version":"emotion2vec+ base"},{"name":"emotion2vec finetuned","short":"e2v FT","tag":"open","acc":63.6,"mf1":0.616,"model_id":null,"model_version":"emotion2vec finetuned"},{"name":"EmotionThinker","short":"EmoThinker","tag":"audio LLM","acc":60.5,"mf1":0.504,"subsample":true,"model_id":null,"model_version":"EmotionThinker"},{"name":"SenseVoice Small","short":"SenseVoice","tag":"open","acc":55.7,"mf1":0.469,"model_id":null,"model_version":"SenseVoice Small"},{"name":"Qwen2.5-Omni 7B","short":"Qwen Omni","tag":"audio LLM","acc":51.4,"mf1":0.44,"subsample":true,"model_id":null,"model_version":"Qwen2.5-Omni 7B"},{"name":"Gemini 3 Flash Preview","short":"Gemini 3 Fl","tag":"API","acc":46,"mf1":0.373,"subsample":true,"model_id":null,"model_version":"Gemini 3 Flash Preview"},{"name":"Gemini 2.5 Flash","short":"Gem 2.5 Fl","tag":"API","acc":45.4,"mf1":0.37,"subsample":true,"model_id":null,"model_version":"Gemini 2.5 Flash"},{"name":"WavLM Vox-Profile","short":"WavLM VP","tag":"open","acc":45.3,"mf1":0.355,"model_id":null,"model_version":"WavLM Vox-Profile"},{"name":"WavLM Odyssey MSP","short":"WavLM Ody","tag":"open","acc":44.6,"mf1":0.369,"model_id":null,"model_version":"WavLM Odyssey MSP"},{"name":"XLS-R SER (hughlan1214)","short":"XLS-R H14","tag":"open","acc":44.4,"mf1":0.411,"model_id":null,"model_version":"XLS-R SER (hughlan1214)"},{"name":"Gemini 2.5 Pro","short":"Gem 2.5 Pro","tag":"API","acc":44,"mf1":0.36,"subsample":true,"model_id":null,"model_version":"Gemini 2.5 Pro"},{"name":"GPT-Audio 1.5","short":"GPT-Audio","tag":"API","acc":43.3,"mf1":0.347,"subsample":true,"model_id":null,"model_version":"GPT-Audio 1.5"},{"name":"HuBERT-large SUPERB","short":"HuBERT-L","tag":"open","acc":40.9,"mf1":0.249,"model_id":null,"model_version":"HuBERT-large SUPERB"},{"name":"MERaLiON-2 10B","short":"MERaLiON","tag":"audio LLM","acc":40.9,"mf1":0.315,"subsample":true,"model_id":null,"model_version":"MERaLiON-2 10B"},{"name":"Claude Opus 4.8 (transcript)","short":"Opus 4.8","tag":"text only","acc":40.3,"mf1":0.298,"subsample":true,"model_id":null,"model_version":"Claude Opus 4.8 (transcript)"},{"name":"XLS-R SER (Hatman)","short":"XLS-R Hat","tag":"open","acc":40.2,"mf1":0.361,"model_id":null,"model_version":"XLS-R SER (Hatman)"},{"name":"Gemini 2.5 Flash-Lite","short":"Gem Fl-Lite","tag":"API","acc":40,"mf1":0.286,"subsample":true,"model_id":null,"model_version":"Gemini 2.5 Flash-Lite"},{"name":"GPT-Audio mini","short":"GPT-A mini","tag":"API","acc":39.7,"mf1":0.3,"subsample":true,"model_id":null,"model_version":"GPT-Audio mini"},{"name":"Claude Sonnet 4.6 (transcript)","short":"Sonnet 4.6","tag":"text only","acc":39.4,"mf1":0.286,"subsample":true,"model_id":null,"model_version":"Claude Sonnet 4.6 (transcript)"},{"name":"Claude Haiku 4.5 (transcript)","short":"Haiku 4.5","tag":"text only","acc":39.3,"mf1":0.293,"subsample":true,"model_id":null,"model_version":"Claude Haiku 4.5 (transcript)"},{"name":"SpeechBrain IEMOCAP","short":"SpeechBrain","tag":"open","acc":39.2,"mf1":0.211,"model_id":null,"model_version":"SpeechBrain IEMOCAP"},{"name":"wav2vec2 SER (Dpngtm)","short":"w2v2 Dpn","tag":"open","acc":38.2,"mf1":0.376,"model_id":null,"model_version":"wav2vec2 SER (Dpngtm)"},{"name":"Voxtral-Mini 3B","short":"Voxtral","tag":"audio LLM","acc":36.6,"mf1":0.204,"subsample":true,"model_id":null,"model_version":"Voxtral-Mini 3B"},{"name":"Whisper SER (firdhokk)","short":"Whisper SER","tag":"open","acc":32.5,"mf1":0.276,"model_id":null,"model_version":"Whisper SER (firdhokk)"},{"name":"wav2vec2-base SUPERB","short":"w2v2-base","tag":"open","acc":31.4,"mf1":0.187,"model_id":null,"model_version":"wav2vec2-base SUPERB"},{"name":"XLS-R SER (firdhokk)","short":"XLS-R Fir","tag":"open","acc":26,"mf1":0.24,"model_id":null,"model_version":"XLS-R SER (firdhokk)"},{"name":"XLS-R RAVDESS","short":"XLS-R RAV","tag":"open","acc":23.8,"mf1":0.118,"model_id":null,"model_version":"XLS-R RAVDESS"},{"name":"wav2vec2 CTC SER (r-f)","short":"w2v2 CTC","tag":"open","acc":16,"mf1":0.098,"model_id":null,"model_version":"wav2vec2 CTC SER (r-f)"},{"name":"XLS-R Russian (Aniemore)","short":"XLS-R Ru","tag":"open","acc":9.1,"mf1":0.079,"model_id":null,"model_version":"XLS-R Russian (Aniemore)"}]},"limitations":["The wearable panel contains 22 recordings and should be treated as a focused product-audio evaluation, not a population estimate.","Vendor-published reference rows were not rerun in the oruk harness.","Raw audio is not redistributed where source licenses do not permit redistribution."]}