{"object":"benchmark.report","report_date":"August 2026","evaluation_date":"2026-07-11","api_version":"v1","generated_at":"2026-07-11T00:00:00Z","panels":[{"id":"emotion","title":"Seven-class emotion, 64 systems","metric":"Accuracy","unit":"%","direction":"higher","sample":"speech-emotion-bench: open models on the full 64,384 clips; closed/API and audio-LLM systems on a fixed 5,000-clip stratified subset","confidence_interval":null,"confidence_interval_note":"Not reported in this release because per-example bootstrap artifacts are not available for every system.","results":[{"model":"oruk Spectra 1","model_id":"oruk-spectra-1","model_version":"Measured on the full 64,384-clip set, 2026-07-11","value":77.8,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"emotion2vec+ seed","model_id":null,"model_version":"Measured on the full 64,384-clip set, 2026-07-11","value":68.7,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"emotion2vec+ large","model_id":null,"model_version":"Measured on the full 64,384-clip set, 2026-07-11","value":68.6,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"emotion2vec+ base","model_id":null,"model_version":"Measured on the full 64,384-clip set, 2026-07-11","value":68.5,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"emotion2vec finetuned","model_id":null,"model_version":"Measured on the full 64,384-clip set, 2026-07-11","value":63.6,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"EmotionThinker","model_id":null,"model_version":"Measured on the fixed 5,000-clip stratified subsample, 2026-07-11","value":60.5,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"SenseVoice Small","model_id":null,"model_version":"Measured on the full 64,384-clip set, 2026-07-11","value":55.7,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"emotion2vec base (probe)","model_id":null,"model_version":"Measured on the full 64,384-clip set, 2026-07-11","value":55.1,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"Kimi-Audio 7B Instruct","model_id":null,"model_version":"Measured on the fixed 5,000-clip stratified subsample, 2026-07-11","value":52.4,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"Qwen2.5-Omni 7B","model_id":null,"model_version":"Measured on the fixed 5,000-clip stratified subsample, 2026-07-11","value":51.4,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"Hume AI legacy prosody","model_id":null,"model_version":"Dated legacy 48-dimension Hume Expression Measurement prosody snapshot scored on the fixed 5,000-clip subset in the 2026-07-11 release. This is not a measurement of Hume's current Tagger or real-time Prosody products.","value":49.6,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"Qwen2-Audio 7B Instruct","model_id":null,"model_version":"Measured on the fixed 5,000-clip stratified subsample, 2026-07-11","value":48.9,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"Step-Audio 2 mini","model_id":null,"model_version":"Measured on the fixed 5,000-clip stratified subsample, 2026-07-11","value":46.5,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"Gemini 3 Flash Preview","model_id":null,"model_version":"Measured on the fixed 5,000-clip stratified subsample, 2026-07-11","value":46,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"Gemini 2.5 Flash","model_id":null,"model_version":"Measured on the fixed 5,000-clip stratified subsample, 2026-07-11","value":45.4,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"WavLM Vox-Profile","model_id":null,"model_version":"Measured on the full 64,384-clip set, 2026-07-11","value":45.3,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"Phi-4 Multimodal","model_id":null,"model_version":"Measured on the fixed 5,000-clip stratified subsample, 2026-07-11","value":44.7,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"WavLM Odyssey MSP","model_id":null,"model_version":"Measured on the full 64,384-clip set, 2026-07-11","value":44.6,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"XLS-R SER (hughlan1214)","model_id":null,"model_version":"Measured on the full 64,384-clip set, 2026-07-11","value":44.4,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"Behavioral Signals","model_id":null,"model_version":"Measured on the fixed 5,000-clip stratified subsample, 2026-07-11","value":44.1,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"Gemini 2.5 Pro","model_id":null,"model_version":"Measured on the fixed 5,000-clip stratified subsample, 2026-07-11","value":44,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"GPT-Audio 1.5","model_id":null,"model_version":"Measured on the fixed 5,000-clip stratified subsample, 2026-07-11","value":43.3,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"DeSTA2 8B","model_id":null,"model_version":"Measured on the fixed 5,000-clip stratified subsample, 2026-07-11","value":43.1,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"Gemini 2.0 Flash","model_id":null,"model_version":"Measured on the fixed 5,000-clip stratified subsample, 2026-07-11","value":42.9,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"audEERING devAIce","model_id":null,"model_version":"Measured on the fixed 5,000-clip stratified subsample, 2026-07-11","value":42.2,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"Ultravox v0.5 8B","model_id":null,"model_version":"Measured on the fixed 5,000-clip stratified subsample, 2026-07-11","value":41.8,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"GPT-5.2 (transcript)","model_id":null,"model_version":"Measured on the fixed 5,000-clip stratified subsample, 2026-07-11","value":41.2,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"HuBERT-large SUPERB","model_id":null,"model_version":"Measured on the full 64,384-clip set, 2026-07-11","value":40.9,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"MERaLiON-2 10B","model_id":null,"model_version":"Measured on the fixed 5,000-clip stratified subsample, 2026-07-11","value":40.9,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"Claude Opus 4.8 (transcript)","model_id":null,"model_version":"Measured on the fixed 5,000-clip stratified subsample, 2026-07-11","value":40.3,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"XLS-R SER (Hatman)","model_id":null,"model_version":"Measured on the full 64,384-clip set, 2026-07-11","value":40.2,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"Gemini 2.5 Flash-Lite","model_id":null,"model_version":"Measured on the fixed 5,000-clip stratified subsample, 2026-07-11","value":40,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"GPT-Audio mini","model_id":null,"model_version":"Measured on the fixed 5,000-clip stratified subsample, 2026-07-11","value":39.7,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"UniSpeech-SAT SUPERB","model_id":null,"model_version":"Measured on the full 64,384-clip set, 2026-07-11","value":39.6,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"Claude Sonnet 4.6 (transcript)","model_id":null,"model_version":"Measured on the fixed 5,000-clip stratified subsample, 2026-07-11","value":39.4,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"Claude Haiku 4.5 (transcript)","model_id":null,"model_version":"Measured on the fixed 5,000-clip stratified subsample, 2026-07-11","value":39.3,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"SpeechBrain IEMOCAP","model_id":null,"model_version":"Measured on the full 64,384-clip set, 2026-07-11","value":39.2,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"SALMONN 13B","model_id":null,"model_version":"Measured on the fixed 5,000-clip stratified subsample, 2026-07-11","value":38.9,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"wav2vec2 SER (Dpngtm)","model_id":null,"model_version":"Measured on the full 64,384-clip set, 2026-07-11","value":38.2,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"Grok 4 (transcript)","model_id":null,"model_version":"Measured on the fixed 5,000-clip stratified subsample, 2026-07-11","value":37.9,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"WavLM-base+ SUPERB","model_id":null,"model_version":"Measured on the full 64,384-clip set, 2026-07-11","value":37.9,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"Baichuan-Audio","model_id":null,"model_version":"Measured on the fixed 5,000-clip stratified subsample, 2026-07-11","value":37.7,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"wav2vec2-large SUPERB","model_id":null,"model_version":"Measured on the full 64,384-clip set, 2026-07-11","value":36.8,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"Voxtral-Mini 3B","model_id":null,"model_version":"Measured on the fixed 5,000-clip stratified subsample, 2026-07-11","value":36.6,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"GLM-4-Voice 9B","model_id":null,"model_version":"Measured on the fixed 5,000-clip stratified subsample, 2026-07-11","value":35.2,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"data2vec-audio SER","model_id":null,"model_version":"Measured on the full 64,384-clip set, 2026-07-11","value":34.4,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"XLSR SER (harshit345)","model_id":null,"model_version":"Measured on the full 64,384-clip set, 2026-07-11","value":33.7,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"GAMA 7B","model_id":null,"model_version":"Measured on the fixed 5,000-clip stratified subsample, 2026-07-11","value":33.4,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"Whisper SER (firdhokk)","model_id":null,"model_version":"Measured on the full 64,384-clip set, 2026-07-11","value":32.5,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"wav2vec2-base SUPERB","model_id":null,"model_version":"Measured on the full 64,384-clip set, 2026-07-11","value":31.4,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"LTU-AS 7B","model_id":null,"model_version":"Measured on the fixed 5,000-clip stratified subsample, 2026-07-11","value":30.8,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"openSMILE ComParE SVM","model_id":null,"model_version":"Measured on the full 64,384-clip set, 2026-07-11","value":30.2,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"DistilHuBERT SUPERB","model_id":null,"model_version":"Measured on the full 64,384-clip set, 2026-07-11","value":29.5,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"OpenVokaturi","model_id":null,"model_version":"Measured on the full 64,384-clip set, 2026-07-11","value":28.7,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"CLAP zero-shot (LAION)","model_id":null,"model_version":"Measured on the full 64,384-clip set, 2026-07-11","value":26.4,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"XLS-R SER (firdhokk)","model_id":null,"model_version":"Measured on the full 64,384-clip set, 2026-07-11","value":26,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"Empath API","model_id":null,"model_version":"Measured on the fixed 5,000-clip stratified subsample, 2026-07-11","value":25.3,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"Mini-Omni2","model_id":null,"model_version":"Measured on the fixed 5,000-clip stratified subsample, 2026-07-11","value":24.6,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"XLS-R RAVDESS","model_id":null,"model_version":"Measured on the full 64,384-clip set, 2026-07-11","value":23.8,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"XLSR ShEMO (m3hrdadfi)","model_id":null,"model_version":"Measured on the full 64,384-clip set, 2026-07-11","value":21.7,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"SpeechGPT","model_id":null,"model_version":"Measured on the fixed 5,000-clip stratified subsample, 2026-07-11","value":19.8,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"AnyGPT","model_id":null,"model_version":"Measured on the fixed 5,000-clip stratified subsample, 2026-07-11","value":17.4,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"wav2vec2 CTC SER (r-f)","model_id":null,"model_version":"Measured on the full 64,384-clip set, 2026-07-11","value":16,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"XLS-R Russian (Aniemore)","model_id":null,"model_version":"Measured on the full 64,384-clip set, 2026-07-11","value":9.1,"measured_in_oruk_harness":true,"vendor_published_reference":false}]},{"id":"iemocap","title":"Valence polarity, nine commercial APIs","metric":"Valence accuracy","unit":"%","direction":"higher","sample":"IEMOCAP test split (2,008 clips): identical audio through nine public APIs; gold labels mapped to negative / neutral / positive. Chance 33%.","confidence_interval":null,"confidence_interval_note":"Not reported in this release because per-example bootstrap artifacts are not available for every system.","results":[{"model":"oruk Spectra 1","model_id":"oruk-spectra-1","model_version":"IEMOCAP valence polarity, n=1947, evaluated 2026-08-01","value":71.5,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"Imentiv","model_id":null,"model_version":"Stopped at free-tier credit limit.","value":63.6,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"Smallest.ai Pulse","model_id":null,"model_version":"Taxonomy has no neutral class.","value":59.3,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"Mistral Voxtral Small","model_id":null,"model_version":"Prompted audio-LLM classification, voxtral-small-latest.","value":54.4,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"Rev AI sentiment","model_id":null,"model_version":"Stopped at free-tier credit limit.","value":53.5,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"Hume EVI prosody","model_id":null,"model_version":"EVI websocket top-1 prosody snapshot; stopped at free-tier credit limit. Not a measurement of other current Hume products.","value":51.3,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"Deepgram nova-3 sentiment","model_id":null,"model_version":"IEMOCAP valence polarity, n=1947, evaluated 2026-08-01","value":46.3,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"AssemblyAI sentiment","model_id":null,"model_version":"33 clips returned no sentiment.","value":36,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"Gladia sentiment","model_id":null,"model_version":"Free tier rate-limited almost immediately; interval too wide to rank.","value":35.7,"measured_in_oruk_harness":true,"vendor_published_reference":false}]},{"id":"mustard","title":"MUStARD, oruk vs Deepgram","metric":"Macro F1","unit":"%","direction":"higher","sample":"MUStARD (690 utterances): identical logistic probe over oruk affect scores vs Deepgram nova-3 transcript sentiment. Trivial always-“not sarcastic” baseline is 33.3%.","confidence_interval":null,"confidence_interval_note":"Not reported in this release because per-example bootstrap artifacts are not available for every system.","results":[{"model":"oruk · Standard 5-fold CV","model_id":"oruk-spectra-1","model_version":"Speaker-dependent, the original paper’s primary protocol; macro-F1 0.562, evaluated 2026-08-01","value":56.2,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"Deepgram · Standard 5-fold CV","model_id":null,"model_version":"Speaker-dependent, the original paper’s primary protocol; macro-F1 0.517, evaluated 2026-08-01","value":51.7,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"oruk · Speaker-independent 5-fold","model_id":"oruk-spectra-1","model_version":"Grouped folds: no speaker appears in both train and test; macro-F1 0.510, evaluated 2026-08-01","value":51,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"Deepgram · Speaker-independent 5-fold","model_id":null,"model_version":"Grouped folds: no speaker appears in both train and test; macro-F1 0.395, evaluated 2026-08-01","value":39.5,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"oruk · Cross-show","model_id":"oruk-spectra-1","model_version":"Train on The Big Bang Theory, Golden Girls, Sarcasmoholics; test on Friends; macro-F1 0.462, evaluated 2026-08-01","value":46.2,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"Deepgram · Cross-show","model_id":null,"model_version":"Train on The Big Bang Theory, Golden Girls, Sarcasmoholics; test on Friends; macro-F1 0.299, evaluated 2026-08-01","value":29.9,"measured_in_oruk_harness":true,"vendor_published_reference":false}]},{"id":"clean-asr","title":"English public four-set average","metric":"WER","unit":"%","direction":"lower","sample":"LibriSpeech clean/other · Common Voice · TED-LIUM","confidence_interval":null,"confidence_interval_note":"Not reported in this release because per-example bootstrap artifacts are not available for every system.","results":[{"model":"oruk Spectra 1","model_id":"oruk-spectra-1","model_version":"2026-07 evaluation release","value":4.44,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"Open ASR A","model_id":null,"model_version":"Internal checkpoint registry, pinned 2026-07-11","value":4.47,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"Azure Speech-to-Text","model_id":null,"model_version":"Published four-set average accessed 2026-07-12","value":5.5,"measured_in_oruk_harness":false,"vendor_published_reference":true},{"model":"Whisper Large V3","model_id":null,"model_version":"Published four-set average accessed 2026-07-12","value":5.7,"measured_in_oruk_harness":false,"vendor_published_reference":true},{"model":"Whisper Medium","model_id":null,"model_version":"Published four-set average accessed 2026-07-12","value":6.1,"measured_in_oruk_harness":false,"vendor_published_reference":true},{"model":"Open ASR B","model_id":null,"model_version":"Internal checkpoint registry, pinned 2026-07-11","value":6.53,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"Whisper Small","model_id":null,"model_version":"Published four-set average accessed 2026-07-12","value":7,"measured_in_oruk_harness":false,"vendor_published_reference":true},{"model":"Google Speech-to-Text","model_id":null,"model_version":"Published four-set average accessed 2026-07-12","value":8.9,"measured_in_oruk_harness":false,"vendor_published_reference":true},{"model":"Picovoice Leopard","model_id":null,"model_version":"Vendor-published value accessed 2026-07-11","value":9.7,"measured_in_oruk_harness":false,"vendor_published_reference":true},{"model":"Picovoice Cheetah","model_id":null,"model_version":"Vendor-published value accessed 2026-07-11","value":10.1,"measured_in_oruk_harness":false,"vendor_published_reference":true}]}],"open_emotion_benchmark":{"name":"speech-emotion-bench","clips":64384,"emotionClasses":["anger","happiness","sadness","fear","disgust","surprise","neutral"],"languages":20,"systems":64,"subsampleClips":5000,"description":"A held-out speech emotion recognition benchmark across ~20 languages and 7 emotion classes. Open models are scored on the full 64,384-clip evaluation; closed/API and audio-LLM models are scored on a fixed 5,000-clip stratified subsample. All rows use the same label mapping and scorer, but not every system sees every clip. Rescoring open models on the same subsample shifts results by less than two points.","evaluation_date":"2026-07-11","dataset_manifest":"/benchmarks/emotion-eval-manifest.json","fairness_note":"The oruk entry is trained in-distribution; other listed systems are evaluated zero-shot across corpora.","results":[{"name":"oruk Spectra 1","short":"Spectra 1","tag":"ours","acc":77.8,"mf1":0.816,"ours":true,"model_id":"oruk-spectra-1","model_version":"2026-07 evaluation release"},{"name":"emotion2vec+ seed","short":"e2v+ seed","tag":"open","acc":68.7,"mf1":0.68,"model_id":null,"model_version":"emotion2vec+ seed"},{"name":"emotion2vec+ large","short":"e2v+ large","tag":"open","acc":68.6,"mf1":0.677,"model_id":null,"model_version":"emotion2vec+ large"},{"name":"emotion2vec+ base","short":"e2v+ base","tag":"open","acc":68.5,"mf1":0.683,"model_id":null,"model_version":"emotion2vec+ base"},{"name":"emotion2vec finetuned","short":"e2v FT","tag":"open","acc":63.6,"mf1":0.616,"model_id":null,"model_version":"emotion2vec finetuned"},{"name":"EmotionThinker","short":"EmoThinker","tag":"audio LLM","acc":60.5,"mf1":0.504,"subsample":true,"model_id":null,"model_version":"EmotionThinker"},{"name":"SenseVoice Small","short":"SenseVoice","tag":"open","acc":55.7,"mf1":0.469,"model_id":null,"model_version":"SenseVoice Small"},{"name":"emotion2vec base (probe)","short":"e2v probe","tag":"open","acc":55.1,"mf1":0.489,"model_id":null,"model_version":"emotion2vec base (probe)"},{"name":"Kimi-Audio 7B Instruct","short":"Kimi-Audio","tag":"audio LLM","acc":52.4,"mf1":0.447,"subsample":true,"model_id":null,"model_version":"Kimi-Audio 7B Instruct"},{"name":"Qwen2.5-Omni 7B","short":"Qwen Omni","tag":"audio LLM","acc":51.4,"mf1":0.44,"subsample":true,"model_id":null,"model_version":"Qwen2.5-Omni 7B"},{"name":"Hume AI legacy prosody","short":"Hume legacy","tag":"API","acc":49.6,"mf1":0.414,"subsample":true,"evaluationNote":"Dated legacy 48-dimension Hume Expression Measurement prosody snapshot scored on the fixed 5,000-clip subset in the 2026-07-11 release. This is not a measurement of Hume's current Tagger or real-time Prosody products.","model_id":null,"model_version":"Hume AI legacy prosody"},{"name":"Qwen2-Audio 7B Instruct","short":"Qwen2-Audio","tag":"audio LLM","acc":48.9,"mf1":0.402,"subsample":true,"model_id":null,"model_version":"Qwen2-Audio 7B Instruct"},{"name":"Step-Audio 2 mini","short":"Step-Audio","tag":"audio LLM","acc":46.5,"mf1":0.381,"subsample":true,"model_id":null,"model_version":"Step-Audio 2 mini"},{"name":"Gemini 3 Flash Preview","short":"Gemini 3 Fl","tag":"API","acc":46,"mf1":0.373,"subsample":true,"model_id":null,"model_version":"Gemini 3 Flash Preview"},{"name":"Gemini 2.5 Flash","short":"Gem 2.5 Fl","tag":"API","acc":45.4,"mf1":0.37,"subsample":true,"model_id":null,"model_version":"Gemini 2.5 Flash"},{"name":"WavLM Vox-Profile","short":"WavLM VP","tag":"open","acc":45.3,"mf1":0.355,"model_id":null,"model_version":"WavLM Vox-Profile"},{"name":"Phi-4 Multimodal","short":"Phi-4 MM","tag":"audio LLM","acc":44.7,"mf1":0.362,"subsample":true,"model_id":null,"model_version":"Phi-4 Multimodal"},{"name":"WavLM Odyssey MSP","short":"WavLM Ody","tag":"open","acc":44.6,"mf1":0.369,"model_id":null,"model_version":"WavLM Odyssey MSP"},{"name":"XLS-R SER (hughlan1214)","short":"XLS-R H14","tag":"open","acc":44.4,"mf1":0.411,"model_id":null,"model_version":"XLS-R SER (hughlan1214)"},{"name":"Behavioral Signals","short":"BehSignals","tag":"API","acc":44.1,"mf1":0.353,"subsample":true,"model_id":null,"model_version":"Behavioral Signals"},{"name":"Gemini 2.5 Pro","short":"Gem 2.5 Pro","tag":"API","acc":44,"mf1":0.36,"subsample":true,"model_id":null,"model_version":"Gemini 2.5 Pro"},{"name":"GPT-Audio 1.5","short":"GPT-Audio","tag":"API","acc":43.3,"mf1":0.347,"subsample":true,"model_id":null,"model_version":"GPT-Audio 1.5"},{"name":"DeSTA2 8B","short":"DeSTA2","tag":"audio LLM","acc":43.1,"mf1":0.341,"subsample":true,"model_id":null,"model_version":"DeSTA2 8B"},{"name":"Gemini 2.0 Flash","short":"Gem 2.0 Fl","tag":"API","acc":42.9,"mf1":0.344,"subsample":true,"model_id":null,"model_version":"Gemini 2.0 Flash"},{"name":"audEERING devAIce","short":"devAIce","tag":"API","acc":42.2,"mf1":0.336,"subsample":true,"model_id":null,"model_version":"audEERING devAIce"},{"name":"Ultravox v0.5 8B","short":"Ultravox","tag":"audio LLM","acc":41.8,"mf1":0.328,"subsample":true,"model_id":null,"model_version":"Ultravox v0.5 8B"},{"name":"GPT-5.2 (transcript)","short":"GPT-5.2","tag":"text only","acc":41.2,"mf1":0.309,"subsample":true,"model_id":null,"model_version":"GPT-5.2 (transcript)"},{"name":"HuBERT-large SUPERB","short":"HuBERT-L","tag":"open","acc":40.9,"mf1":0.249,"model_id":null,"model_version":"HuBERT-large SUPERB"},{"name":"MERaLiON-2 10B","short":"MERaLiON","tag":"audio LLM","acc":40.9,"mf1":0.315,"subsample":true,"model_id":null,"model_version":"MERaLiON-2 10B"},{"name":"Claude Opus 4.8 (transcript)","short":"Opus 4.8","tag":"text only","acc":40.3,"mf1":0.298,"subsample":true,"model_id":null,"model_version":"Claude Opus 4.8 (transcript)"},{"name":"XLS-R SER (Hatman)","short":"XLS-R Hat","tag":"open","acc":40.2,"mf1":0.361,"model_id":null,"model_version":"XLS-R SER (Hatman)"},{"name":"Gemini 2.5 Flash-Lite","short":"Gem Fl-Lite","tag":"API","acc":40,"mf1":0.286,"subsample":true,"model_id":null,"model_version":"Gemini 2.5 Flash-Lite"},{"name":"GPT-Audio mini","short":"GPT-A mini","tag":"API","acc":39.7,"mf1":0.3,"subsample":true,"model_id":null,"model_version":"GPT-Audio mini"},{"name":"UniSpeech-SAT SUPERB","short":"UniSp-SAT","tag":"open","acc":39.6,"mf1":0.301,"model_id":null,"model_version":"UniSpeech-SAT SUPERB"},{"name":"Claude Sonnet 4.6 (transcript)","short":"Sonnet 4.6","tag":"text only","acc":39.4,"mf1":0.286,"subsample":true,"model_id":null,"model_version":"Claude Sonnet 4.6 (transcript)"},{"name":"Claude Haiku 4.5 (transcript)","short":"Haiku 4.5","tag":"text only","acc":39.3,"mf1":0.293,"subsample":true,"model_id":null,"model_version":"Claude Haiku 4.5 (transcript)"},{"name":"SpeechBrain IEMOCAP","short":"SpeechBrain","tag":"open","acc":39.2,"mf1":0.211,"model_id":null,"model_version":"SpeechBrain IEMOCAP"},{"name":"SALMONN 13B","short":"SALMONN","tag":"audio LLM","acc":38.9,"mf1":0.287,"subsample":true,"model_id":null,"model_version":"SALMONN 13B"},{"name":"wav2vec2 SER (Dpngtm)","short":"w2v2 Dpn","tag":"open","acc":38.2,"mf1":0.376,"model_id":null,"model_version":"wav2vec2 SER (Dpngtm)"},{"name":"Grok 4 (transcript)","short":"Grok 4","tag":"text only","acc":37.9,"mf1":0.278,"subsample":true,"model_id":null,"model_version":"Grok 4 (transcript)"},{"name":"WavLM-base+ SUPERB","short":"WavLM-b+","tag":"open","acc":37.9,"mf1":0.246,"model_id":null,"model_version":"WavLM-base+ SUPERB"},{"name":"Baichuan-Audio","short":"Baichuan-A","tag":"audio LLM","acc":37.7,"mf1":0.271,"subsample":true,"model_id":null,"model_version":"Baichuan-Audio"},{"name":"wav2vec2-large SUPERB","short":"w2v2-large","tag":"open","acc":36.8,"mf1":0.242,"model_id":null,"model_version":"wav2vec2-large SUPERB"},{"name":"Voxtral-Mini 3B","short":"Voxtral","tag":"audio LLM","acc":36.6,"mf1":0.204,"subsample":true,"model_id":null,"model_version":"Voxtral-Mini 3B"},{"name":"GLM-4-Voice 9B","short":"GLM-4-V","tag":"audio LLM","acc":35.2,"mf1":0.238,"subsample":true,"model_id":null,"model_version":"GLM-4-Voice 9B"},{"name":"data2vec-audio SER","short":"data2vec","tag":"open","acc":34.4,"mf1":0.229,"model_id":null,"model_version":"data2vec-audio SER"},{"name":"XLSR SER (harshit345)","short":"XLSR H345","tag":"open","acc":33.7,"mf1":0.286,"model_id":null,"model_version":"XLSR SER (harshit345)"},{"name":"GAMA 7B","short":"GAMA","tag":"audio LLM","acc":33.4,"mf1":0.221,"subsample":true,"model_id":null,"model_version":"GAMA 7B"},{"name":"Whisper SER (firdhokk)","short":"Whisper SER","tag":"open","acc":32.5,"mf1":0.276,"model_id":null,"model_version":"Whisper SER (firdhokk)"},{"name":"wav2vec2-base SUPERB","short":"w2v2-base","tag":"open","acc":31.4,"mf1":0.187,"model_id":null,"model_version":"wav2vec2-base SUPERB"},{"name":"LTU-AS 7B","short":"LTU-AS","tag":"audio LLM","acc":30.8,"mf1":0.198,"subsample":true,"model_id":null,"model_version":"LTU-AS 7B"},{"name":"openSMILE ComParE SVM","short":"openSMILE","tag":"open","acc":30.2,"mf1":0.203,"model_id":null,"model_version":"openSMILE ComParE SVM"},{"name":"DistilHuBERT SUPERB","short":"dHuBERT","tag":"open","acc":29.5,"mf1":0.186,"model_id":null,"model_version":"DistilHuBERT SUPERB"},{"name":"OpenVokaturi","short":"Vokaturi","tag":"open","acc":28.7,"mf1":0.192,"model_id":null,"model_version":"OpenVokaturi"},{"name":"CLAP zero-shot (LAION)","short":"CLAP ZS","tag":"open","acc":26.4,"mf1":0.177,"model_id":null,"model_version":"CLAP zero-shot (LAION)"},{"name":"XLS-R SER (firdhokk)","short":"XLS-R Fir","tag":"open","acc":26,"mf1":0.24,"model_id":null,"model_version":"XLS-R SER (firdhokk)"},{"name":"Empath API","short":"Empath","tag":"API","acc":25.3,"mf1":0.156,"subsample":true,"model_id":null,"model_version":"Empath API"},{"name":"Mini-Omni2","short":"Mini-Omni2","tag":"audio LLM","acc":24.6,"mf1":0.143,"subsample":true,"model_id":null,"model_version":"Mini-Omni2"},{"name":"XLS-R RAVDESS","short":"XLS-R RAV","tag":"open","acc":23.8,"mf1":0.118,"model_id":null,"model_version":"XLS-R RAVDESS"},{"name":"XLSR ShEMO (m3hrdadfi)","short":"XLSR ShEMO","tag":"open","acc":21.7,"mf1":0.148,"model_id":null,"model_version":"XLSR ShEMO (m3hrdadfi)"},{"name":"SpeechGPT","short":"SpeechGPT","tag":"audio LLM","acc":19.8,"mf1":0.117,"subsample":true,"model_id":null,"model_version":"SpeechGPT"},{"name":"AnyGPT","short":"AnyGPT","tag":"audio LLM","acc":17.4,"mf1":0.104,"subsample":true,"model_id":null,"model_version":"AnyGPT"},{"name":"wav2vec2 CTC SER (r-f)","short":"w2v2 CTC","tag":"open","acc":16,"mf1":0.098,"model_id":null,"model_version":"wav2vec2 CTC SER (r-f)"},{"name":"XLS-R Russian (Aniemore)","short":"XLS-R Ru","tag":"open","acc":9.1,"mf1":0.079,"model_id":null,"model_version":"XLS-R Russian (Aniemore)"}]},"limitations":["Vendor-published reference rows were not rerun in the oruk harness.","Open emotion models use the full 64,384 clips; closed/API and audio-LLM systems use a fixed 5,000-clip stratified subset. Rows share a label mapping and scorer, not identical audio.","The Hume row is a legacy 48-dimension Expression Measurement prosody snapshot, not a measurement of current Hume Tagger or Prosody.","Raw audio is not redistributed where source licenses do not permit redistribution."]}