{"object":"benchmark.report","report_date":"August 2026","evaluation_date":"2026-07-11","report_revision":"2026-09-06","model_scope":"Resonance 1, July–August 2026 evaluation checkpoint. Results apply to the recorded build and protocol; they are not a new evaluation of subsequent API releases or Fourier.","api_version":"v1","generated_at":"2026-09-06T00:00:00Z","panels":[{"id":"emotion","title":"Seven-class emotion, 64 systems","metric":"Accuracy","unit":"%","direction":"higher","sample":"speech-emotion-bench: open models on the full 64,384 clips; closed/API and audio-LLM systems on a fixed 5,000-clip stratified subset","confidence_interval":null,"confidence_interval_scope":"per result; no single interval describes the entire panel","confidence_interval_note":"Approximate 95% Wilson intervals calculated from published, rounded accuracy and clip counts. They assume independent clips, do not adjust for repeated speakers or training overlap, and are not a paired significance test. Macro-F1 intervals are unavailable.","results":[{"model":"oruk Resonance 1","model_id":"oruk-resonance-1","model_version":"Recorded 2026-07-14; full 64,384-clip evaluation checkpoint","value":77.8,"n":64384,"confidence_interval":{"level":0.95,"lower":77.47732569511845,"upper":78.11935701772465},"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"emotion2vec+ seed","model_id":null,"model_version":"Measured on the full 64,384-clip set, 2026-07-11","value":68.7,"n":64384,"confidence_interval":{"level":0.95,"lower":68.34069979889459,"upper":69.05706878852146},"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"emotion2vec+ large","model_id":null,"model_version":"Measured on the full 64,384-clip set, 2026-07-11","value":68.6,"n":64384,"confidence_interval":{"level":0.95,"lower":68.24039526194028,"upper":68.95738525816337},"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"emotion2vec+ base","model_id":null,"model_version":"Measured on the full 64,384-clip set, 2026-07-11","value":68.5,"n":64384,"confidence_interval":{"level":0.95,"lower":68.14009265643058,"upper":68.8576997963607},"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"emotion2vec finetuned","model_id":null,"model_version":"Measured on the full 64,384-clip set, 2026-07-11","value":63.6,"n":64384,"confidence_interval":{"level":0.95,"lower":63.227538332946565,"upper":63.970838821537825},"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"EmotionThinker","model_id":null,"model_version":"Measured on the fixed 5,000-clip stratified subsample, 2026-07-11","value":60.5,"n":5000,"confidence_interval":{"level":0.95,"lower":59.1374101041857,"upper":61.84646756292511},"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"SenseVoice Small","model_id":null,"model_version":"Measured on the full 64,384-clip set, 2026-07-11","value":55.7,"n":64384,"confidence_interval":{"level":0.95,"lower":55.315967012895605,"upper":56.083352823910374},"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"emotion2vec base (probe)","model_id":null,"model_version":"Measured on the full 64,384-clip set, 2026-07-11","value":55.1,"n":64384,"confidence_interval":{"level":0.95,"lower":54.71549935814317,"upper":55.48389207478848},"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"Kimi-Audio 7B Instruct","model_id":null,"model_version":"Measured on the fixed 5,000-clip stratified subsample, 2026-07-11","value":52.4,"n":5000,"confidence_interval":{"level":0.95,"lower":51.014355939736554,"upper":53.78195895560307},"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"Hume AI legacy prosody","model_id":null,"model_version":"Dated legacy 48-dimension Hume Expression Measurement prosody snapshot scored on the fixed 5,000-clip subset in the 2026-07-11 release. This is not a measurement of Hume's current Tagger or real-time Prosody products.","value":49.6,"n":5000,"confidence_interval":{"level":0.95,"lower":48.214954212303724,"upper":50.98565997180635},"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"Qwen2-Audio 7B Instruct","model_id":null,"model_version":"Measured on the fixed 5,000-clip stratified subsample, 2026-07-11","value":48.9,"n":5000,"confidence_interval":{"level":0.95,"lower":47.515782373255774,"upper":50.285906633046906},"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"Step-Audio 2 mini","model_id":null,"model_version":"Measured on the fixed 5,000-clip stratified subsample, 2026-07-11","value":46.5,"n":5000,"confidence_interval":{"level":0.95,"lower":45.120685655325396,"upper":47.88468845563768},"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"Gemini 3 Flash Preview","model_id":null,"model_version":"Measured on the fixed 5,000-clip stratified subsample, 2026-07-11","value":46,"n":5000,"confidence_interval":{"level":0.95,"lower":44.62211071391747,"upper":47.38403112718319},"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"Gemini 2.5 Flash","model_id":null,"model_version":"Measured on the fixed 5,000-clip stratified subsample, 2026-07-11","value":45.4,"n":5000,"confidence_interval":{"level":0.95,"lower":44.02400531888717,"upper":46.78305779837857},"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"WavLM Vox-Profile","model_id":null,"model_version":"Measured on the full 64,384-clip set, 2026-07-11","value":45.3,"n":64384,"confidence_interval":{"level":0.95,"lower":44.915779821967575,"upper":45.684781014350314},"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"Phi-4 Multimodal","model_id":null,"model_version":"Measured on the fixed 5,000-clip stratified subsample, 2026-07-11","value":44.7,"n":5000,"confidence_interval":{"level":0.95,"lower":43.3264709293336,"upper":46.08166701012477},"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"WavLM Odyssey MSP","model_id":null,"model_version":"Measured on the full 64,384-clip set, 2026-07-11","value":44.6,"n":64384,"confidence_interval":{"level":0.95,"lower":44.2163704757956,"upper":44.9842738893356},"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"XLS-R SER (hughlan1214)","model_id":null,"model_version":"Measured on the full 64,384-clip set, 2026-07-11","value":44.4,"n":64384,"confidence_interval":{"level":0.95,"lower":44.0165533687335,"upper":44.784114861772935},"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"Behavioral Signals","model_id":null,"model_version":"Measured on the fixed 5,000-clip stratified subsample, 2026-07-11","value":44.1,"n":5000,"confidence_interval":{"level":0.95,"lower":42.72880391741794,"upper":45.48025529820553},"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"Gemini 2.5 Pro","model_id":null,"model_version":"Measured on the fixed 5,000-clip stratified subsample, 2026-07-11","value":44,"n":5000,"confidence_interval":{"level":0.95,"lower":42.629212518056306,"upper":45.38000024359468},"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"GPT-Audio 1.5","model_id":null,"model_version":"Measured on the fixed 5,000-clip stratified subsample, 2026-07-11","value":43.3,"n":5000,"confidence_interval":{"level":0.95,"lower":41.932231414284,"upper":44.678056169559596},"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"DeSTA2 8B","model_id":null,"model_version":"Measured on the fixed 5,000-clip stratified subsample, 2026-07-11","value":43.1,"n":5000,"confidence_interval":{"level":0.95,"lower":41.733145096435685,"upper":44.477449579462935},"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"Gemini 2.0 Flash","model_id":null,"model_version":"Measured on the fixed 5,000-clip stratified subsample, 2026-07-11","value":42.9,"n":5000,"confidence_interval":{"level":0.95,"lower":41.53408157563847,"upper":44.27682019231518},"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"audEERING devAIce","model_id":null,"model_version":"Measured on the fixed 5,000-clip stratified subsample, 2026-07-11","value":42.2,"n":5000,"confidence_interval":{"level":0.95,"lower":40.83753935482758,"upper":43.574437235318705},"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"Ultravox v0.5 8B","model_id":null,"model_version":"Measured on the fixed 5,000-clip stratified subsample, 2026-07-11","value":41.8,"n":5000,"confidence_interval":{"level":0.95,"lower":40.43964151675721,"upper":43.17294925749912},"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"GPT-5.2 (transcript)","model_id":null,"model_version":"Measured on the fixed 5,000-clip stratified subsample, 2026-07-11","value":41.2,"n":5000,"confidence_interval":{"level":0.95,"lower":39.84296793097681,"upper":42.57054411944462},"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"HuBERT-large SUPERB","model_id":null,"model_version":"Measured on the full 64,384-clip set, 2026-07-11","value":40.9,"n":64384,"confidence_interval":{"level":0.95,"lower":40.520782274949994,"upper":41.28030359962294},"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"MERaLiON-2 10B","model_id":null,"model_version":"Measured on the fixed 5,000-clip stratified subsample, 2026-07-11","value":40.9,"n":5000,"confidence_interval":{"level":0.95,"lower":39.54470941929705,"upper":42.26926326920693},"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"Claude Opus 4.8 (transcript)","model_id":null,"model_version":"Measured on the fixed 5,000-clip stratified subsample, 2026-07-11","value":40.3,"n":5000,"confidence_interval":{"level":0.95,"lower":38.94834982876064,"upper":41.666544135908424},"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"XLS-R SER (Hatman)","model_id":null,"model_version":"Measured on the full 64,384-clip set, 2026-07-11","value":40.2,"n":64384,"confidence_interval":{"level":0.95,"lower":39.821864672558554,"upper":40.5793047308277},"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"Gemini 2.5 Flash-Lite","model_id":null,"model_version":"Measured on the fixed 5,000-clip stratified subsample, 2026-07-11","value":40,"n":5000,"confidence_interval":{"level":0.95,"lower":38.650249118689864,"upper":41.365105484061765},"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"GPT-Audio mini","model_id":null,"model_version":"Measured on the fixed 5,000-clip stratified subsample, 2026-07-11","value":39.7,"n":5000,"confidence_interval":{"level":0.95,"lower":38.352201389658944,"upper":41.06361385117523},"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"UniSpeech-SAT SUPERB","model_id":null,"model_version":"Measured on the full 64,384-clip set, 2026-07-11","value":39.6,"n":64384,"confidence_interval":{"level":0.95,"lower":39.22285631042337,"upper":39.978384689088564},"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"Claude Sonnet 4.6 (transcript)","model_id":null,"model_version":"Measured on the fixed 5,000-clip stratified subsample, 2026-07-11","value":39.4,"n":5000,"confidence_interval":{"level":0.95,"lower":38.05420684381434,"upper":40.762069035102364},"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"Claude Haiku 4.5 (transcript)","model_id":null,"model_version":"Measured on the fixed 5,000-clip stratified subsample, 2026-07-11","value":39.3,"n":5000,"confidence_interval":{"level":0.95,"lower":37.9548871830096,"upper":40.66154224193463},"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"SpeechBrain IEMOCAP","model_id":null,"model_version":"Measured on the full 64,384-clip set, 2026-07-11","value":39.2,"n":64384,"confidence_interval":{"level":0.95,"lower":38.82355038881531,"upper":39.57773834144708},"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"SALMONN 13B","model_id":null,"model_version":"Measured on the fixed 5,000-clip stratified subsample, 2026-07-11","value":38.9,"n":5000,"confidence_interval":{"level":0.95,"lower":37.557668024559526,"upper":40.25937558449478},"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"wav2vec2 SER (Dpngtm)","model_id":null,"model_version":"Measured on the full 64,384-clip set, 2026-07-11","value":38.2,"n":64384,"confidence_interval":{"level":0.95,"lower":37.825402078503636,"upper":38.5760059786349},"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"Grok 4 (transcript)","model_id":null,"model_version":"Measured on the fixed 5,000-clip stratified subsample, 2026-07-11","value":37.9,"n":5000,"confidence_interval":{"level":0.95,"lower":36.565039457497974,"upper":39.25353961183149},"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"WavLM-base+ SUPERB","model_id":null,"model_version":"Measured on the full 64,384-clip set, 2026-07-11","value":37.9,"n":64384,"confidence_interval":{"level":0.95,"lower":37.52599030043892,"upper":38.275453554762464},"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"Baichuan-Audio","model_id":null,"model_version":"Measured on the fixed 5,000-clip stratified subsample, 2026-07-11","value":37.7,"n":5000,"confidence_interval":{"level":0.95,"lower":36.366586192110994,"upper":39.0522999692735},"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"wav2vec2-large SUPERB","model_id":null,"model_version":"Measured on the full 64,384-clip set, 2026-07-11","value":36.8,"n":64384,"confidence_interval":{"level":0.95,"lower":36.42827788236608,"upper":37.173297232399065},"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"Voxtral-Mini 3B","model_id":null,"model_version":"Measured on the fixed 5,000-clip stratified subsample, 2026-07-11","value":36.6,"n":5000,"confidence_interval":{"level":0.95,"lower":35.27553009469843,"upper":37.94504507298874},"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"GLM-4-Voice 9B","model_id":null,"model_version":"Measured on the fixed 5,000-clip stratified subsample, 2026-07-11","value":35.2,"n":5000,"confidence_interval":{"level":0.95,"lower":33.887998949200934,"upper":36.53472586287147},"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"data2vec-audio SER","model_id":null,"model_version":"Measured on the full 64,384-clip set, 2026-07-11","value":34.4,"n":64384,"confidence_interval":{"level":0.95,"lower":34.03399783043823,"upper":34.767863668829655},"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"XLSR SER (harshit345)","model_id":null,"model_version":"Measured on the full 64,384-clip set, 2026-07-11","value":33.7,"n":64384,"confidence_interval":{"level":0.95,"lower":33.335859432782236,"upper":34.06608559529897},"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"GAMA 7B","model_id":null,"model_version":"Measured on the fixed 5,000-clip stratified subsample, 2026-07-11","value":33.4,"n":5000,"confidence_interval":{"level":0.95,"lower":32.1058655788011,"upper":34.719623061766576},"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"Whisper SER (firdhokk)","model_id":null,"model_version":"Measured on the full 64,384-clip set, 2026-07-11","value":32.5,"n":64384,"confidence_interval":{"level":0.95,"lower":32.13925997917077,"upper":32.86282824116182},"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"wav2vec2-base SUPERB","model_id":null,"model_version":"Measured on the full 64,384-clip set, 2026-07-11","value":31.4,"n":64384,"confidence_interval":{"level":0.95,"lower":31.042614741836623,"upper":31.759604738059714},"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"Qwen2.5-Omni 7B","model_id":null,"model_version":"Fixed 5,000-clip stratified subset; recorded in the 2026-07-14 run. Published values corrected on 2026-09-06 to match that run.","value":31.3,"n":5000,"confidence_interval":{"level":0.95,"lower":30.029419238848178,"upper":32.59929386829735},"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"LTU-AS 7B","model_id":null,"model_version":"Measured on the fixed 5,000-clip stratified subsample, 2026-07-11","value":30.8,"n":5000,"confidence_interval":{"level":0.95,"lower":29.535472430307706,"upper":32.09400840697539},"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"openSMILE ComParE SVM","model_id":null,"model_version":"Measured on the full 64,384-clip set, 2026-07-11","value":30.2,"n":64384,"confidence_interval":{"level":0.95,"lower":29.84654132637849,"upper":30.555821345769225},"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"DistilHuBERT SUPERB","model_id":null,"model_version":"Measured on the full 64,384-clip set, 2026-07-11","value":29.5,"n":64384,"confidence_interval":{"level":0.95,"lower":29.148963918473093,"upper":29.85348228248793},"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"OpenVokaturi","model_id":null,"model_version":"Measured on the full 64,384-clip set, 2026-07-11","value":28.7,"n":64384,"confidence_interval":{"level":0.95,"lower":28.35185488285914,"upper":29.050686779602795},"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"CLAP zero-shot (LAION)","model_id":null,"model_version":"Measured on the full 64,384-clip set, 2026-07-11","value":26.4,"n":64384,"confidence_interval":{"level":0.95,"lower":26.060922478384313,"upper":26.741893635892772},"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"XLS-R SER (firdhokk)","model_id":null,"model_version":"Measured on the full 64,384-clip set, 2026-07-11","value":26,"n":64384,"confidence_interval":{"level":0.95,"lower":25.66261854430113,"upper":26.340245300726405},"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"Empath API","model_id":null,"model_version":"Measured on the fixed 5,000-clip stratified subsample, 2026-07-11","value":25.3,"n":5000,"confidence_interval":{"level":0.95,"lower":24.11426361207914,"upper":26.523662256717355},"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"Mini-Omni2","model_id":null,"model_version":"Measured on the fixed 5,000-clip stratified subsample, 2026-07-11","value":24.6,"n":5000,"confidence_interval":{"level":0.95,"lower":23.42601937293379,"upper":25.81298131805531},"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"XLS-R RAVDESS","model_id":null,"model_version":"Measured on the full 64,384-clip set, 2026-07-11","value":23.8,"n":64384,"confidence_interval":{"level":0.95,"lower":23.47261688206623,"upper":24.130509482088833},"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"XLSR ShEMO (m3hrdadfi)","model_id":null,"model_version":"Measured on the full 64,384-clip set, 2026-07-11","value":21.7,"n":64384,"confidence_interval":{"level":0.95,"lower":21.383289990354147,"upper":22.02008696024082},"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"SpeechGPT","model_id":null,"model_version":"Measured on the fixed 5,000-clip stratified subsample, 2026-07-11","value":19.8,"n":5000,"confidence_interval":{"level":0.95,"lower":18.718802222116988,"upper":20.92756867819289},"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"AnyGPT","model_id":null,"model_version":"Measured on the fixed 5,000-clip stratified subsample, 2026-07-11","value":17.4,"n":5000,"confidence_interval":{"level":0.95,"lower":16.37429546570668,"upper":18.47576053926358},"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"wav2vec2 CTC SER (r-f)","model_id":null,"model_version":"Measured on the full 64,384-clip set, 2026-07-11","value":16,"n":64384,"confidence_interval":{"level":0.95,"lower":15.718847032401625,"upper":16.285210081387383},"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"XLS-R Russian (Aniemore)","model_id":null,"model_version":"Measured on the full 64,384-clip set, 2026-07-11","value":9.1,"n":64384,"confidence_interval":{"level":0.95,"lower":8.880271483247538,"upper":9.324608985986888},"measured_in_oruk_harness":true,"vendor_published_reference":false}]},{"id":"iemocap","title":"Valence polarity, nine commercial APIs","metric":"Valence accuracy","unit":"%","direction":"higher","sample":"IEMOCAP test split (2,008 clips): identical audio through nine public APIs; gold labels mapped to negative / neutral / positive. Chance 33%.","confidence_interval":null,"confidence_interval_scope":"per result; no single interval describes the entire panel","confidence_interval_note":"95% Wilson intervals on accuracy; bootstrap intervals on UAR. Coverage differs by system, and smaller n widens the interval.","results":[{"model":"oruk Resonance 1","model_id":"oruk-resonance-1","model_version":"IEMOCAP valence polarity, n=1947, evaluated 2026-08-01","value":71.5,"n":1947,"confidence_interval":{"level":0.95,"lower":69.5,"upper":73.5},"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"Imentiv","model_id":null,"model_version":"Stopped at free-tier credit limit.","value":63.6,"n":118,"confidence_interval":{"level":0.95,"lower":54.6,"upper":71.7},"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"Smallest.ai Pulse","model_id":null,"model_version":"Taxonomy has no neutral class.","value":59.3,"n":1947,"confidence_interval":{"level":0.95,"lower":57.1,"upper":61.4},"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"Mistral Voxtral Small","model_id":null,"model_version":"Prompted audio-LLM classification, voxtral-small-latest.","value":54.4,"n":1947,"confidence_interval":{"level":0.95,"lower":52.2,"upper":56.6},"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"Rev AI sentiment","model_id":null,"model_version":"Stopped at free-tier credit limit.","value":53.5,"n":381,"confidence_interval":{"level":0.95,"lower":48.5,"upper":58.5},"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"Hume EVI prosody","model_id":null,"model_version":"EVI websocket top-1 prosody snapshot; stopped at free-tier credit limit. Not a measurement of other current Hume products.","value":51.3,"n":154,"confidence_interval":{"level":0.95,"lower":43.5,"upper":59.1},"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"Deepgram nova-3 sentiment","model_id":null,"model_version":"IEMOCAP valence polarity, n=1947, evaluated 2026-08-01","value":46.3,"n":1947,"confidence_interval":{"level":0.95,"lower":44.1,"upper":48.5},"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"AssemblyAI sentiment","model_id":null,"model_version":"33 clips returned no sentiment.","value":36,"n":1917,"confidence_interval":{"level":0.95,"lower":33.9,"upper":38.2},"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"Gladia sentiment","model_id":null,"model_version":"Free tier rate-limited almost immediately; interval too wide to rank.","value":35.7,"n":28,"confidence_interval":{"level":0.95,"lower":20.7,"upper":54.2},"measured_in_oruk_harness":true,"vendor_published_reference":false}]},{"id":"mustard","title":"MUStARD, oruk vs Deepgram","metric":"Macro F1","unit":"%","direction":"higher","sample":"MUStARD (690 utterances): identical logistic probe over oruk affect scores vs Deepgram nova-3 transcript sentiment. Trivial always-“not sarcastic” baseline is 33.3%.","confidence_interval":null,"confidence_interval_scope":"per result; no single interval describes the entire panel","confidence_interval_note":"95% bootstrap intervals on out-of-fold macro-F1 for each setup. Overlapping intervals do not establish a reliable lead.","results":[{"model":"oruk · Standard 5-fold CV","model_id":"oruk-resonance-1","model_version":"Speaker-dependent, the original paper’s primary protocol; macro-F1 0.562, evaluated 2026-08-01","value":56.2,"n":690,"confidence_interval":{"level":0.95,"lower":52.400000000000006,"upper":60.099999999999994},"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"Deepgram · Standard 5-fold CV","model_id":null,"model_version":"Speaker-dependent, the original paper’s primary protocol; macro-F1 0.517, evaluated 2026-08-01","value":51.7,"n":690,"confidence_interval":{"level":0.95,"lower":47.8,"upper":55.300000000000004},"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"oruk · Speaker-independent 5-fold","model_id":"oruk-resonance-1","model_version":"Grouped folds: no speaker appears in both train and test; macro-F1 0.510, evaluated 2026-08-01","value":51,"n":690,"confidence_interval":{"level":0.95,"lower":47.099999999999994,"upper":54.800000000000004},"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"Deepgram · Speaker-independent 5-fold","model_id":null,"model_version":"Grouped folds: no speaker appears in both train and test; macro-F1 0.395, evaluated 2026-08-01","value":39.5,"n":690,"confidence_interval":{"level":0.95,"lower":35.8,"upper":43.1},"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"oruk · Cross-show","model_id":"oruk-resonance-1","model_version":"Train on The Big Bang Theory, Golden Girls, Sarcasmoholics; test on Friends; macro-F1 0.462, evaluated 2026-08-01","value":46.2,"n":null,"confidence_interval":{"level":0.95,"lower":40.9,"upper":51.5},"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"Deepgram · Cross-show","model_id":null,"model_version":"Train on The Big Bang Theory, Golden Girls, Sarcasmoholics; test on Friends; macro-F1 0.299, evaluated 2026-08-01","value":29.9,"n":null,"confidence_interval":{"level":0.95,"lower":27.3,"upper":32.2},"measured_in_oruk_harness":true,"vendor_published_reference":false}]},{"id":"clean-asr","title":"English public four-set average","metric":"WER","unit":"%","direction":"lower","sample":"LibriSpeech clean/other · Common Voice · TED-LIUM","confidence_interval":null,"confidence_interval_scope":"per result; no single interval describes the entire panel","confidence_interval_note":"Confidence intervals are unavailable: this release contains aggregate WER without the per-clip errors and sample counts needed to estimate uncertainty.","results":[{"model":"oruk Resonance 1","model_id":"oruk-resonance-1","model_version":"2026-07 evaluation release","value":4.44,"n":null,"confidence_interval":null,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"Open ASR A","model_id":null,"model_version":"Internal checkpoint registry, pinned 2026-07-11","value":4.47,"n":null,"confidence_interval":null,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"Azure Speech-to-Text","model_id":null,"model_version":"Published four-set average accessed 2026-07-12","value":5.5,"n":null,"confidence_interval":null,"measured_in_oruk_harness":false,"vendor_published_reference":true},{"model":"Whisper Large V3","model_id":null,"model_version":"Published four-set average accessed 2026-07-12","value":5.7,"n":null,"confidence_interval":null,"measured_in_oruk_harness":false,"vendor_published_reference":true},{"model":"Whisper Medium","model_id":null,"model_version":"Published four-set average accessed 2026-07-12","value":6.1,"n":null,"confidence_interval":null,"measured_in_oruk_harness":false,"vendor_published_reference":true},{"model":"Open ASR B","model_id":null,"model_version":"Internal checkpoint registry, pinned 2026-07-11","value":6.53,"n":null,"confidence_interval":null,"measured_in_oruk_harness":true,"vendor_published_reference":false},{"model":"Whisper Small","model_id":null,"model_version":"Published four-set average accessed 2026-07-12","value":7,"n":null,"confidence_interval":null,"measured_in_oruk_harness":false,"vendor_published_reference":true},{"model":"Google Speech-to-Text","model_id":null,"model_version":"Published four-set average accessed 2026-07-12","value":8.9,"n":null,"confidence_interval":null,"measured_in_oruk_harness":false,"vendor_published_reference":true},{"model":"Picovoice Leopard","model_id":null,"model_version":"Vendor-published value accessed 2026-07-11","value":9.7,"n":null,"confidence_interval":null,"measured_in_oruk_harness":false,"vendor_published_reference":true},{"model":"Picovoice Cheetah","model_id":null,"model_version":"Vendor-published value accessed 2026-07-11","value":10.1,"n":null,"confidence_interval":null,"measured_in_oruk_harness":false,"vendor_published_reference":true}]}],"open_emotion_benchmark":{"name":"speech-emotion-bench","clips":64384,"emotionClasses":["anger","happiness","sadness","fear","disgust","surprise","neutral"],"languages":20,"systems":64,"subsampleClips":5000,"description":"A held-out speech emotion recognition benchmark across ~20 languages and 7 emotion classes. Open models are scored on the full 64,384-clip evaluation; closed/API and audio-LLM models are scored on a fixed 5,000-clip stratified subsample. All rows use the same label mapping and scorer, but not every system sees every clip. Rescoring open models on the same subsample shifts results by less than two points.","evaluation_date":"2026-07-11","dataset_manifest":"/benchmarks/emotion-eval-manifest.json","fairness_note":"The oruk entry is trained in-distribution; other listed systems are evaluated zero-shot across corpora.","results":[{"name":"oruk Resonance 1","short":"Resonance 1","tag":"ours","acc":77.8,"mf1":0.816,"ours":true,"model_id":"oruk-resonance-1","model_version":"2026-07 evaluation release"},{"name":"emotion2vec+ seed","short":"e2v+ seed","tag":"open","acc":68.7,"mf1":0.68,"model_id":null,"model_version":"emotion2vec+ seed"},{"name":"emotion2vec+ large","short":"e2v+ large","tag":"open","acc":68.6,"mf1":0.677,"model_id":null,"model_version":"emotion2vec+ large"},{"name":"emotion2vec+ base","short":"e2v+ base","tag":"open","acc":68.5,"mf1":0.683,"model_id":null,"model_version":"emotion2vec+ base"},{"name":"emotion2vec finetuned","short":"e2v FT","tag":"open","acc":63.6,"mf1":0.616,"model_id":null,"model_version":"emotion2vec finetuned"},{"name":"EmotionThinker","short":"EmoThinker","tag":"audio LLM","acc":60.5,"mf1":0.504,"subsample":true,"model_id":null,"model_version":"EmotionThinker"},{"name":"SenseVoice Small","short":"SenseVoice","tag":"open","acc":55.7,"mf1":0.469,"model_id":null,"model_version":"SenseVoice Small"},{"name":"emotion2vec base (probe)","short":"e2v probe","tag":"open","acc":55.1,"mf1":0.489,"model_id":null,"model_version":"emotion2vec base (probe)"},{"name":"Kimi-Audio 7B Instruct","short":"Kimi-Audio","tag":"audio LLM","acc":52.4,"mf1":0.447,"subsample":true,"model_id":null,"model_version":"Kimi-Audio 7B Instruct"},{"name":"Hume AI legacy prosody","short":"Hume legacy","tag":"API","acc":49.6,"mf1":0.414,"subsample":true,"evaluationNote":"Dated legacy 48-dimension Hume Expression Measurement prosody snapshot scored on the fixed 5,000-clip subset in the 2026-07-11 release. This is not a measurement of Hume's current Tagger or real-time Prosody products.","model_id":null,"model_version":"Hume AI legacy prosody"},{"name":"Qwen2-Audio 7B Instruct","short":"Qwen2-Audio","tag":"audio LLM","acc":48.9,"mf1":0.402,"subsample":true,"model_id":null,"model_version":"Qwen2-Audio 7B Instruct"},{"name":"Step-Audio 2 mini","short":"Step-Audio","tag":"audio LLM","acc":46.5,"mf1":0.381,"subsample":true,"model_id":null,"model_version":"Step-Audio 2 mini"},{"name":"Gemini 3 Flash Preview","short":"Gemini 3 Fl","tag":"API","acc":46,"mf1":0.373,"subsample":true,"model_id":null,"model_version":"Gemini 3 Flash Preview"},{"name":"Gemini 2.5 Flash","short":"Gem 2.5 Fl","tag":"API","acc":45.4,"mf1":0.37,"subsample":true,"model_id":null,"model_version":"Gemini 2.5 Flash"},{"name":"WavLM Vox-Profile","short":"WavLM VP","tag":"open","acc":45.3,"mf1":0.355,"model_id":null,"model_version":"WavLM Vox-Profile"},{"name":"Phi-4 Multimodal","short":"Phi-4 MM","tag":"audio LLM","acc":44.7,"mf1":0.362,"subsample":true,"model_id":null,"model_version":"Phi-4 Multimodal"},{"name":"WavLM Odyssey MSP","short":"WavLM Ody","tag":"open","acc":44.6,"mf1":0.369,"model_id":null,"model_version":"WavLM Odyssey MSP"},{"name":"XLS-R SER (hughlan1214)","short":"XLS-R H14","tag":"open","acc":44.4,"mf1":0.411,"model_id":null,"model_version":"XLS-R SER (hughlan1214)"},{"name":"Behavioral Signals","short":"BehSignals","tag":"API","acc":44.1,"mf1":0.353,"subsample":true,"model_id":null,"model_version":"Behavioral Signals"},{"name":"Gemini 2.5 Pro","short":"Gem 2.5 Pro","tag":"API","acc":44,"mf1":0.36,"subsample":true,"model_id":null,"model_version":"Gemini 2.5 Pro"},{"name":"GPT-Audio 1.5","short":"GPT-Audio","tag":"API","acc":43.3,"mf1":0.347,"subsample":true,"model_id":null,"model_version":"GPT-Audio 1.5"},{"name":"DeSTA2 8B","short":"DeSTA2","tag":"audio LLM","acc":43.1,"mf1":0.341,"subsample":true,"model_id":null,"model_version":"DeSTA2 8B"},{"name":"Gemini 2.0 Flash","short":"Gem 2.0 Fl","tag":"API","acc":42.9,"mf1":0.344,"subsample":true,"model_id":null,"model_version":"Gemini 2.0 Flash"},{"name":"audEERING devAIce","short":"devAIce","tag":"API","acc":42.2,"mf1":0.336,"subsample":true,"model_id":null,"model_version":"audEERING devAIce"},{"name":"Ultravox v0.5 8B","short":"Ultravox","tag":"audio LLM","acc":41.8,"mf1":0.328,"subsample":true,"model_id":null,"model_version":"Ultravox v0.5 8B"},{"name":"GPT-5.2 (transcript)","short":"GPT-5.2","tag":"text only","acc":41.2,"mf1":0.309,"subsample":true,"model_id":null,"model_version":"GPT-5.2 (transcript)"},{"name":"HuBERT-large SUPERB","short":"HuBERT-L","tag":"open","acc":40.9,"mf1":0.249,"model_id":null,"model_version":"HuBERT-large SUPERB"},{"name":"MERaLiON-2 10B","short":"MERaLiON","tag":"audio LLM","acc":40.9,"mf1":0.315,"subsample":true,"model_id":null,"model_version":"MERaLiON-2 10B"},{"name":"Claude Opus 4.8 (transcript)","short":"Opus 4.8","tag":"text only","acc":40.3,"mf1":0.298,"subsample":true,"model_id":null,"model_version":"Claude Opus 4.8 (transcript)"},{"name":"XLS-R SER (Hatman)","short":"XLS-R Hat","tag":"open","acc":40.2,"mf1":0.361,"model_id":null,"model_version":"XLS-R SER (Hatman)"},{"name":"Gemini 2.5 Flash-Lite","short":"Gem Fl-Lite","tag":"API","acc":40,"mf1":0.286,"subsample":true,"model_id":null,"model_version":"Gemini 2.5 Flash-Lite"},{"name":"GPT-Audio mini","short":"GPT-A mini","tag":"API","acc":39.7,"mf1":0.3,"subsample":true,"model_id":null,"model_version":"GPT-Audio mini"},{"name":"UniSpeech-SAT SUPERB","short":"UniSp-SAT","tag":"open","acc":39.6,"mf1":0.301,"model_id":null,"model_version":"UniSpeech-SAT SUPERB"},{"name":"Claude Sonnet 4.6 (transcript)","short":"Sonnet 4.6","tag":"text only","acc":39.4,"mf1":0.286,"subsample":true,"model_id":null,"model_version":"Claude Sonnet 4.6 (transcript)"},{"name":"Claude Haiku 4.5 (transcript)","short":"Haiku 4.5","tag":"text only","acc":39.3,"mf1":0.293,"subsample":true,"model_id":null,"model_version":"Claude Haiku 4.5 (transcript)"},{"name":"SpeechBrain IEMOCAP","short":"SpeechBrain","tag":"open","acc":39.2,"mf1":0.211,"model_id":null,"model_version":"SpeechBrain IEMOCAP"},{"name":"SALMONN 13B","short":"SALMONN","tag":"audio LLM","acc":38.9,"mf1":0.287,"subsample":true,"model_id":null,"model_version":"SALMONN 13B"},{"name":"wav2vec2 SER (Dpngtm)","short":"w2v2 Dpn","tag":"open","acc":38.2,"mf1":0.376,"model_id":null,"model_version":"wav2vec2 SER (Dpngtm)"},{"name":"Grok 4 (transcript)","short":"Grok 4","tag":"text only","acc":37.9,"mf1":0.278,"subsample":true,"model_id":null,"model_version":"Grok 4 (transcript)"},{"name":"WavLM-base+ SUPERB","short":"WavLM-b+","tag":"open","acc":37.9,"mf1":0.246,"model_id":null,"model_version":"WavLM-base+ SUPERB"},{"name":"Baichuan-Audio","short":"Baichuan-A","tag":"audio LLM","acc":37.7,"mf1":0.271,"subsample":true,"model_id":null,"model_version":"Baichuan-Audio"},{"name":"wav2vec2-large SUPERB","short":"w2v2-large","tag":"open","acc":36.8,"mf1":0.242,"model_id":null,"model_version":"wav2vec2-large SUPERB"},{"name":"Voxtral-Mini 3B","short":"Voxtral","tag":"audio LLM","acc":36.6,"mf1":0.204,"subsample":true,"model_id":null,"model_version":"Voxtral-Mini 3B"},{"name":"GLM-4-Voice 9B","short":"GLM-4-V","tag":"audio LLM","acc":35.2,"mf1":0.238,"subsample":true,"model_id":null,"model_version":"GLM-4-Voice 9B"},{"name":"data2vec-audio SER","short":"data2vec","tag":"open","acc":34.4,"mf1":0.229,"model_id":null,"model_version":"data2vec-audio SER"},{"name":"XLSR SER (harshit345)","short":"XLSR H345","tag":"open","acc":33.7,"mf1":0.286,"model_id":null,"model_version":"XLSR SER (harshit345)"},{"name":"GAMA 7B","short":"GAMA","tag":"audio LLM","acc":33.4,"mf1":0.221,"subsample":true,"model_id":null,"model_version":"GAMA 7B"},{"name":"Whisper SER (firdhokk)","short":"Whisper SER","tag":"open","acc":32.5,"mf1":0.276,"model_id":null,"model_version":"Whisper SER (firdhokk)"},{"name":"wav2vec2-base SUPERB","short":"w2v2-base","tag":"open","acc":31.4,"mf1":0.187,"model_id":null,"model_version":"wav2vec2-base SUPERB"},{"name":"Qwen2.5-Omni 7B","short":"Qwen Omni","tag":"audio LLM","acc":31.3,"mf1":0.068,"subsample":true,"evaluationNote":"Fixed 5,000-clip stratified subset; recorded in the 2026-07-14 run. Published values corrected on 2026-09-06 to match that run.","model_id":null,"model_version":"Qwen2.5-Omni 7B"},{"name":"LTU-AS 7B","short":"LTU-AS","tag":"audio LLM","acc":30.8,"mf1":0.198,"subsample":true,"model_id":null,"model_version":"LTU-AS 7B"},{"name":"openSMILE ComParE SVM","short":"openSMILE","tag":"open","acc":30.2,"mf1":0.203,"model_id":null,"model_version":"openSMILE ComParE SVM"},{"name":"DistilHuBERT SUPERB","short":"dHuBERT","tag":"open","acc":29.5,"mf1":0.186,"model_id":null,"model_version":"DistilHuBERT SUPERB"},{"name":"OpenVokaturi","short":"Vokaturi","tag":"open","acc":28.7,"mf1":0.192,"model_id":null,"model_version":"OpenVokaturi"},{"name":"CLAP zero-shot (LAION)","short":"CLAP ZS","tag":"open","acc":26.4,"mf1":0.177,"model_id":null,"model_version":"CLAP zero-shot (LAION)"},{"name":"XLS-R SER (firdhokk)","short":"XLS-R Fir","tag":"open","acc":26,"mf1":0.24,"model_id":null,"model_version":"XLS-R SER (firdhokk)"},{"name":"Empath API","short":"Empath","tag":"API","acc":25.3,"mf1":0.156,"subsample":true,"model_id":null,"model_version":"Empath API"},{"name":"Mini-Omni2","short":"Mini-Omni2","tag":"audio LLM","acc":24.6,"mf1":0.143,"subsample":true,"model_id":null,"model_version":"Mini-Omni2"},{"name":"XLS-R RAVDESS","short":"XLS-R RAV","tag":"open","acc":23.8,"mf1":0.118,"model_id":null,"model_version":"XLS-R RAVDESS"},{"name":"XLSR ShEMO (m3hrdadfi)","short":"XLSR ShEMO","tag":"open","acc":21.7,"mf1":0.148,"model_id":null,"model_version":"XLSR ShEMO (m3hrdadfi)"},{"name":"SpeechGPT","short":"SpeechGPT","tag":"audio LLM","acc":19.8,"mf1":0.117,"subsample":true,"model_id":null,"model_version":"SpeechGPT"},{"name":"AnyGPT","short":"AnyGPT","tag":"audio LLM","acc":17.4,"mf1":0.104,"subsample":true,"model_id":null,"model_version":"AnyGPT"},{"name":"wav2vec2 CTC SER (r-f)","short":"w2v2 CTC","tag":"open","acc":16,"mf1":0.098,"model_id":null,"model_version":"wav2vec2 CTC SER (r-f)"},{"name":"XLS-R Russian (Aniemore)","short":"XLS-R Ru","tag":"open","acc":9.1,"mf1":0.079,"model_id":null,"model_version":"XLS-R Russian (Aniemore)"}]},"limitations":["Resonance 1, July–August 2026 evaluation checkpoint. Results apply to the recorded build and protocol; they are not a new evaluation of subsequent API releases or Fourier.","Vendor-published reference rows were not rerun in the oruk harness.","Open emotion models use the full 64,384 clips; closed/API and audio-LLM systems use a fixed 5,000-clip stratified subset. Rows share a label mapping and scorer, not identical audio.","The Hume row is a legacy 48-dimension Expression Measurement prosody snapshot, not a measurement of current Hume Tagger or Prosody.","Raw audio is not redistributed where source licenses do not permit redistribution."]}