{
  "measured_on": "2026-09-12",
  "scope": "Local model accuracy and native engine capacity; archived pilot transport measurements are labeled separately. Current native-key public results: /benchmarks/orukeet-public-2026-09-12.json",
  "hardware": "Single A100 SXM4 40GB, us-central1-c",
  "accuracy": {
    "dataset": "Earnings22",
    "orukeet": {
      "records": 2741,
      "hours": 5.427600555555556,
      "words": 48899,
      "errors": 4990,
      "wer_percent": 10.204707662733389,
      "failed_requests": 0,
      "model_sha256": "9adaf30015fc4655a7fb5bba27a1d0eae0a361e83d186aff8fa6350f1c20d0e7",
      "manifest_sha256": "bf4838e399574452c46b3103fa9df3b642bc7e591bc2c1df25beb55db15678a8",
      "scorer_sha256": "e5343c36391fb8848cb8dc8b286314a2b5aedf079b38c7a70702e98e30311f37",
      "normalization": "Same Transformers Whisper EnglishTextNormalizer and English spelling map as original comparison",
      "concurrency": 32
    },
    "whisper_turbo_local": {
      "precision": "F16",
      "beam_size": 5,
      "words": 48899,
      "errors": 5584,
      "wer_percent": 11.419456430601853,
      "records": 2741
    },
    "manifest_sha256": "bf4838e399574452c46b3103fa9df3b642bc7e591bc2c1df25beb55db15678a8"
  },
  "public_streaming": {
    "results": {
      "direct": {
        "requests": 24,
        "errors": 0,
        "text_differences": 0,
        "latency_ms": {
          "p50": 95.93912499258295,
          "p95": 137.56084559718147,
          "p99": 329.06439393991593
        },
        "server_ms_median": 24.749156495090574
      },
      "tunnel": {
        "requests": 24,
        "errors": 0,
        "text_differences": 0,
        "latency_ms": {
          "p50": 127.22195801325142,
          "p95": 145.55319999344647,
          "p99": 146.1411876627244
        },
        "server_ms_median": 23.524063508375548
      }
    },
    "protocol": "Pooled all requests from two independent replay phases, six fixtures, two passes per phase; no outliers excluded; 20 ms PCM frames at real-time capture cadence; last source sample to full transcript",
    "phases": [
      "stream-public",
      "stream-public-recheck"
    ],
    "authentication": "Pilot service key; historical measurements, before native-account integration"
  },
  "native_throughput": {
    "base_url": "http://127.0.0.1:8080",
    "measured_at": "2026-09-12T15:23:24.970458+00:00",
    "manifest_sha256": "a9c3bcc42d1b0113071cd81ccac2170702334898d0d61ca86752325028b1baf5",
    "script_sha256": "214287be99243e0346525f7d69f1f4c574ed4f59ac2a4c10da30eb7263d72b9e",
    "unique_clips": 132,
    "results": [
      {
        "concurrency": 32,
        "run": 0,
        "requests": 10032,
        "successes": 10032,
        "errors": 0,
        "elapsed_s": 171.74941987902275,
        "successful_audio_s": 109711.1205,
        "audio_realtime_factor_x": 638.7859742249992,
        "requests_per_s": 58.41067764343171,
        "latency_ms": {
          "p50": 500.9460869914619,
          "p95": 1057.0526069262996,
          "p99": 1355.970695973374
        },
        "asr_ms": {
          "p50": 470.095,
          "p95": 1014.5129999999999,
          "p99": 1302.8230000000003
        },
        "transcript_mismatches": 115
      }
    ],
    "protocol": "closed-loop; warm model; preloaded audio; no response cache; includes HTTP/upload/decode/service queue/inference; successful audio only in throughput"
  },
  "hosted_whisper": {
    "nagaai": {
      "usd_per_audio_minute": "0.0001",
      "source": "https://naga.ac/models/whisper-large-v3-turbo/specifications"
    },
    "deepinfra": {
      "usd_per_audio_minute": "0.0002",
      "source": "https://deepinfra.com/openai/whisper-large-v3-turbo"
    }
  },
  "native_key_rest": {
    "measured_at": "2026-09-12T20:32:49Z",
    "worker_version": "f7595c99-229a-4e12-b46e-4c5149c02e2a",
    "fixture": {
      "duration": 1.88,
      "sha256": "6141b1079dc8d39f0741d185a3941d907a441ab6c61a7c3f2c8474d9cc013931",
      "source": "Public speech benchmark fixture; replayed from local client"
    },
    "n": 7,
    "warmup_requests_excluded": 1,
    "client_ms": {
      "median": 437.97604204155505,
      "p95": 617.6490205107256
    },
    "asr_ms_median": 11.751061014365405,
    "boundary": "Complete public REST request from one client, including native Oruk authentication, prepaid credit reservation, upload, inference, and response. Warm HTTP connection. No optional tasks. Small single-clip replay; not a global or sustained-load guarantee."
  }
}