[
 {
  "benchmark": "throughput",
  "engine": "nano",
  "model": "unsloth/Meta-Llama-3.1-8B-Instruct",
  "device": "cuda",
  "num_seqs": 256,
  "lengths": [
   100,
   1024
  ],
  "prompt_tokens": 145789,
  "output_tokens": 137547,
  "seconds": 27.53,
  "output_tok_s": 4997.0,
  "steps": 1029,
  "preemptions": 0,
  "cow_copies": 0,
  "tokens_run": 283080,
  "kv_blocks": 27662,
  "median_step_ms": 22.7,
  "slowest_steps_ms": [
   258.2,
   259.6,
   261.1,
   261.2,
   261.5
  ],
  "alloc_retries": 0,
  "gpu": "NVIDIA H100 80GB HBM3"
 },
 {
  "benchmark": "throughput",
  "engine": "vllm",
  "model": "unsloth/Meta-Llama-3.1-8B-Instruct",
  "device": "cuda",
  "num_seqs": 256,
  "lengths": [
   100,
   1024
  ],
  "prompt_tokens": 145789,
  "output_tokens": 137547,
  "seconds": 18.74,
  "output_tok_s": 7339.5,
  "eager": true,
  "gpu": "NVIDIA H100 80GB HBM3"
 },
 {
  "benchmark": "throughput",
  "engine": "vllm",
  "model": "unsloth/Meta-Llama-3.1-8B-Instruct",
  "device": "cuda",
  "num_seqs": 256,
  "lengths": [
   100,
   1024
  ],
  "prompt_tokens": 145789,
  "output_tokens": 137547,
  "seconds": 16.8,
  "output_tok_s": 8189.4,
  "eager": false,
  "gpu": "NVIDIA H100 80GB HBM3"
 },
 {
  "benchmark": "stall",
  "model": "unsloth/Meta-Llama-3.1-8B-Instruct",
  "device": "cuda",
  "decoders": 32,
  "long_prompt": 8000,
  "chunked": true,
  "token_budget": 512,
  "steady_step_ms": 16.7,
  "longest_step_ms": 24.9,
  "steps_to_first_token": 17,
  "long_prompt_ttft_ms": 389.0,
  "step_ms_after_arrival": [
   21.0,
   21.2,
   21.3,
   21.6,
   21.8,
   22.1,
   22.3,
   22.7,
   22.9,
   23.3,
   23.5,
   23.7,
   24.1,
   24.4,
   24.6,
   24.9,
   23.5,
   17.6,
   17.6,
   16.8,
   16.4,
   16.6,
   17.0,
   16.2,
   16.7,
   16.5,
   16.8
  ],
  "gpu": "NVIDIA H100 80GB HBM3"
 },
 {
  "benchmark": "stall",
  "model": "unsloth/Meta-Llama-3.1-8B-Instruct",
  "device": "cuda",
  "decoders": 32,
  "long_prompt": 8000,
  "chunked": true,
  "token_budget": 2048,
  "steady_step_ms": 16.5,
  "longest_step_ms": 76.0,
  "steps_to_first_token": 4,
  "long_prompt_ttft_ms": 284.6,
  "step_ms_after_arrival": [
   65.4,
   69.7,
   73.5,
   76.0,
   16.1,
   17.4,
   17.6,
   17.5,
   17.1,
   16.3,
   15.8,
   16.6,
   17.1,
   16.1
  ],
  "gpu": "NVIDIA H100 80GB HBM3"
 },
 {
  "benchmark": "stall",
  "model": "unsloth/Meta-Llama-3.1-8B-Instruct",
  "device": "cuda",
  "decoders": 32,
  "long_prompt": 8000,
  "chunked": false,
  "token_budget": 8032,
  "steady_step_ms": 17.1,
  "longest_step_ms": 265.0,
  "steps_to_first_token": 1,
  "long_prompt_ttft_ms": 265.0,
  "step_ms_after_arrival": [
   265.0,
   16.7,
   16.9,
   15.8,
   17.0,
   16.6,
   15.5,
   16.4,
   16.2,
   16.3,
   16.8
  ],
  "gpu": "NVIDIA H100 80GB HBM3"
 }
]
