{
 "vllm": "0.32.0",
 "target": "unsloth/Meta-Llama-3.1-8B-Instruct",
 "gpu": "NVIDIA A10G",
 "configs": [
  {
   "config": "off",
   "speculative_config": null,
   "startup_s": 304.2,
   "kv_pool_tokens": 29760,
   "runs": [
    {
     "workload": "chat",
     "concurrency": 1,
     "requests": 4,
     "seconds": 33.89,
     "output_tokens": 1019,
     "output_tok_s": 30.1,
     "prompt_tokens_p50": 63,
     "completion_tokens_p50": 255,
     "request_decode_tok_s_p50": 30.2,
     "ttft_ms_p50": 73.1,
     "spec": {}
    },
    {
     "workload": "chat",
     "concurrency": 16,
     "requests": 32,
     "seconds": 19.05,
     "output_tokens": 8088,
     "output_tok_s": 424.5,
     "prompt_tokens_p50": 63,
     "completion_tokens_p50": 256,
     "request_decode_tok_s_p50": 27.2,
     "ttft_ms_p50": 134.9,
     "spec": {}
    },
    {
     "workload": "code",
     "concurrency": 1,
     "requests": 4,
     "seconds": 41.04,
     "output_tokens": 1223,
     "output_tok_s": 29.8,
     "prompt_tokens_p50": 374,
     "completion_tokens_p50": 306,
     "request_decode_tok_s_p50": 30.1,
     "ttft_ms_p50": 151.1,
     "spec": {}
    },
    {
     "workload": "code",
     "concurrency": 16,
     "requests": 32,
     "seconds": 26.11,
     "output_tokens": 9729,
     "output_tok_s": 372.6,
     "prompt_tokens_p50": 367,
     "completion_tokens_p50": 306,
     "request_decode_tok_s_p50": 25.0,
     "ttft_ms_p50": 535.8,
     "spec": {}
    },
    {
     "workload": "chat_t1",
     "concurrency": 1,
     "requests": 4,
     "seconds": 35.77,
     "output_tokens": 1001,
     "output_tok_s": 28.0,
     "prompt_tokens_p50": 62,
     "completion_tokens_p50": 256,
     "request_decode_tok_s_p50": 30.1,
     "ttft_ms_p50": 47.5,
     "spec": {}
    }
   ],
   "server_log": [
    "(EngineCore pid=97) INFO 11-11 05:31:35 [core.py:133] Initializing a V1 LLM engine (v0.32.0) with config: model='unsloth/Meta-Llama-3.1-8B-Instruct', speculative_config=None, tokenizer='unsloth/Meta-Llama-3.1-8B-Instruct', skip_tokenizer_init=False, tokenizer_mode=auto, revision=main, tokenizer_revi"
   ]
  },
  {
   "config": "ngram gpu",
   "speculative_config": {
    "method": "ngram_gpu",
    "num_speculative_tokens": 5,
    "prompt_lookup_min": 2,
    "prompt_lookup_max": 5
   },
   "startup_s": 120.1,
   "kv_pool_tokens": 29056,
   "runs": [
    {
     "workload": "chat",
     "concurrency": 1,
     "requests": 4,
     "seconds": 33.12,
     "output_tokens": 1014,
     "output_tok_s": 30.6,
     "prompt_tokens_p50": 63,
     "completion_tokens_p50": 256,
     "request_decode_tok_s_p50": 30.9,
     "ttft_ms_p50": 47.4,
     "spec": {
      "drafts": 948,
      "draft_tokens": 4740,
      "accepted_tokens": 63,
      "mean_acceptance_length": 1.066,
      "draft_acceptance_rate": 0.0133,
      "draft_tokens_per_draft": 5.0,
      "accepted_through_position": [
       0.039,
       0.0158,
       0.0084,
       0.0021,
       0.0011
      ]
     },
     "answers_identical_to_off": 0,
     "answers_shared_prefix": [
      0.228,
      0.65,
      0.13,
      0.08
     ]
    },
    {
     "workload": "chat",
     "concurrency": 16,
     "requests": 32,
     "seconds": 20.49,
     "output_tokens": 8043,
     "output_tok_s": 392.5,
     "prompt_tokens_p50": 63,
     "completion_tokens_p50": 256,
     "request_decode_tok_s_p50": 25.4,
     "ttft_ms_p50": 154.8,
     "spec": {
      "drafts": 7569,
      "draft_tokens": 37845,
      "accepted_tokens": 447,
      "mean_acceptance_length": 1.059,
      "draft_acceptance_rate": 0.0118,
      "draft_tokens_per_draft": 5.0,
      "accepted_through_position": [
       0.0354,
       0.0118,
       0.0061,
       0.0034,
       0.0024
      ]
     }
    },
    {
     "workload": "code",
     "concurrency": 1,
     "requests": 4,
     "seconds": 10.13,
     "output_tokens": 1223,
     "output_tok_s": 120.7,
     "prompt_tokens_p50": 374,
     "completion_tokens_p50": 306,
     "request_decode_tok_s_p50": 128.8,
     "ttft_ms_p50": 152.5,
     "spec": {
      "drafts": 271,
      "draft_tokens": 1355,
      "accepted_tokens": 948,
      "mean_acceptance_length": 4.498,
      "draft_acceptance_rate": 0.6996,
      "draft_tokens_per_draft": 5.0,
      "accepted_through_position": [
       0.738,
       0.7011,
       0.6937,
       0.6827,
       0.6827
      ]
     },
     "answers_identical_to_off": 4,
     "answers_shared_prefix": [
      1.0,
      1.0,
      1.0,
      1.0
     ]
    },
    {
     "workload": "code",
     "concurrency": 16,
     "requests": 32,
     "seconds": 8.8,
     "output_tokens": 9729,
     "output_tok_s": 1105.4,
     "prompt_tokens_p50": 367,
     "completion_tokens_p50": 306,
     "request_decode_tok_s_p50": 89.0,
     "ttft_ms_p50": 589.6,
     "spec": {
      "drafts": 2023,
      "draft_tokens": 10115,
      "accepted_tokens": 7674,
      "mean_acceptance_length": 4.793,
      "draft_acceptance_rate": 0.7587,
      "draft_tokens_per_draft": 5.0,
      "accepted_through_position": [
       0.8067,
       0.7588,
       0.7538,
       0.7375,
       0.7365
      ]
     }
    },
    {
     "workload": "chat_t1",
     "concurrency": 1,
     "requests": 4,
     "seconds": 34.62,
     "output_tokens": 964,
     "output_tok_s": 27.8,
     "prompt_tokens_p50": 62,
     "completion_tokens_p50": 250,
     "request_decode_tok_s_p50": 30.2,
     "ttft_ms_p50": 78.1,
     "spec": {
      "drafts": 914,
      "draft_tokens": 4570,
      "accepted_tokens": 46,
      "mean_acceptance_length": 1.05,
      "draft_acceptance_rate": 0.0101,
      "draft_tokens_per_draft": 5.0,
      "accepted_through_position": [
       0.0339,
       0.0088,
       0.0044,
       0.0022,
       0.0011
      ]
     }
    }
   ],
   "server_log": [
    "(APIServer pid=448) INFO 11-11 05:40:43 [metrics.py:120] SpecDecoding metrics: Mean acceptance length: 1.34, Accepted throughput: 3.89 tokens/s, Drafted throughput: 57.91 tokens/s, Accepted: 46 tokens, Drafted: 685 tokens, Per-position acceptance rate: 0.095, 0.066, 0.058, 0.058, 0.058, Avg Draft ac",
    "(APIServer pid=448) INFO 11-11 05:40:53 [metrics.py:120] SpecDecoding metrics: Mean acceptance length: 1.07, Accepted throughput: 1.90 tokens/s, Drafted throughput: 143.48 tokens/s, Accepted: 19 tokens, Drafted: 1435 tokens, Per-position acceptance rate: 0.031, 0.017, 0.010, 0.003, 0.003, Avg Draft ",
    "(APIServer pid=448) INFO 11-11 05:41:03 [metrics.py:120] SpecDecoding metrics: Mean acceptance length: 1.06, Accepted throughput: 1.70 tokens/s, Drafted throughput: 142.96 tokens/s, Accepted: 17 tokens, Drafted: 1430 tokens, Per-position acceptance rate: 0.038, 0.014, 0.007, 0.000, 0.000, Avg Draft ",
    "(APIServer pid=448) INFO 11-11 05:41:13 [metrics.py:120] SpecDecoding metrics: Mean acceptance length: 1.07, Accepted throughput: 2.10 tokens/s, Drafted throughput: 143.48 tokens/s, Accepted: 21 tokens, Drafted: 1435 tokens, Per-position acceptance rate: 0.045, 0.017, 0.007, 0.003, 0.000, Avg Draft ",
    "(APIServer pid=448) INFO 11-11 05:41:23 [metrics.py:120] SpecDecoding metrics: Mean acceptance length: 1.04, Accepted throughput: 9.40 tokens/s, Drafted throughput: 1304.55 tokens/s, Accepted: 94 tokens, Drafted: 13050 tokens, Per-position acceptance rate: 0.022, 0.006, 0.004, 0.002, 0.002, Avg Draf",
    "(APIServer pid=448) INFO 11-11 05:41:33 [metrics.py:120] SpecDecoding metrics: Mean acceptance length: 1.06, Accepted throughput: 22.70 tokens/s, Drafted throughput: 1887.59 tokens/s, Accepted: 227 tokens, Drafted: 18880 tokens, Per-position acceptance rate: 0.036, 0.012, 0.006, 0.003, 0.002, Avg Dr",
    "(APIServer pid=448) INFO 11-11 05:41:43 [metrics.py:120] SpecDecoding metrics: Mean acceptance length: 1.42, Accepted throughput: 56.90 tokens/s, Drafted throughput: 679.45 tokens/s, Accepted: 569 tokens, Drafted: 6795 tokens, Per-position acceptance rate: 0.124, 0.085, 0.074, 0.068, 0.067, Avg Draf",
    "(APIServer pid=448) INFO 11-11 05:41:53 [metrics.py:120] SpecDecoding metrics: Mean acceptance length: 4.73, Accepted throughput: 306.33 tokens/s, Drafted throughput: 410.40 tokens/s, Accepted: 3064 tokens, Drafted: 4105 tokens, Per-position acceptance rate: 0.767, 0.749, 0.744, 0.736, 0.736, Avg Dr",
    "(APIServer pid=448) INFO 11-11 05:42:03 [metrics.py:120] SpecDecoding metrics: Mean acceptance length: 4.79, Accepted throughput: 511.92 tokens/s, Drafted throughput: 675.89 tokens/s, Accepted: 5120 tokens, Drafted: 6760 tokens, Per-position acceptance rate: 0.821, 0.757, 0.751, 0.730, 0.729, Avg Dr",
    "(APIServer pid=448) INFO 11-11 05:42:13 [metrics.py:120] SpecDecoding metrics: Mean acceptance length: 1.05, Accepted throughput: 1.30 tokens/s, Drafted throughput: 142.48 tokens/s, Accepted: 13 tokens, Drafted: 1425 tokens, Per-position acceptance rate: 0.042, 0.004, 0.000, 0.000, 0.000, Avg Draft ",
    "(APIServer pid=448) INFO 11-11 05:42:23 [metrics.py:120] SpecDecoding metrics: Mean acceptance length: 1.07, Accepted throughput: 2.10 tokens/s, Drafted throughput: 142.99 tokens/s, Accepted: 21 tokens, Drafted: 1430 tokens, Per-position acceptance rate: 0.038, 0.021, 0.010, 0.003, 0.000, Avg Draft ",
    "(APIServer pid=448) INFO 11-11 05:42:33 [metrics.py:120] SpecDecoding metrics: Mean acceptance length: 1.04, Accepted throughput: 1.00 tokens/s, Drafted throughput: 141.98 tokens/s, Accepted: 10 tokens, Drafted: 1420 tokens, Per-position acceptance rate: 0.021, 0.004, 0.004, 0.004, 0.004, Avg Draft "
   ]
  },
  {
   "config": "eagle3 k=3",
   "speculative_config": {
    "method": "eagle3",
    "model": "RedHatAI/Llama-3.1-8B-Instruct-speculator.eagle3",
    "num_speculative_tokens": 3
   },
   "startup_s": 150.1,
   "kv_pool_tokens": 10000,
   "runs": [
    {
     "workload": "chat",
     "concurrency": 1,
     "requests": 4,
     "seconds": 16.8,
     "output_tokens": 1009,
     "output_tok_s": 60.1,
     "prompt_tokens_p50": 63,
     "completion_tokens_p50": 256,
     "request_decode_tok_s_p50": 60.4,
     "ttft_ms_p50": 82.5,
     "spec": {
      "drafts": 411,
      "draft_tokens": 1233,
      "accepted_tokens": 598,
      "mean_acceptance_length": 2.455,
      "draft_acceptance_rate": 0.485,
      "draft_tokens_per_draft": 3.0,
      "accepted_through_position": [
       0.7032,
       0.4453,
       0.3066
      ]
     },
     "answers_identical_to_off": 0,
     "answers_shared_prefix": [
      0.228,
      0.804,
      0.13,
      0.509
     ]
    },
    {
     "workload": "chat",
     "concurrency": 16,
     "requests": 32,
     "seconds": 10.73,
     "output_tokens": 8037,
     "output_tok_s": 749.2,
     "prompt_tokens_p50": 63,
     "completion_tokens_p50": 256,
     "request_decode_tok_s_p50": 52.3,
     "ttft_ms_p50": 173.9,
     "spec": {
      "drafts": 3296,
      "draft_tokens": 9888,
      "accepted_tokens": 4747,
      "mean_acceptance_length": 2.44,
      "draft_acceptance_rate": 0.4801,
      "draft_tokens_per_draft": 3.0,
      "accepted_through_position": [
       0.6954,
       0.4609,
       0.284
      ]
     }
    },
    {
     "workload": "code",
     "concurrency": 1,
     "requests": 4,
     "seconds": 14.71,
     "output_tokens": 1224,
     "output_tok_s": 83.2,
     "prompt_tokens_p50": 374,
     "completion_tokens_p50": 306,
     "request_decode_tok_s_p50": 87.0,
     "ttft_ms_p50": 161.8,
     "spec": {
      "drafts": 346,
      "draft_tokens": 1038,
      "accepted_tokens": 886,
      "mean_acceptance_length": 3.561,
      "draft_acceptance_rate": 0.8536,
      "draft_tokens_per_draft": 3.0,
      "accepted_through_position": [
       0.9249,
       0.8497,
       0.7861
      ]
     },
     "answers_identical_to_off": 3,
     "answers_shared_prefix": [
      1.0,
      0.447,
      1.0,
      1.0
     ]
    },
    {
     "workload": "code",
     "concurrency": 16,
     "requests": 32,
     "seconds": 11.57,
     "output_tokens": 9729,
     "output_tok_s": 841.1,
     "prompt_tokens_p50": 367,
     "completion_tokens_p50": 306,
     "request_decode_tok_s_p50": 62.9,
     "ttft_ms_p50": 625.3,
     "spec": {
      "drafts": 2679,
      "draft_tokens": 8037,
      "accepted_tokens": 7091,
      "mean_acceptance_length": 3.647,
      "draft_acceptance_rate": 0.8823,
      "draft_tokens_per_draft": 3.0,
      "accepted_through_position": [
       0.9436,
       0.882,
       0.8212
      ]
     }
    },
    {
     "workload": "chat_t1",
     "concurrency": 1,
     "requests": 4,
     "seconds": 19.15,
     "output_tokens": 998,
     "output_tok_s": 52.1,
     "prompt_tokens_p50": 62,
     "completion_tokens_p50": 255,
     "request_decode_tok_s_p50": 59.7,
     "ttft_ms_p50": 83.2,
     "spec": {
      "drafts": 409,
      "draft_tokens": 1227,
      "accepted_tokens": 594,
      "mean_acceptance_length": 2.452,
      "draft_acceptance_rate": 0.4841,
      "draft_tokens_per_draft": 3.0,
      "accepted_through_position": [
       0.7042,
       0.467,
       0.2812
      ]
     }
    }
   ],
   "server_log": [
    "(EngineCore pid=708) INFO 11-11 05:44:22 [decorators.py:743] [eagle_head] saved AOT compiled function to /root/.cache/vllm/torch_compile_cache/torch_aot_compile/ecc1e13a08a99eae469143fd0d3b2049753fe4a42bcd52f8446d5fc26dc0adec/rank_0_0/model",
    "(EngineCore pid=708) INFO 11-11 05:44:22 [monitor.py:62] [eagle_head] torch.compile took 4.62 s in total",
    "(EngineCore pid=708) INFO 11-11 05:44:22 [monitor.py:91] [eagle_head] Initial profiling/warmup run took 0.00 s",
    "(EngineCore pid=708) WARNING 11-11 05:45:10 [vllm.py:2276] max_num_scheduled_tokens is set to 2048 based on the speculative decoding settings. This may lead to suboptimal performance. Consider increasing max_num_batched_tokens to accommodate the additional draft token slots, or decrease num_speculat",
    "(APIServer pid=672) INFO 11-11 05:45:21 [metrics.py:120] SpecDecoding metrics: Mean acceptance length: 2.68, Accepted throughput: 21.29 tokens/s, Drafted throughput: 38.06 tokens/s, Accepted: 250 tokens, Drafted: 447 tokens, Per-position acceptance rate: 0.772, 0.530, 0.376, Avg Draft acceptance rat",
    "(APIServer pid=672) INFO 11-11 05:45:31 [metrics.py:120] SpecDecoding metrics: Mean acceptance length: 2.40, Accepted throughput: 34.30 tokens/s, Drafted throughput: 73.49 tokens/s, Accepted: 343 tokens, Drafted: 735 tokens, Per-position acceptance rate: 0.694, 0.420, 0.286, Avg Draft acceptance rat",
    "(APIServer pid=672) INFO 11-11 05:45:41 [metrics.py:120] SpecDecoding metrics: Mean acceptance length: 2.46, Accepted throughput: 282.42 tokens/s, Drafted throughput: 581.24 tokens/s, Accepted: 2825 tokens, Drafted: 5814 tokens, Per-position acceptance rate: 0.696, 0.466, 0.296, Avg Draft acceptance",
    "(APIServer pid=672) INFO 11-11 05:45:51 [metrics.py:120] SpecDecoding metrics: Mean acceptance length: 2.49, Accepted throughput: 224.20 tokens/s, Drafted throughput: 452.39 tokens/s, Accepted: 2242 tokens, Drafted: 4524 tokens, Per-position acceptance rate: 0.709, 0.479, 0.299, Avg Draft acceptance",
    "(APIServer pid=672) INFO 11-11 05:46:01 [metrics.py:120] SpecDecoding metrics: Mean acceptance length: 3.56, Accepted throughput: 60.00 tokens/s, Drafted throughput: 70.20 tokens/s, Accepted: 600 tokens, Drafted: 702 tokens, Per-position acceptance rate: 0.923, 0.846, 0.795, Avg Draft acceptance rat",
    "(APIServer pid=672) INFO 11-11 05:46:11 [metrics.py:120] SpecDecoding metrics: Mean acceptance length: 3.63, Accepted throughput: 398.41 tokens/s, Drafted throughput: 454.40 tokens/s, Accepted: 3985 tokens, Drafted: 4545 tokens, Per-position acceptance rate: 0.939, 0.873, 0.818, Avg Draft acceptance",
    "(APIServer pid=672) INFO 11-11 05:46:21 [metrics.py:120] SpecDecoding metrics: Mean acceptance length: 3.62, Accepted throughput: 323.97 tokens/s, Drafted throughput: 371.06 tokens/s, Accepted: 3240 tokens, Drafted: 3711 tokens, Per-position acceptance rate: 0.942, 0.875, 0.803, Avg Draft acceptance",
    "(APIServer pid=672) INFO 11-11 05:46:31 [metrics.py:120] SpecDecoding metrics: Mean acceptance length: 2.38, Accepted throughput: 33.70 tokens/s, Drafted throughput: 73.19 tokens/s, Accepted: 337 tokens, Drafted: 732 tokens, Per-position acceptance rate: 0.660, 0.447, 0.275, Avg Draft acceptance rat"
   ]
  }
 ]
}
