[ { "case": "ladder-0", "kind": "ladder", "target_tokens": 0, "prompt_sha256": "3965f79a076c8829b8f9083b6f023262873c2a459645e1f0f979e0f3ddb28460", "input_ids_sha256": "1f563f25a0c1605cb77f6cd9e6f55a0a40c46f09abd4722cf438b1b86a5c1d52", "prompt_tokens": 11, "output_tokens": 1000, "expected_llama_prompt_tokens": 11, "start_monotonic_s": 1196080.27718086, "end_monotonic_s": 1196101.213703163, "api_ttft_s": 0.04309014789760113, "api_e2e_s": 20.936522302916273, "api_tail_s": 20.89335623406805, "first_event_tokens": 1, "api_tail_tok_s": 47.81424242272105, "finish_reason": "length", "fresh": false, "usage": { "prompt_tokens": 11, "total_tokens": 1011, "completion_tokens": 1000 }, "engine_metric_isolated": true, "engine_prefill_s": 0.04048360092565417, "engine_decode_s": 20.89348189602606, "engine_computed_prompt_tokens": 11.0, "engine_prefill_tok_s": 271.7149598475905, "engine_decode_tok_s": 47.8139548482826, "engine_timing_note": "prefill=first scheduled to first new token; decode=first to last new token", "metrics_delta": { "vllm:spec_decode_num_drafts_total": 595.0, "vllm:spec_decode_num_draft_tokens_total": 4165.0, "vllm:spec_decode_num_accepted_tokens_total": 405.0, "vllm:spec_decode_num_accepted_tokens_per_pos_total": 405.0, "vllm:estimated_flops_per_gpu_total": 0.0, "vllm:estimated_read_bytes_per_gpu_total": 0.0, "vllm:estimated_write_bytes_per_gpu_total": 0.0, "vllm:prefix_cache_queries_total": 11.0, "vllm:prefix_cache_hits_total": 0.0, "vllm:external_prefix_cache_queries_total": 0.0, "vllm:external_prefix_cache_hits_total": 0.0, "vllm:mm_cache_queries_total": 0.0, "vllm:mm_cache_hits_total": 0.0, "vllm:num_preemptions_total": 0.0, "vllm:prompt_tokens_total": 11.0, "vllm:prompt_tokens_by_source_total": 11.0, "vllm:prompt_tokens_cached_total": 0.0, "vllm:generation_tokens_total": 1000.0, "vllm:request_success_total": 1.0, "vllm:request_prompt_tokens_count": 1.0, "vllm:request_prompt_tokens_sum": 11.0, "vllm:request_generation_tokens_count": 1.0, "vllm:request_generation_tokens_sum": 1000.0, "vllm:iteration_tokens_total_count": 596.0, "vllm:iteration_tokens_total_sum": 1011.0, "vllm:request_max_num_generation_tokens_count": 1.0, "vllm:request_max_num_generation_tokens_sum": 1000.0, "vllm:request_params_n_count": 1.0, "vllm:request_params_n_sum": 1.0, "vllm:request_params_max_tokens_count": 1.0, "vllm:request_params_max_tokens_sum": 1000.0, "vllm:time_to_first_token_seconds_count": 1.0, "vllm:time_to_first_token_seconds_sum": 0.04205942153930664, "vllm:inter_token_latency_seconds_count": 595.0, "vllm:inter_token_latency_seconds_sum": 20.89348189602606, "vllm:request_time_per_output_token_seconds_count": 1.0, "vllm:request_time_per_output_token_seconds_sum": 0.02091439629231838, "vllm:e2e_request_latency_seconds_count": 1.0, "vllm:e2e_request_latency_seconds_sum": 20.93531608581543, "vllm:request_queue_time_seconds_count": 1.0, "vllm:request_queue_time_seconds_sum": 1.6110017895698547e-05, "vllm:request_inference_time_seconds_count": 1.0, "vllm:request_inference_time_seconds_sum": 20.933965496951714, "vllm:request_prefill_time_seconds_count": 1.0, "vllm:request_prefill_time_seconds_sum": 0.04048360092565417, "vllm:request_decode_time_seconds_count": 1.0, "vllm:request_decode_time_seconds_sum": 20.89348189602606, "vllm:request_prefill_kv_computed_tokens_count": 1.0, "vllm:request_prefill_kv_computed_tokens_sum": 11.0, "vllm:tool_call_parser_invocations_total": 0.0 }, "repetition": 1 }, { "case": "ladder-8000", "kind": "ladder", "target_tokens": 8000, "prompt_sha256": "ef9d16376787f5d95c40f8f6f0ee77d902e03a9173cf110c21007bcdec639e98", "input_ids_sha256": "8adc690f4ef99b0b9b6ff75ad2e742bcf2737662529af5706170a23202bc7753", "prompt_tokens": 7887, "output_tokens": 1000, "expected_llama_prompt_tokens": 7887, "start_monotonic_s": 1196101.242522404, "end_monotonic_s": 1196108.251923461, "api_ttft_s": 2.4032898978330195, "api_e2e_s": 7.009401056915522, "api_tail_s": 4.606037628138438, "first_event_tokens": 1, "api_tail_tok_s": 216.8892398744369, "finish_reason": "length", "fresh": false, "usage": { "prompt_tokens": 7887, "total_tokens": 8887, "completion_tokens": 1000 }, "engine_metric_isolated": true, "engine_prefill_s": 2.395547545980662, "engine_decode_s": 4.6096581681631505, "engine_computed_prompt_tokens": 7887.0, "engine_prefill_tok_s": 3292.3579468222615, "engine_decode_tok_s": 216.71888967812117, "engine_timing_note": "prefill=first scheduled to first new token; decode=first to last new token", "metrics_delta": { "vllm:spec_decode_num_drafts_total": 126.0, "vllm:spec_decode_num_draft_tokens_total": 882.0, "vllm:spec_decode_num_accepted_tokens_total": 879.0, "vllm:spec_decode_num_accepted_tokens_per_pos_total": 879.0, "vllm:estimated_flops_per_gpu_total": 0.0, "vllm:estimated_read_bytes_per_gpu_total": 0.0, "vllm:estimated_write_bytes_per_gpu_total": 0.0, "vllm:prefix_cache_queries_total": 7887.0, "vllm:prefix_cache_hits_total": 0.0, "vllm:external_prefix_cache_queries_total": 0.0, "vllm:external_prefix_cache_hits_total": 0.0, "vllm:mm_cache_queries_total": 0.0, "vllm:mm_cache_hits_total": 0.0, "vllm:num_preemptions_total": 0.0, "vllm:prompt_tokens_total": 7887.0, "vllm:prompt_tokens_by_source_total": 7887.0, "vllm:prompt_tokens_cached_total": 0.0, "vllm:generation_tokens_total": 1000.0, "vllm:request_success_total": 1.0, "vllm:request_prompt_tokens_count": 1.0, "vllm:request_prompt_tokens_sum": 7887.0, "vllm:request_generation_tokens_count": 1.0, "vllm:request_generation_tokens_sum": 1000.0, "vllm:iteration_tokens_total_count": 127.0, "vllm:iteration_tokens_total_sum": 8887.0, "vllm:request_max_num_generation_tokens_count": 1.0, "vllm:request_max_num_generation_tokens_sum": 1000.0, "vllm:request_params_n_count": 1.0, "vllm:request_params_n_sum": 1.0, "vllm:request_params_max_tokens_count": 1.0, "vllm:request_params_max_tokens_sum": 1000.0, "vllm:time_to_first_token_seconds_count": 1.0, "vllm:time_to_first_token_seconds_sum": 2.3977553844451904, "vllm:inter_token_latency_seconds_count": 126.0, "vllm:inter_token_latency_seconds_sum": 4.6096581681631505, "vllm:request_time_per_output_token_seconds_count": 1.0, "vllm:request_time_per_output_token_seconds_sum": 0.0046142724406037575, "vllm:e2e_request_latency_seconds_count": 1.0, "vllm:e2e_request_latency_seconds_sum": 7.007390260696411, "vllm:request_queue_time_seconds_count": 1.0, "vllm:request_queue_time_seconds_sum": 1.5829922631382942e-05, "vllm:request_inference_time_seconds_count": 1.0, "vllm:request_inference_time_seconds_sum": 7.005205714143813, "vllm:request_prefill_time_seconds_count": 1.0, "vllm:request_prefill_time_seconds_sum": 2.395547545980662, "vllm:request_decode_time_seconds_count": 1.0, "vllm:request_decode_time_seconds_sum": 4.6096581681631505, "vllm:request_prefill_kv_computed_tokens_count": 1.0, "vllm:request_prefill_kv_computed_tokens_sum": 7887.0, "vllm:tool_call_parser_invocations_total": 0.0 }, "repetition": 1 }, { "case": "ladder-16000", "kind": "ladder", "target_tokens": 16000, "prompt_sha256": "17610c10a63b400e07668fc3a875a67548796ea1dd18e67702fb066da804a2b8", "input_ids_sha256": "179aac711afb83a653889e3dbf76a4c9367280452995ef638bbb341c717662da", "prompt_tokens": 16050, "output_tokens": 1000, "expected_llama_prompt_tokens": 16050, "start_monotonic_s": 1196108.300297856, "end_monotonic_s": 1196116.090706973, "api_ttft_s": 3.128112129867077, "api_e2e_s": 7.790409117005765, "api_tail_s": 4.662218424957246, "first_event_tokens": 1, "api_tail_tok_s": 214.27567499889523, "finish_reason": "length", "fresh": false, "usage": { "prompt_tokens": 16050, "total_tokens": 17050, "completion_tokens": 1000 }, "engine_metric_isolated": true, "engine_prefill_s": 3.1135875680483878, "engine_decode_s": 4.671008790144697, "engine_computed_prompt_tokens": 9890.0, "engine_prefill_tok_s": 3176.400144158817, "engine_decode_tok_s": 213.87242989304102, "engine_timing_note": "prefill=first scheduled to first new token; decode=first to last new token", "metrics_delta": { "vllm:spec_decode_num_drafts_total": 126.0, "vllm:spec_decode_num_draft_tokens_total": 882.0, "vllm:spec_decode_num_accepted_tokens_total": 879.0, "vllm:spec_decode_num_accepted_tokens_per_pos_total": 879.0, "vllm:estimated_flops_per_gpu_total": 0.0, "vllm:estimated_read_bytes_per_gpu_total": 0.0, "vllm:estimated_write_bytes_per_gpu_total": 0.0, "vllm:prefix_cache_queries_total": 16050.0, "vllm:prefix_cache_hits_total": 6160.0, "vllm:external_prefix_cache_queries_total": 0.0, "vllm:external_prefix_cache_hits_total": 0.0, "vllm:mm_cache_queries_total": 0.0, "vllm:mm_cache_hits_total": 0.0, "vllm:num_preemptions_total": 0.0, "vllm:prompt_tokens_total": 16050.0, "vllm:prompt_tokens_by_source_total": 16050.0, "vllm:prompt_tokens_cached_total": 6160.0, "vllm:generation_tokens_total": 1000.0, "vllm:request_success_total": 1.0, "vllm:request_prompt_tokens_count": 1.0, "vllm:request_prompt_tokens_sum": 16050.0, "vllm:request_generation_tokens_count": 1.0, "vllm:request_generation_tokens_sum": 1000.0, "vllm:iteration_tokens_total_count": 127.0, "vllm:iteration_tokens_total_sum": 10890.0, "vllm:request_max_num_generation_tokens_count": 1.0, "vllm:request_max_num_generation_tokens_sum": 1000.0, "vllm:request_params_n_count": 1.0, "vllm:request_params_n_sum": 1.0, "vllm:request_params_max_tokens_count": 1.0, "vllm:request_params_max_tokens_sum": 1000.0, "vllm:time_to_first_token_seconds_count": 1.0, "vllm:time_to_first_token_seconds_sum": 3.1167991161346436, "vllm:inter_token_latency_seconds_count": 126.0, "vllm:inter_token_latency_seconds_sum": 4.671008790144697, "vllm:request_time_per_output_token_seconds_count": 1.0, "vllm:request_time_per_output_token_seconds_sum": 0.0046756844746193155, "vllm:e2e_request_latency_seconds_count": 1.0, "vllm:e2e_request_latency_seconds_sum": 7.7875425815582275, "vllm:request_queue_time_seconds_count": 1.0, "vllm:request_queue_time_seconds_sum": 1.5969853848218918e-05, "vllm:request_inference_time_seconds_count": 1.0, "vllm:request_inference_time_seconds_sum": 7.784596358193085, "vllm:request_prefill_time_seconds_count": 1.0, "vllm:request_prefill_time_seconds_sum": 3.1135875680483878, "vllm:request_decode_time_seconds_count": 1.0, "vllm:request_decode_time_seconds_sum": 4.671008790144697, "vllm:request_prefill_kv_computed_tokens_count": 1.0, "vllm:request_prefill_kv_computed_tokens_sum": 9890.0, "vllm:tool_call_parser_invocations_total": 0.0 }, "repetition": 1 }, { "case": "ladder-32000", "kind": "ladder", "target_tokens": 32000, "prompt_sha256": "d2b2c90bc484f3c1ca1711f0a2bac03a3ea0f5c9234d48231dc746d998114fb8", "input_ids_sha256": "f6305d07154b3068b74b6ca47f8d0755c1fa9943495ffb02c16fa4c36d0a5abf", "prompt_tokens": 32460, "output_tokens": 1000, "expected_llama_prompt_tokens": 32460, "start_monotonic_s": 1196116.169815519, "end_monotonic_s": 1196130.134734322, "api_ttft_s": 8.26323701418005, "api_e2e_s": 13.964918803190812, "api_tail_s": 5.701603097841144, "first_event_tokens": 1, "api_tail_tok_s": 175.21387982587942, "finish_reason": "length", "fresh": false, "usage": { "prompt_tokens": 32460, "total_tokens": 33460, "completion_tokens": 1000 }, "engine_metric_isolated": true, "engine_prefill_s": 8.232100912835449, "engine_decode_s": 5.7214583090972155, "engine_computed_prompt_tokens": 17500.0, "engine_prefill_tok_s": 2125.8242804961355, "engine_decode_tok_s": 174.60583404262042, "engine_timing_note": "prefill=first scheduled to first new token; decode=first to last new token", "metrics_delta": { "vllm:spec_decode_num_drafts_total": 136.0, "vllm:spec_decode_num_draft_tokens_total": 952.0, "vllm:spec_decode_num_accepted_tokens_total": 865.0, "vllm:spec_decode_num_accepted_tokens_per_pos_total": 865.0, "vllm:estimated_flops_per_gpu_total": 0.0, "vllm:estimated_read_bytes_per_gpu_total": 0.0, "vllm:estimated_write_bytes_per_gpu_total": 0.0, "vllm:prefix_cache_queries_total": 32460.0, "vllm:prefix_cache_hits_total": 14960.0, "vllm:external_prefix_cache_queries_total": 0.0, "vllm:external_prefix_cache_hits_total": 0.0, "vllm:mm_cache_queries_total": 0.0, "vllm:mm_cache_hits_total": 0.0, "vllm:num_preemptions_total": 0.0, "vllm:prompt_tokens_total": 32460.0, "vllm:prompt_tokens_by_source_total": 32460.0, "vllm:prompt_tokens_cached_total": 14960.0, "vllm:generation_tokens_total": 1000.0, "vllm:request_success_total": 1.0, "vllm:request_prompt_tokens_count": 1.0, "vllm:request_prompt_tokens_sum": 32460.0, "vllm:request_generation_tokens_count": 1.0, "vllm:request_generation_tokens_sum": 1000.0, "vllm:iteration_tokens_total_count": 137.0, "vllm:iteration_tokens_total_sum": 18500.0, "vllm:request_max_num_generation_tokens_count": 1.0, "vllm:request_max_num_generation_tokens_sum": 1000.0, "vllm:request_params_n_count": 1.0, "vllm:request_params_n_sum": 1.0, "vllm:request_params_max_tokens_count": 1.0, "vllm:request_params_max_tokens_sum": 1000.0, "vllm:time_to_first_token_seconds_count": 1.0, "vllm:time_to_first_token_seconds_sum": 8.239188194274902, "vllm:inter_token_latency_seconds_count": 136.0, "vllm:inter_token_latency_seconds_sum": 5.7214583090972155, "vllm:request_time_per_output_token_seconds_count": 1.0, "vllm:request_time_per_output_token_seconds_sum": 0.005727185494591808, "vllm:e2e_request_latency_seconds_count": 1.0, "vllm:e2e_request_latency_seconds_sum": 13.960050821304321, "vllm:request_queue_time_seconds_count": 1.0, "vllm:request_queue_time_seconds_sum": 1.6840174794197083e-05, "vllm:request_inference_time_seconds_count": 1.0, "vllm:request_inference_time_seconds_sum": 13.953559221932665, "vllm:request_prefill_time_seconds_count": 1.0, "vllm:request_prefill_time_seconds_sum": 8.232100912835449, "vllm:request_decode_time_seconds_count": 1.0, "vllm:request_decode_time_seconds_sum": 5.7214583090972155, "vllm:request_prefill_kv_computed_tokens_count": 1.0, "vllm:request_prefill_kv_computed_tokens_sum": 17500.0, "vllm:tool_call_parser_invocations_total": 0.0 }, "repetition": 1 }, { "case": "pp512", "kind": "pp0", "target_tokens": 512, "prompt_sha256": "00687b0956e2b6962bfb5759e88b406712148dd82281feffa52d7da81ac07992", "input_ids_sha256": "4a7c2a4ee25dbaec6b96571f42a173757ab2d90e3a9d05d47a1832357c2a9db9", "prompt_tokens": 512, "output_tokens": 64, "expected_llama_prompt_tokens": null, "start_monotonic_s": 1196130.155491043, "end_monotonic_s": 1196130.649135922, "api_ttft_s": 0.17570216697640717, "api_e2e_s": 0.4936448789667338, "api_tail_s": 0.3178433298598975, "first_event_tokens": 1, "api_tail_tok_s": 198.21086076517585, "finish_reason": "length", "fresh": true, "usage": { "prompt_tokens": 512, "total_tokens": 576, "completion_tokens": 64 }, "engine_metric_isolated": true, "engine_prefill_s": 0.17243467480875552, "engine_decode_s": 0.31767305615358055, "engine_computed_prompt_tokens": 512.0, "engine_prefill_tok_s": 2969.2403837444576, "engine_decode_tok_s": 198.31710237818328, "engine_timing_note": "prefill=first scheduled to first new token; decode=first to last new token", "metrics_delta": { "vllm:spec_decode_num_drafts_total": 9.0, "vllm:spec_decode_num_draft_tokens_total": 63.0, "vllm:spec_decode_num_accepted_tokens_total": 59.0, "vllm:spec_decode_num_accepted_tokens_per_pos_total": 59.0, "vllm:estimated_flops_per_gpu_total": 0.0, "vllm:estimated_read_bytes_per_gpu_total": 0.0, "vllm:estimated_write_bytes_per_gpu_total": 0.0, "vllm:prefix_cache_queries_total": 512.0, "vllm:prefix_cache_hits_total": 0.0, "vllm:external_prefix_cache_queries_total": 0.0, "vllm:external_prefix_cache_hits_total": 0.0, "vllm:mm_cache_queries_total": 0.0, "vllm:mm_cache_hits_total": 0.0, "vllm:num_preemptions_total": 0.0, "vllm:prompt_tokens_total": 512.0, "vllm:prompt_tokens_by_source_total": 512.0, "vllm:prompt_tokens_cached_total": 0.0, "vllm:generation_tokens_total": 64.0, "vllm:request_success_total": 1.0, "vllm:request_prompt_tokens_count": 1.0, "vllm:request_prompt_tokens_sum": 512.0, "vllm:request_generation_tokens_count": 1.0, "vllm:request_generation_tokens_sum": 64.0, "vllm:iteration_tokens_total_count": 10.0, "vllm:iteration_tokens_total_sum": 576.0, "vllm:request_max_num_generation_tokens_count": 1.0, "vllm:request_max_num_generation_tokens_sum": 64.0, "vllm:request_params_n_count": 1.0, "vllm:request_params_n_sum": 1.0, "vllm:request_params_max_tokens_count": 1.0, "vllm:request_params_max_tokens_sum": 64.0, "vllm:time_to_first_token_seconds_count": 1.0, "vllm:time_to_first_token_seconds_sum": 0.17433834075927734, "vllm:inter_token_latency_seconds_count": 9.0, "vllm:inter_token_latency_seconds_sum": 0.31767305615358055, "vllm:request_time_per_output_token_seconds_count": 1.0, "vllm:request_time_per_output_token_seconds_sum": 0.005042429462755245, "vllm:e2e_request_latency_seconds_count": 1.0, "vllm:e2e_request_latency_seconds_sum": 0.492290735244751, "vllm:request_queue_time_seconds_count": 1.0, "vllm:request_queue_time_seconds_sum": 1.740013249218464e-05, "vllm:request_inference_time_seconds_count": 1.0, "vllm:request_inference_time_seconds_sum": 0.49010773096233606, "vllm:request_prefill_time_seconds_count": 1.0, "vllm:request_prefill_time_seconds_sum": 0.17243467480875552, "vllm:request_decode_time_seconds_count": 1.0, "vllm:request_decode_time_seconds_sum": 0.31767305615358055, "vllm:request_prefill_kv_computed_tokens_count": 1.0, "vllm:request_prefill_kv_computed_tokens_sum": 512.0, "vllm:tool_call_parser_invocations_total": 0.0 }, "repetition": 1 }, { "case": "pp2048", "kind": "pp0", "target_tokens": 2048, "prompt_sha256": "c21aa67b768e16f833142c7aad2014477820ee970ac9e41290a35eaa3f105b72", "input_ids_sha256": "c0fd1fc7b2d01c93a8897da47cb5aca81bb77a25869d459434bccef5236dc0c0", "prompt_tokens": 1978, "output_tokens": 64, "expected_llama_prompt_tokens": null, "start_monotonic_s": 1196130.670533777, "end_monotonic_s": 1196131.635677203, "api_ttft_s": 0.6374484680127352, "api_e2e_s": 0.9651434260886163, "api_tail_s": 0.32760115596465766, "first_event_tokens": 1, "api_tail_tok_s": 192.307013735924, "finish_reason": "length", "fresh": true, "usage": { "prompt_tokens": 1978, "total_tokens": 2042, "completion_tokens": 64 }, "engine_metric_isolated": true, "engine_prefill_s": 0.6326777129434049, "engine_decode_s": 0.3290959990117699, "engine_computed_prompt_tokens": 1978.0, "engine_prefill_tok_s": 3126.394307139026, "engine_decode_tok_s": 191.43350326099483, "engine_timing_note": "prefill=first scheduled to first new token; decode=first to last new token", "metrics_delta": { "vllm:spec_decode_num_drafts_total": 9.0, "vllm:spec_decode_num_draft_tokens_total": 63.0, "vllm:spec_decode_num_accepted_tokens_total": 59.0, "vllm:spec_decode_num_accepted_tokens_per_pos_total": 59.0, "vllm:estimated_flops_per_gpu_total": 0.0, "vllm:estimated_read_bytes_per_gpu_total": 0.0, "vllm:estimated_write_bytes_per_gpu_total": 0.0, "vllm:prefix_cache_queries_total": 1978.0, "vllm:prefix_cache_hits_total": 0.0, "vllm:external_prefix_cache_queries_total": 0.0, "vllm:external_prefix_cache_hits_total": 0.0, "vllm:mm_cache_queries_total": 0.0, "vllm:mm_cache_hits_total": 0.0, "vllm:num_preemptions_total": 0.0, "vllm:prompt_tokens_total": 1978.0, "vllm:prompt_tokens_by_source_total": 1978.0, "vllm:prompt_tokens_cached_total": 0.0, "vllm:generation_tokens_total": 64.0, "vllm:request_success_total": 1.0, "vllm:request_prompt_tokens_count": 1.0, "vllm:request_prompt_tokens_sum": 1978.0, "vllm:request_generation_tokens_count": 1.0, "vllm:request_generation_tokens_sum": 64.0, "vllm:iteration_tokens_total_count": 10.0, "vllm:iteration_tokens_total_sum": 2042.0, "vllm:request_max_num_generation_tokens_count": 1.0, "vllm:request_max_num_generation_tokens_sum": 64.0, "vllm:request_params_n_count": 1.0, "vllm:request_params_n_sum": 1.0, "vllm:request_params_max_tokens_count": 1.0, "vllm:request_params_max_tokens_sum": 64.0, "vllm:time_to_first_token_seconds_count": 1.0, "vllm:time_to_first_token_seconds_sum": 0.6350290775299072, "vllm:inter_token_latency_seconds_count": 9.0, "vllm:inter_token_latency_seconds_sum": 0.3290959990117699, "vllm:request_time_per_output_token_seconds_count": 1.0, "vllm:request_time_per_output_token_seconds_sum": 0.005223746016059842, "vllm:e2e_request_latency_seconds_count": 1.0, "vllm:e2e_request_latency_seconds_sum": 0.9635326862335205, "vllm:request_queue_time_seconds_count": 1.0, "vllm:request_queue_time_seconds_sum": 2.1660001948475838e-05, "vllm:request_inference_time_seconds_count": 1.0, "vllm:request_inference_time_seconds_sum": 0.9617737119551748, "vllm:request_prefill_time_seconds_count": 1.0, "vllm:request_prefill_time_seconds_sum": 0.6326777129434049, "vllm:request_decode_time_seconds_count": 1.0, "vllm:request_decode_time_seconds_sum": 0.3290959990117699, "vllm:request_prefill_kv_computed_tokens_count": 1.0, "vllm:request_prefill_kv_computed_tokens_sum": 1978.0, "vllm:tool_call_parser_invocations_total": 0.0 }, "repetition": 1 }, { "case": "pp8192", "kind": "pp0", "target_tokens": 8192, "prompt_sha256": "a29d179294fb683902960258e22071ae39bfbf7ebc69c21818c88bda08aa04bc", "input_ids_sha256": "737621140e7867e679c4d71011d3dc9f26ccc84238c657bf1348a8b9d0ebaa2e", "prompt_tokens": 8077, "output_tokens": 64, "expected_llama_prompt_tokens": null, "start_monotonic_s": 1196131.66350291, "end_monotonic_s": 1196134.589653312, "api_ttft_s": 2.4136410870123655, "api_e2e_s": 2.9261504020541906, "api_tail_s": 0.5124234440736473, "first_event_tokens": 1, "api_tail_tok_s": 122.94519450391388, "finish_reason": "length", "fresh": true, "usage": { "prompt_tokens": 8077, "total_tokens": 8141, "completion_tokens": 64 }, "engine_metric_isolated": true, "engine_prefill_s": 2.405595608986914, "engine_decode_s": 0.5165442740544677, "engine_computed_prompt_tokens": 8077.0, "engine_prefill_tok_s": 3357.5884366539585, "engine_decode_tok_s": 121.96437588108253, "engine_timing_note": "prefill=first scheduled to first new token; decode=first to last new token", "metrics_delta": { "vllm:spec_decode_num_drafts_total": 14.0, "vllm:spec_decode_num_draft_tokens_total": 98.0, "vllm:spec_decode_num_accepted_tokens_total": 49.0, "vllm:spec_decode_num_accepted_tokens_per_pos_total": 49.0, "vllm:estimated_flops_per_gpu_total": 0.0, "vllm:estimated_read_bytes_per_gpu_total": 0.0, "vllm:estimated_write_bytes_per_gpu_total": 0.0, "vllm:prefix_cache_queries_total": 8077.0, "vllm:prefix_cache_hits_total": 0.0, "vllm:external_prefix_cache_queries_total": 0.0, "vllm:external_prefix_cache_hits_total": 0.0, "vllm:mm_cache_queries_total": 0.0, "vllm:mm_cache_hits_total": 0.0, "vllm:num_preemptions_total": 0.0, "vllm:prompt_tokens_total": 8077.0, "vllm:prompt_tokens_by_source_total": 8077.0, "vllm:prompt_tokens_cached_total": 0.0, "vllm:generation_tokens_total": 64.0, "vllm:request_success_total": 1.0, "vllm:request_prompt_tokens_count": 1.0, "vllm:request_prompt_tokens_sum": 8077.0, "vllm:request_generation_tokens_count": 1.0, "vllm:request_generation_tokens_sum": 64.0, "vllm:iteration_tokens_total_count": 15.0, "vllm:iteration_tokens_total_sum": 8141.0, "vllm:request_max_num_generation_tokens_count": 1.0, "vllm:request_max_num_generation_tokens_sum": 64.0, "vllm:request_params_n_count": 1.0, "vllm:request_params_n_sum": 1.0, "vllm:request_params_max_tokens_count": 1.0, "vllm:request_params_max_tokens_sum": 64.0, "vllm:time_to_first_token_seconds_count": 1.0, "vllm:time_to_first_token_seconds_sum": 2.4078023433685303, "vllm:inter_token_latency_seconds_count": 14.0, "vllm:inter_token_latency_seconds_sum": 0.5165442740544677, "vllm:request_time_per_output_token_seconds_count": 1.0, "vllm:request_time_per_output_token_seconds_sum": 0.00819911546118203, "vllm:e2e_request_latency_seconds_count": 1.0, "vllm:e2e_request_latency_seconds_sum": 2.9241394996643066, "vllm:request_queue_time_seconds_count": 1.0, "vllm:request_queue_time_seconds_sum": 1.71000137925148e-05, "vllm:request_inference_time_seconds_count": 1.0, "vllm:request_inference_time_seconds_sum": 2.922139883041382, "vllm:request_prefill_time_seconds_count": 1.0, "vllm:request_prefill_time_seconds_sum": 2.405595608986914, "vllm:request_decode_time_seconds_count": 1.0, "vllm:request_decode_time_seconds_sum": 0.5165442740544677, "vllm:request_prefill_kv_computed_tokens_count": 1.0, "vllm:request_prefill_kv_computed_tokens_sum": 8077.0, "vllm:tool_call_parser_invocations_total": 0.0 }, "repetition": 1 }, { "case": "real-design", "kind": "real", "target_tokens": null, "prompt_sha256": "e6c65ab2c1586c13c0e3e223551dbd804d5f0ae382f7cf3c665c297b3d18b291", "input_ids_sha256": "6af826ccc823c0456c53eddecd1e695f6dcfe2337aa5d71a24a30aa7f2413d1a", "prompt_tokens": 73, "output_tokens": 1200, "expected_llama_prompt_tokens": null, "start_monotonic_s": 1196134.605079844, "end_monotonic_s": 1196145.728212626, "api_ttft_s": 0.5218966230750084, "api_e2e_s": 11.123132782056928, "api_tail_s": 10.601160467136651, "first_event_tokens": 1, "api_tail_tok_s": 113.10082549140463, "finish_reason": "length", "fresh": true, "usage": { "prompt_tokens": 73, "total_tokens": 1273, "completion_tokens": 1200 }, "engine_metric_isolated": true, "engine_prefill_s": 0.5191686828620732, "engine_decode_s": 10.601501125143841, "engine_computed_prompt_tokens": 73.0, "engine_prefill_tok_s": 140.60940578612252, "engine_decode_tok_s": 113.09719122288278, "engine_timing_note": "prefill=first scheduled to first new token; decode=first to last new token", "metrics_delta": { "vllm:spec_decode_num_drafts_total": 299.0, "vllm:spec_decode_num_draft_tokens_total": 2093.0, "vllm:spec_decode_num_accepted_tokens_total": 901.0, "vllm:spec_decode_num_accepted_tokens_per_pos_total": 901.0, "vllm:estimated_flops_per_gpu_total": 0.0, "vllm:estimated_read_bytes_per_gpu_total": 0.0, "vllm:estimated_write_bytes_per_gpu_total": 0.0, "vllm:prefix_cache_queries_total": 73.0, "vllm:prefix_cache_hits_total": 0.0, "vllm:external_prefix_cache_queries_total": 0.0, "vllm:external_prefix_cache_hits_total": 0.0, "vllm:mm_cache_queries_total": 0.0, "vllm:mm_cache_hits_total": 0.0, "vllm:num_preemptions_total": 0.0, "vllm:prompt_tokens_total": 73.0, "vllm:prompt_tokens_by_source_total": 73.0, "vllm:prompt_tokens_cached_total": 0.0, "vllm:generation_tokens_total": 1200.0, "vllm:request_success_total": 1.0, "vllm:request_prompt_tokens_count": 1.0, "vllm:request_prompt_tokens_sum": 73.0, "vllm:request_generation_tokens_count": 1.0, "vllm:request_generation_tokens_sum": 1200.0, "vllm:iteration_tokens_total_count": 300.0, "vllm:iteration_tokens_total_sum": 1273.0, "vllm:request_max_num_generation_tokens_count": 1.0, "vllm:request_max_num_generation_tokens_sum": 1200.0, "vllm:request_params_n_count": 1.0, "vllm:request_params_n_sum": 1.0, "vllm:request_params_max_tokens_count": 1.0, "vllm:request_params_max_tokens_sum": 1200.0, "vllm:time_to_first_token_seconds_count": 1.0, "vllm:time_to_first_token_seconds_sum": 0.5208108425140381, "vllm:inter_token_latency_seconds_count": 299.0, "vllm:inter_token_latency_seconds_sum": 10.601501125143841, "vllm:request_time_per_output_token_seconds_count": 1.0, "vllm:request_time_per_output_token_seconds_sum": 0.008841952564757172, "vllm:e2e_request_latency_seconds_count": 1.0, "vllm:e2e_request_latency_seconds_sum": 11.121902227401733, "vllm:request_queue_time_seconds_count": 1.0, "vllm:request_queue_time_seconds_sum": 1.4930032193660736e-05, "vllm:request_inference_time_seconds_count": 1.0, "vllm:request_inference_time_seconds_sum": 11.120669808005914, "vllm:request_prefill_time_seconds_count": 1.0, "vllm:request_prefill_time_seconds_sum": 0.5191686828620732, "vllm:request_decode_time_seconds_count": 1.0, "vllm:request_decode_time_seconds_sum": 10.601501125143841, "vllm:request_prefill_kv_computed_tokens_count": 1.0, "vllm:request_prefill_kv_computed_tokens_sum": 73.0, "vllm:tool_call_parser_invocations_total": 0.0 }, "repetition": 1 }, { "case": "real-review", "kind": "real", "target_tokens": null, "prompt_sha256": "c0bb3ba0a22668094376e5fb184030cde49af12100e7a7bfb5b3a23a98bc2928", "input_ids_sha256": "a8cfd61ed5bd0fdfc172e8b8ee552b21dce5eb3e89a6f7348a0cb7eb91e3b2d6", "prompt_tokens": 58, "output_tokens": 1200, "expected_llama_prompt_tokens": null, "start_monotonic_s": 1196145.74326838, "end_monotonic_s": 1196164.264415723, "api_ttft_s": 0.36731018009595573, "api_e2e_s": 18.521147343097255, "api_tail_s": 18.153749441029504, "first_event_tokens": 1, "api_tail_tok_s": 66.04696202813761, "finish_reason": "length", "fresh": true, "usage": { "prompt_tokens": 58, "total_tokens": 1258, "completion_tokens": 1200 }, "engine_metric_isolated": true, "engine_prefill_s": 0.3645533788949251, "engine_decode_s": 18.153787312097847, "engine_computed_prompt_tokens": 58.0, "engine_prefill_tok_s": 159.09878596055282, "engine_decode_tok_s": 66.04682424592336, "engine_timing_note": "prefill=first scheduled to first new token; decode=first to last new token", "metrics_delta": { "vllm:spec_decode_num_drafts_total": 511.0, "vllm:spec_decode_num_draft_tokens_total": 3577.0, "vllm:spec_decode_num_accepted_tokens_total": 690.0, "vllm:spec_decode_num_accepted_tokens_per_pos_total": 690.0, "vllm:estimated_flops_per_gpu_total": 0.0, "vllm:estimated_read_bytes_per_gpu_total": 0.0, "vllm:estimated_write_bytes_per_gpu_total": 0.0, "vllm:prefix_cache_queries_total": 58.0, "vllm:prefix_cache_hits_total": 0.0, "vllm:external_prefix_cache_queries_total": 0.0, "vllm:external_prefix_cache_hits_total": 0.0, "vllm:mm_cache_queries_total": 0.0, "vllm:mm_cache_hits_total": 0.0, "vllm:num_preemptions_total": 0.0, "vllm:prompt_tokens_total": 58.0, "vllm:prompt_tokens_by_source_total": 58.0, "vllm:prompt_tokens_cached_total": 0.0, "vllm:generation_tokens_total": 1200.0, "vllm:request_success_total": 1.0, "vllm:request_prompt_tokens_count": 1.0, "vllm:request_prompt_tokens_sum": 58.0, "vllm:request_generation_tokens_count": 1.0, "vllm:request_generation_tokens_sum": 1200.0, "vllm:iteration_tokens_total_count": 512.0, "vllm:iteration_tokens_total_sum": 1258.0, "vllm:request_max_num_generation_tokens_count": 1.0, "vllm:request_max_num_generation_tokens_sum": 1200.0, "vllm:request_params_n_count": 1.0, "vllm:request_params_n_sum": 1.0, "vllm:request_params_max_tokens_count": 1.0, "vllm:request_params_max_tokens_sum": 1200.0, "vllm:time_to_first_token_seconds_count": 1.0, "vllm:time_to_first_token_seconds_sum": 0.3661201000213623, "vllm:inter_token_latency_seconds_count": 511.0, "vllm:inter_token_latency_seconds_sum": 18.153787312097847, "vllm:request_time_per_output_token_seconds_count": 1.0, "vllm:request_time_per_output_token_seconds_sum": 0.015140773404585361, "vllm:e2e_request_latency_seconds_count": 1.0, "vllm:e2e_request_latency_seconds_sum": 18.51960515975952, "vllm:request_queue_time_seconds_count": 1.0, "vllm:request_queue_time_seconds_sum": 1.5770085155963898e-05, "vllm:request_inference_time_seconds_count": 1.0, "vllm:request_inference_time_seconds_sum": 18.518340690992773, "vllm:request_prefill_time_seconds_count": 1.0, "vllm:request_prefill_time_seconds_sum": 0.3645533788949251, "vllm:request_decode_time_seconds_count": 1.0, "vllm:request_decode_time_seconds_sum": 18.153787312097847, "vllm:request_prefill_kv_computed_tokens_count": 1.0, "vllm:request_prefill_kv_computed_tokens_sum": 58.0, "vllm:tool_call_parser_invocations_total": 0.0 }, "repetition": 1 }, { "case": "real-qa", "kind": "real", "target_tokens": null, "prompt_sha256": "d94fa364eca08c2c5166c9545153dee2d53b4ab1ce5ff5edcb6e242d2853683b", "input_ids_sha256": "8be6c502b1c9a99e43226c694ac8e06b80662a5321dbbdd92a17aa8ab1a680e1", "prompt_tokens": 61, "output_tokens": 1200, "expected_llama_prompt_tokens": null, "start_monotonic_s": 1196164.281611193, "end_monotonic_s": 1196177.053726193, "api_ttft_s": 0.058026405051350594, "api_e2e_s": 12.772115000057966, "api_tail_s": 12.713989912997931, "first_event_tokens": 1, "api_tail_tok_s": 94.3055648309287, "finish_reason": "length", "fresh": true, "usage": { "prompt_tokens": 61, "total_tokens": 1261, "completion_tokens": 1200 }, "engine_metric_isolated": true, "engine_prefill_s": 0.05516102095134556, "engine_decode_s": 12.714295499958098, "engine_computed_prompt_tokens": 61.0, "engine_prefill_tok_s": 1105.8533534722767, "engine_decode_tok_s": 94.30329820507566, "engine_timing_note": "prefill=first scheduled to first new token; decode=first to last new token", "metrics_delta": { "vllm:spec_decode_num_drafts_total": 358.0, "vllm:spec_decode_num_draft_tokens_total": 2506.0, "vllm:spec_decode_num_accepted_tokens_total": 846.0, "vllm:spec_decode_num_accepted_tokens_per_pos_total": 846.0, "vllm:estimated_flops_per_gpu_total": 0.0, "vllm:estimated_read_bytes_per_gpu_total": 0.0, "vllm:estimated_write_bytes_per_gpu_total": 0.0, "vllm:prefix_cache_queries_total": 61.0, "vllm:prefix_cache_hits_total": 0.0, "vllm:external_prefix_cache_queries_total": 0.0, "vllm:external_prefix_cache_hits_total": 0.0, "vllm:mm_cache_queries_total": 0.0, "vllm:mm_cache_hits_total": 0.0, "vllm:num_preemptions_total": 0.0, "vllm:prompt_tokens_total": 61.0, "vllm:prompt_tokens_by_source_total": 61.0, "vllm:prompt_tokens_cached_total": 0.0, "vllm:generation_tokens_total": 1200.0, "vllm:request_success_total": 1.0, "vllm:request_prompt_tokens_count": 1.0, "vllm:request_prompt_tokens_sum": 61.0, "vllm:request_generation_tokens_count": 1.0, "vllm:request_generation_tokens_sum": 1200.0, "vllm:iteration_tokens_total_count": 359.0, "vllm:iteration_tokens_total_sum": 1261.0, "vllm:request_max_num_generation_tokens_count": 1.0, "vllm:request_max_num_generation_tokens_sum": 1200.0, "vllm:request_params_n_count": 1.0, "vllm:request_params_n_sum": 1.0, "vllm:request_params_max_tokens_count": 1.0, "vllm:request_params_max_tokens_sum": 1200.0, "vllm:time_to_first_token_seconds_count": 1.0, "vllm:time_to_first_token_seconds_sum": 0.056700706481933594, "vllm:inter_token_latency_seconds_count": 358.0, "vllm:inter_token_latency_seconds_sum": 12.714295499958098, "vllm:request_time_per_output_token_seconds_count": 1.0, "vllm:request_time_per_output_token_seconds_sum": 0.010604082985786578, "vllm:e2e_request_latency_seconds_count": 1.0, "vllm:e2e_request_latency_seconds_sum": 12.770626544952393, "vllm:request_queue_time_seconds_count": 1.0, "vllm:request_queue_time_seconds_sum": 1.638103276491165e-05, "vllm:request_inference_time_seconds_count": 1.0, "vllm:request_inference_time_seconds_sum": 12.769456520909443, "vllm:request_prefill_time_seconds_count": 1.0, "vllm:request_prefill_time_seconds_sum": 0.05516102095134556, "vllm:request_decode_time_seconds_count": 1.0, "vllm:request_decode_time_seconds_sum": 12.714295499958098, "vllm:request_prefill_kv_computed_tokens_count": 1.0, "vllm:request_prefill_kv_computed_tokens_sum": 61.0, "vllm:tool_call_parser_invocations_total": 0.0 }, "repetition": 1 } ]