[ { "case": "ladder-0", "kind": "ladder", "target_tokens": 0, "prompt_sha256": "3965f79a076c8829b8f9083b6f023262873c2a459645e1f0f979e0f3ddb28460", "input_ids_sha256": "1f563f25a0c1605cb77f6cd9e6f55a0a40c46f09abd4722cf438b1b86a5c1d52", "prompt_tokens": 11, "output_tokens": 1000, "expected_llama_prompt_tokens": 11, "start_monotonic_s": 1196277.011326534, "end_monotonic_s": 1196297.944572812, "api_ttft_s": 0.043322253972291946, "api_e2e_s": 20.933246277971193, "api_tail_s": 20.889855550136417, "first_event_tokens": 1, "api_tail_tok_s": 47.82225504634838, "finish_reason": "length", "fresh": false, "usage": { "prompt_tokens": 11, "total_tokens": 1011, "completion_tokens": 1000 }, "engine_metric_isolated": true, "engine_prefill_s": 0.04069624585099518, "engine_decode_s": 20.8900685030967, "engine_computed_prompt_tokens": 11.0, "engine_prefill_tok_s": 270.2952021735687, "engine_decode_tok_s": 47.82176754719164, "engine_timing_note": "prefill=first scheduled to first new token; decode=first to last new token", "metrics_delta": { "vllm:spec_decode_num_drafts_total": 595.0, "vllm:spec_decode_num_draft_tokens_total": 4165.0, "vllm:spec_decode_num_accepted_tokens_total": 405.0, "vllm:spec_decode_num_accepted_tokens_per_pos_total": 405.0, "vllm:estimated_flops_per_gpu_total": 0.0, "vllm:estimated_read_bytes_per_gpu_total": 0.0, "vllm:estimated_write_bytes_per_gpu_total": 0.0, "vllm:prefix_cache_queries_total": 11.0, "vllm:prefix_cache_hits_total": 0.0, "vllm:external_prefix_cache_queries_total": 0.0, "vllm:external_prefix_cache_hits_total": 0.0, "vllm:mm_cache_queries_total": 0.0, "vllm:mm_cache_hits_total": 0.0, "vllm:num_preemptions_total": 0.0, "vllm:prompt_tokens_total": 11.0, "vllm:prompt_tokens_by_source_total": 11.0, "vllm:prompt_tokens_cached_total": 0.0, "vllm:generation_tokens_total": 1000.0, "vllm:request_success_total": 1.0, "vllm:request_prompt_tokens_count": 1.0, "vllm:request_prompt_tokens_sum": 11.0, "vllm:request_generation_tokens_count": 1.0, "vllm:request_generation_tokens_sum": 1000.0, "vllm:iteration_tokens_total_count": 596.0, "vllm:iteration_tokens_total_sum": 1011.0, "vllm:request_max_num_generation_tokens_count": 1.0, "vllm:request_max_num_generation_tokens_sum": 1000.0, "vllm:request_params_n_count": 1.0, "vllm:request_params_n_sum": 1.0, "vllm:request_params_max_tokens_count": 1.0, "vllm:request_params_max_tokens_sum": 1000.0, "vllm:time_to_first_token_seconds_count": 1.0, "vllm:time_to_first_token_seconds_sum": 0.04229307174682617, "vllm:inter_token_latency_seconds_count": 595.0, "vllm:inter_token_latency_seconds_sum": 20.8900685030967, "vllm:request_time_per_output_token_seconds_count": 1.0, "vllm:request_time_per_output_token_seconds_sum": 0.02091097948257928, "vllm:e2e_request_latency_seconds_count": 1.0, "vllm:e2e_request_latency_seconds_sum": 20.93203353881836, "vllm:request_queue_time_seconds_count": 1.0, "vllm:request_queue_time_seconds_sum": 1.5910016372799873e-05, "vllm:request_inference_time_seconds_count": 1.0, "vllm:request_inference_time_seconds_sum": 20.930764748947695, "vllm:request_prefill_time_seconds_count": 1.0, "vllm:request_prefill_time_seconds_sum": 0.04069624585099518, "vllm:request_decode_time_seconds_count": 1.0, "vllm:request_decode_time_seconds_sum": 20.8900685030967, "vllm:request_prefill_kv_computed_tokens_count": 1.0, "vllm:request_prefill_kv_computed_tokens_sum": 11.0, "vllm:tool_call_parser_invocations_total": 0.0 }, "repetition": 3 }, { "case": "ladder-8000", "kind": "ladder", "target_tokens": 8000, "prompt_sha256": "ef9d16376787f5d95c40f8f6f0ee77d902e03a9173cf110c21007bcdec639e98", "input_ids_sha256": "8adc690f4ef99b0b9b6ff75ad2e742bcf2737662529af5706170a23202bc7753", "prompt_tokens": 7887, "output_tokens": 1000, "expected_llama_prompt_tokens": 7887, "start_monotonic_s": 1196297.973128301, "end_monotonic_s": 1196304.934075493, "api_ttft_s": 2.3533540298230946, "api_e2e_s": 6.960947191808373, "api_tail_s": 4.6074794821906835, "first_event_tokens": 1, "api_tail_tok_s": 216.82136705360065, "finish_reason": "length", "fresh": false, "usage": { "prompt_tokens": 7887, "total_tokens": 8887, "completion_tokens": 1000 }, "engine_metric_isolated": true, "engine_prefill_s": 2.345408257097006, "engine_decode_s": 4.611316894879565, "engine_computed_prompt_tokens": 7887.0, "engine_prefill_tok_s": 3362.740783458321, "engine_decode_tok_s": 216.64093420022724, "engine_timing_note": "prefill=first scheduled to first new token; decode=first to last new token", "metrics_delta": { "vllm:spec_decode_num_drafts_total": 126.0, "vllm:spec_decode_num_draft_tokens_total": 882.0, "vllm:spec_decode_num_accepted_tokens_total": 879.0, "vllm:spec_decode_num_accepted_tokens_per_pos_total": 879.0, "vllm:estimated_flops_per_gpu_total": 0.0, "vllm:estimated_read_bytes_per_gpu_total": 0.0, "vllm:estimated_write_bytes_per_gpu_total": 0.0, "vllm:prefix_cache_queries_total": 7887.0, "vllm:prefix_cache_hits_total": 0.0, "vllm:external_prefix_cache_queries_total": 0.0, "vllm:external_prefix_cache_hits_total": 0.0, "vllm:mm_cache_queries_total": 0.0, "vllm:mm_cache_hits_total": 0.0, "vllm:num_preemptions_total": 0.0, "vllm:prompt_tokens_total": 7887.0, "vllm:prompt_tokens_by_source_total": 7887.0, "vllm:prompt_tokens_cached_total": 0.0, "vllm:generation_tokens_total": 1000.0, "vllm:request_success_total": 1.0, "vllm:request_prompt_tokens_count": 1.0, "vllm:request_prompt_tokens_sum": 7887.0, "vllm:request_generation_tokens_count": 1.0, "vllm:request_generation_tokens_sum": 1000.0, "vllm:iteration_tokens_total_count": 127.0, "vllm:iteration_tokens_total_sum": 8887.0, "vllm:request_max_num_generation_tokens_count": 1.0, "vllm:request_max_num_generation_tokens_sum": 1000.0, "vllm:request_params_n_count": 1.0, "vllm:request_params_n_sum": 1.0, "vllm:request_params_max_tokens_count": 1.0, "vllm:request_params_max_tokens_sum": 1000.0, "vllm:time_to_first_token_seconds_count": 1.0, "vllm:time_to_first_token_seconds_sum": 2.3478291034698486, "vllm:inter_token_latency_seconds_count": 126.0, "vllm:inter_token_latency_seconds_sum": 4.611316894879565, "vllm:request_time_per_output_token_seconds_count": 1.0, "vllm:request_time_per_output_token_seconds_sum": 0.004615932827707281, "vllm:e2e_request_latency_seconds_count": 1.0, "vllm:e2e_request_latency_seconds_sum": 6.958890676498413, "vllm:request_queue_time_seconds_count": 1.0, "vllm:request_queue_time_seconds_sum": 1.628883183002472e-05, "vllm:request_inference_time_seconds_count": 1.0, "vllm:request_inference_time_seconds_sum": 6.9567251519765705, "vllm:request_prefill_time_seconds_count": 1.0, "vllm:request_prefill_time_seconds_sum": 2.345408257097006, "vllm:request_decode_time_seconds_count": 1.0, "vllm:request_decode_time_seconds_sum": 4.611316894879565, "vllm:request_prefill_kv_computed_tokens_count": 1.0, "vllm:request_prefill_kv_computed_tokens_sum": 7887.0, "vllm:tool_call_parser_invocations_total": 0.0 }, "repetition": 3 }, { "case": "ladder-16000", "kind": "ladder", "target_tokens": 16000, "prompt_sha256": "17610c10a63b400e07668fc3a875a67548796ea1dd18e67702fb066da804a2b8", "input_ids_sha256": "179aac711afb83a653889e3dbf76a4c9367280452995ef638bbb341c717662da", "prompt_tokens": 16050, "output_tokens": 1000, "expected_llama_prompt_tokens": 16050, "start_monotonic_s": 1196304.979702789, "end_monotonic_s": 1196312.780209883, "api_ttft_s": 3.138414175948128, "api_e2e_s": 7.800507094012573, "api_tail_s": 4.662015894893557, "first_event_tokens": 1, "api_tail_tok_s": 214.28498369004578, "finish_reason": "length", "fresh": false, "usage": { "prompt_tokens": 16050, "total_tokens": 17050, "completion_tokens": 1000 }, "engine_metric_isolated": true, "engine_prefill_s": 3.123006857931614, "engine_decode_s": 4.67241561296396, "engine_computed_prompt_tokens": 9890.0, "engine_prefill_tok_s": 3166.819814974792, "engine_decode_tok_s": 213.80803480499495, "engine_timing_note": "prefill=first scheduled to first new token; decode=first to last new token", "metrics_delta": { "vllm:spec_decode_num_drafts_total": 126.0, "vllm:spec_decode_num_draft_tokens_total": 882.0, "vllm:spec_decode_num_accepted_tokens_total": 879.0, "vllm:spec_decode_num_accepted_tokens_per_pos_total": 879.0, "vllm:estimated_flops_per_gpu_total": 0.0, "vllm:estimated_read_bytes_per_gpu_total": 0.0, "vllm:estimated_write_bytes_per_gpu_total": 0.0, "vllm:prefix_cache_queries_total": 16050.0, "vllm:prefix_cache_hits_total": 6160.0, "vllm:external_prefix_cache_queries_total": 0.0, "vllm:external_prefix_cache_hits_total": 0.0, "vllm:mm_cache_queries_total": 0.0, "vllm:mm_cache_hits_total": 0.0, "vllm:num_preemptions_total": 0.0, "vllm:prompt_tokens_total": 16050.0, "vllm:prompt_tokens_by_source_total": 16050.0, "vllm:prompt_tokens_cached_total": 6160.0, "vllm:generation_tokens_total": 1000.0, "vllm:request_success_total": 1.0, "vllm:request_prompt_tokens_count": 1.0, "vllm:request_prompt_tokens_sum": 16050.0, "vllm:request_generation_tokens_count": 1.0, "vllm:request_generation_tokens_sum": 1000.0, "vllm:iteration_tokens_total_count": 127.0, "vllm:iteration_tokens_total_sum": 10890.0, "vllm:request_max_num_generation_tokens_count": 1.0, "vllm:request_max_num_generation_tokens_sum": 1000.0, "vllm:request_params_n_count": 1.0, "vllm:request_params_n_sum": 1.0, "vllm:request_params_max_tokens_count": 1.0, "vllm:request_params_max_tokens_sum": 1000.0, "vllm:time_to_first_token_seconds_count": 1.0, "vllm:time_to_first_token_seconds_sum": 3.125706434249878, "vllm:inter_token_latency_seconds_count": 126.0, "vllm:inter_token_latency_seconds_sum": 4.67241561296396, "vllm:request_time_per_output_token_seconds_count": 1.0, "vllm:request_time_per_output_token_seconds_sum": 0.004677092705669655, "vllm:e2e_request_latency_seconds_count": 1.0, "vllm:e2e_request_latency_seconds_sum": 7.797708034515381, "vllm:request_queue_time_seconds_count": 1.0, "vllm:request_queue_time_seconds_sum": 1.8798978999257088e-05, "vllm:request_inference_time_seconds_count": 1.0, "vllm:request_inference_time_seconds_sum": 7.7954224708955735, "vllm:request_prefill_time_seconds_count": 1.0, "vllm:request_prefill_time_seconds_sum": 3.123006857931614, "vllm:request_decode_time_seconds_count": 1.0, "vllm:request_decode_time_seconds_sum": 4.67241561296396, "vllm:request_prefill_kv_computed_tokens_count": 1.0, "vllm:request_prefill_kv_computed_tokens_sum": 9890.0, "vllm:tool_call_parser_invocations_total": 0.0 }, "repetition": 3 }, { "case": "ladder-32000", "kind": "ladder", "target_tokens": 32000, "prompt_sha256": "d2b2c90bc484f3c1ca1711f0a2bac03a3ea0f5c9234d48231dc746d998114fb8", "input_ids_sha256": "f6305d07154b3068b74b6ca47f8d0755c1fa9943495ffb02c16fa4c36d0a5abf", "prompt_tokens": 32460, "output_tokens": 1000, "expected_llama_prompt_tokens": 32460, "start_monotonic_s": 1196312.851753163, "end_monotonic_s": 1196324.092953769, "api_ttft_s": 6.066811121068895, "api_e2e_s": 11.24120060610585, "api_tail_s": 5.1743061719462276, "first_event_tokens": 1, "api_tail_tok_s": 193.06936366006403, "finish_reason": "length", "fresh": false, "usage": { "prompt_tokens": 32460, "total_tokens": 33460, "completion_tokens": 1000 }, "engine_metric_isolated": true, "engine_prefill_s": 6.04286873806268, "engine_decode_s": 5.190062701003626, "engine_computed_prompt_tokens": 17500.0, "engine_prefill_tok_s": 2895.9755305905305, "engine_decode_tok_s": 192.48322372036446, "engine_timing_note": "prefill=first scheduled to first new token; decode=first to last new token", "metrics_delta": { "vllm:spec_decode_num_drafts_total": 136.0, "vllm:spec_decode_num_draft_tokens_total": 952.0, "vllm:spec_decode_num_accepted_tokens_total": 865.0, "vllm:spec_decode_num_accepted_tokens_per_pos_total": 865.0, "vllm:estimated_flops_per_gpu_total": 0.0, "vllm:estimated_read_bytes_per_gpu_total": 0.0, "vllm:estimated_write_bytes_per_gpu_total": 0.0, "vllm:prefix_cache_queries_total": 32460.0, "vllm:prefix_cache_hits_total": 14960.0, "vllm:external_prefix_cache_queries_total": 0.0, "vllm:external_prefix_cache_hits_total": 0.0, "vllm:mm_cache_queries_total": 0.0, "vllm:mm_cache_hits_total": 0.0, "vllm:num_preemptions_total": 0.0, "vllm:prompt_tokens_total": 32460.0, "vllm:prompt_tokens_by_source_total": 32460.0, "vllm:prompt_tokens_cached_total": 14960.0, "vllm:generation_tokens_total": 1000.0, "vllm:request_success_total": 1.0, "vllm:request_prompt_tokens_count": 1.0, "vllm:request_prompt_tokens_sum": 32460.0, "vllm:request_generation_tokens_count": 1.0, "vllm:request_generation_tokens_sum": 1000.0, "vllm:iteration_tokens_total_count": 137.0, "vllm:iteration_tokens_total_sum": 18500.0, "vllm:request_max_num_generation_tokens_count": 1.0, "vllm:request_max_num_generation_tokens_sum": 1000.0, "vllm:request_params_n_count": 1.0, "vllm:request_params_n_sum": 1.0, "vllm:request_params_max_tokens_count": 1.0, "vllm:request_params_max_tokens_sum": 1000.0, "vllm:time_to_first_token_seconds_count": 1.0, "vllm:time_to_first_token_seconds_sum": 6.046747446060181, "vllm:inter_token_latency_seconds_count": 136.0, "vllm:inter_token_latency_seconds_sum": 5.190062701003626, "vllm:request_time_per_output_token_seconds_count": 1.0, "vllm:request_time_per_output_token_seconds_sum": 0.005195257958962585, "vllm:e2e_request_latency_seconds_count": 1.0, "vllm:e2e_request_latency_seconds_sum": 11.236663341522217, "vllm:request_queue_time_seconds_count": 1.0, "vllm:request_queue_time_seconds_sum": 1.491885632276535e-05, "vllm:request_inference_time_seconds_count": 1.0, "vllm:request_inference_time_seconds_sum": 11.232931439066306, "vllm:request_prefill_time_seconds_count": 1.0, "vllm:request_prefill_time_seconds_sum": 6.04286873806268, "vllm:request_decode_time_seconds_count": 1.0, "vllm:request_decode_time_seconds_sum": 5.190062701003626, "vllm:request_prefill_kv_computed_tokens_count": 1.0, "vllm:request_prefill_kv_computed_tokens_sum": 17500.0, "vllm:tool_call_parser_invocations_total": 0.0 }, "repetition": 3 }, { "case": "pp512", "kind": "pp0", "target_tokens": 512, "prompt_sha256": "00687b0956e2b6962bfb5759e88b406712148dd82281feffa52d7da81ac07992", "input_ids_sha256": "4a7c2a4ee25dbaec6b96571f42a173757ab2d90e3a9d05d47a1832357c2a9db9", "prompt_tokens": 512, "output_tokens": 64, "expected_llama_prompt_tokens": null, "start_monotonic_s": 1196324.114862564, "end_monotonic_s": 1196324.609190815, "api_ttft_s": 0.17672426998615265, "api_e2e_s": 0.49432825087569654, "api_tail_s": 0.31751304888166487, "first_event_tokens": 1, "api_tail_tok_s": 198.41704214014746, "finish_reason": "length", "fresh": true, "usage": { "prompt_tokens": 512, "total_tokens": 576, "completion_tokens": 64 }, "engine_metric_isolated": true, "engine_prefill_s": 0.17339939903467894, "engine_decode_s": 0.3177713069599122, "engine_computed_prompt_tokens": 512.0, "engine_prefill_tok_s": 2952.7207294276886, "engine_decode_tok_s": 198.25578527751608, "engine_timing_note": "prefill=first scheduled to first new token; decode=first to last new token", "metrics_delta": { "vllm:spec_decode_num_drafts_total": 9.0, "vllm:spec_decode_num_draft_tokens_total": 63.0, "vllm:spec_decode_num_accepted_tokens_total": 59.0, "vllm:spec_decode_num_accepted_tokens_per_pos_total": 59.0, "vllm:estimated_flops_per_gpu_total": 0.0, "vllm:estimated_read_bytes_per_gpu_total": 0.0, "vllm:estimated_write_bytes_per_gpu_total": 0.0, "vllm:prefix_cache_queries_total": 512.0, "vllm:prefix_cache_hits_total": 0.0, "vllm:external_prefix_cache_queries_total": 0.0, "vllm:external_prefix_cache_hits_total": 0.0, "vllm:mm_cache_queries_total": 0.0, "vllm:mm_cache_hits_total": 0.0, "vllm:num_preemptions_total": 0.0, "vllm:prompt_tokens_total": 512.0, "vllm:prompt_tokens_by_source_total": 512.0, "vllm:prompt_tokens_cached_total": 0.0, "vllm:generation_tokens_total": 64.0, "vllm:request_success_total": 1.0, "vllm:request_prompt_tokens_count": 1.0, "vllm:request_prompt_tokens_sum": 512.0, "vllm:request_generation_tokens_count": 1.0, "vllm:request_generation_tokens_sum": 64.0, "vllm:iteration_tokens_total_count": 10.0, "vllm:iteration_tokens_total_sum": 576.0, "vllm:request_max_num_generation_tokens_count": 1.0, "vllm:request_max_num_generation_tokens_sum": 64.0, "vllm:request_params_n_count": 1.0, "vllm:request_params_n_sum": 1.0, "vllm:request_params_max_tokens_count": 1.0, "vllm:request_params_max_tokens_sum": 64.0, "vllm:time_to_first_token_seconds_count": 1.0, "vllm:time_to_first_token_seconds_sum": 0.17525625228881836, "vllm:inter_token_latency_seconds_count": 9.0, "vllm:inter_token_latency_seconds_sum": 0.3177713069599122, "vllm:request_time_per_output_token_seconds_count": 1.0, "vllm:request_time_per_output_token_seconds_sum": 0.005043988999363702, "vllm:e2e_request_latency_seconds_count": 1.0, "vllm:e2e_request_latency_seconds_sum": 0.49288225173950195, "vllm:request_queue_time_seconds_count": 1.0, "vllm:request_queue_time_seconds_sum": 1.6189180314540863e-05, "vllm:request_inference_time_seconds_count": 1.0, "vllm:request_inference_time_seconds_sum": 0.4911707059945911, "vllm:request_prefill_time_seconds_count": 1.0, "vllm:request_prefill_time_seconds_sum": 0.17339939903467894, "vllm:request_decode_time_seconds_count": 1.0, "vllm:request_decode_time_seconds_sum": 0.3177713069599122, "vllm:request_prefill_kv_computed_tokens_count": 1.0, "vllm:request_prefill_kv_computed_tokens_sum": 512.0, "vllm:tool_call_parser_invocations_total": 0.0 }, "repetition": 3 }, { "case": "pp2048", "kind": "pp0", "target_tokens": 2048, "prompt_sha256": "c21aa67b768e16f833142c7aad2014477820ee970ac9e41290a35eaa3f105b72", "input_ids_sha256": "c0fd1fc7b2d01c93a8897da47cb5aca81bb77a25869d459434bccef5236dc0c0", "prompt_tokens": 1978, "output_tokens": 64, "expected_llama_prompt_tokens": null, "start_monotonic_s": 1196324.626969188, "end_monotonic_s": 1196325.593572279, "api_ttft_s": 0.638121159048751, "api_e2e_s": 0.9666030909866095, "api_tail_s": 0.32839534897357225, "first_event_tokens": 1, "api_tail_tok_s": 191.8419374601738, "finish_reason": "length", "fresh": true, "usage": { "prompt_tokens": 1978, "total_tokens": 2042, "completion_tokens": 64 }, "engine_metric_isolated": true, "engine_prefill_s": 0.6337535521015525, "engine_decode_s": 0.3293782689142972, "engine_computed_prompt_tokens": 1978.0, "engine_prefill_tok_s": 3121.0870431272087, "engine_decode_tok_s": 191.26944897628422, "engine_timing_note": "prefill=first scheduled to first new token; decode=first to last new token", "metrics_delta": { "vllm:spec_decode_num_drafts_total": 9.0, "vllm:spec_decode_num_draft_tokens_total": 63.0, "vllm:spec_decode_num_accepted_tokens_total": 59.0, "vllm:spec_decode_num_accepted_tokens_per_pos_total": 59.0, "vllm:estimated_flops_per_gpu_total": 0.0, "vllm:estimated_read_bytes_per_gpu_total": 0.0, "vllm:estimated_write_bytes_per_gpu_total": 0.0, "vllm:prefix_cache_queries_total": 1978.0, "vllm:prefix_cache_hits_total": 0.0, "vllm:external_prefix_cache_queries_total": 0.0, "vllm:external_prefix_cache_hits_total": 0.0, "vllm:mm_cache_queries_total": 0.0, "vllm:mm_cache_hits_total": 0.0, "vllm:num_preemptions_total": 0.0, "vllm:prompt_tokens_total": 1978.0, "vllm:prompt_tokens_by_source_total": 1978.0, "vllm:prompt_tokens_cached_total": 0.0, "vllm:generation_tokens_total": 64.0, "vllm:request_success_total": 1.0, "vllm:request_prompt_tokens_count": 1.0, "vllm:request_prompt_tokens_sum": 1978.0, "vllm:request_generation_tokens_count": 1.0, "vllm:request_generation_tokens_sum": 64.0, "vllm:iteration_tokens_total_count": 10.0, "vllm:iteration_tokens_total_sum": 2042.0, "vllm:request_max_num_generation_tokens_count": 1.0, "vllm:request_max_num_generation_tokens_sum": 64.0, "vllm:request_params_n_count": 1.0, "vllm:request_params_n_sum": 1.0, "vllm:request_params_max_tokens_count": 1.0, "vllm:request_params_max_tokens_sum": 64.0, "vllm:time_to_first_token_seconds_count": 1.0, "vllm:time_to_first_token_seconds_sum": 0.6356761455535889, "vllm:inter_token_latency_seconds_count": 9.0, "vllm:inter_token_latency_seconds_sum": 0.3293782689142972, "vllm:request_time_per_output_token_seconds_count": 1.0, "vllm:request_time_per_output_token_seconds_sum": 0.005228226490703147, "vllm:e2e_request_latency_seconds_count": 1.0, "vllm:e2e_request_latency_seconds_sum": 0.9649860858917236, "vllm:request_queue_time_seconds_count": 1.0, "vllm:request_queue_time_seconds_sum": 1.4318851754069328e-05, "vllm:request_inference_time_seconds_count": 1.0, "vllm:request_inference_time_seconds_sum": 0.9631318210158497, "vllm:request_prefill_time_seconds_count": 1.0, "vllm:request_prefill_time_seconds_sum": 0.6337535521015525, "vllm:request_decode_time_seconds_count": 1.0, "vllm:request_decode_time_seconds_sum": 0.3293782689142972, "vllm:request_prefill_kv_computed_tokens_count": 1.0, "vllm:request_prefill_kv_computed_tokens_sum": 1978.0, "vllm:tool_call_parser_invocations_total": 0.0 }, "repetition": 3 }, { "case": "pp8192", "kind": "pp0", "target_tokens": 8192, "prompt_sha256": "a29d179294fb683902960258e22071ae39bfbf7ebc69c21818c88bda08aa04bc", "input_ids_sha256": "737621140e7867e679c4d71011d3dc9f26ccc84238c657bf1348a8b9d0ebaa2e", "prompt_tokens": 8077, "output_tokens": 64, "expected_llama_prompt_tokens": null, "start_monotonic_s": 1196325.621543482, "end_monotonic_s": 1196328.557168559, "api_ttft_s": 2.4233408181462437, "api_e2e_s": 2.9356250769924372, "api_tail_s": 0.5122065148316324, "first_event_tokens": 1, "api_tail_tok_s": 122.99726414200092, "finish_reason": "length", "fresh": true, "usage": { "prompt_tokens": 8077, "total_tokens": 8141, "completion_tokens": 64 }, "engine_metric_isolated": true, "engine_prefill_s": 2.4150411367882043, "engine_decode_s": 0.5166103800293058, "engine_computed_prompt_tokens": 8077.0, "engine_prefill_tok_s": 3344.456488530755, "engine_decode_tok_s": 121.94876919899711, "engine_timing_note": "prefill=first scheduled to first new token; decode=first to last new token", "metrics_delta": { "vllm:spec_decode_num_drafts_total": 14.0, "vllm:spec_decode_num_draft_tokens_total": 98.0, "vllm:spec_decode_num_accepted_tokens_total": 49.0, "vllm:spec_decode_num_accepted_tokens_per_pos_total": 49.0, "vllm:estimated_flops_per_gpu_total": 0.0, "vllm:estimated_read_bytes_per_gpu_total": 0.0, "vllm:estimated_write_bytes_per_gpu_total": 0.0, "vllm:prefix_cache_queries_total": 8077.0, "vllm:prefix_cache_hits_total": 0.0, "vllm:external_prefix_cache_queries_total": 0.0, "vllm:external_prefix_cache_hits_total": 0.0, "vllm:mm_cache_queries_total": 0.0, "vllm:mm_cache_hits_total": 0.0, "vllm:num_preemptions_total": 0.0, "vllm:prompt_tokens_total": 8077.0, "vllm:prompt_tokens_by_source_total": 8077.0, "vllm:prompt_tokens_cached_total": 0.0, "vllm:generation_tokens_total": 64.0, "vllm:request_success_total": 1.0, "vllm:request_prompt_tokens_count": 1.0, "vllm:request_prompt_tokens_sum": 8077.0, "vllm:request_generation_tokens_count": 1.0, "vllm:request_generation_tokens_sum": 64.0, "vllm:iteration_tokens_total_count": 15.0, "vllm:iteration_tokens_total_sum": 8141.0, "vllm:request_max_num_generation_tokens_count": 1.0, "vllm:request_max_num_generation_tokens_sum": 64.0, "vllm:request_params_n_count": 1.0, "vllm:request_params_n_sum": 1.0, "vllm:request_params_max_tokens_count": 1.0, "vllm:request_params_max_tokens_sum": 64.0, "vllm:time_to_first_token_seconds_count": 1.0, "vllm:time_to_first_token_seconds_sum": 2.417365789413452, "vllm:inter_token_latency_seconds_count": 14.0, "vllm:inter_token_latency_seconds_sum": 0.5166103800293058, "vllm:request_time_per_output_token_seconds_count": 1.0, "vllm:request_time_per_output_token_seconds_sum": 0.008200164762369955, "vllm:e2e_request_latency_seconds_count": 1.0, "vllm:e2e_request_latency_seconds_sum": 2.9336299896240234, "vllm:request_queue_time_seconds_count": 1.0, "vllm:request_queue_time_seconds_sum": 1.5348196029663086e-05, "vllm:request_inference_time_seconds_count": 1.0, "vllm:request_inference_time_seconds_sum": 2.93165151681751, "vllm:request_prefill_time_seconds_count": 1.0, "vllm:request_prefill_time_seconds_sum": 2.4150411367882043, "vllm:request_decode_time_seconds_count": 1.0, "vllm:request_decode_time_seconds_sum": 0.5166103800293058, "vllm:request_prefill_kv_computed_tokens_count": 1.0, "vllm:request_prefill_kv_computed_tokens_sum": 8077.0, "vllm:tool_call_parser_invocations_total": 0.0 }, "repetition": 3 }, { "case": "real-design", "kind": "real", "target_tokens": null, "prompt_sha256": "e6c65ab2c1586c13c0e3e223551dbd804d5f0ae382f7cf3c665c297b3d18b291", "input_ids_sha256": "6af826ccc823c0456c53eddecd1e695f6dcfe2337aa5d71a24a30aa7f2413d1a", "prompt_tokens": 73, "output_tokens": 1200, "expected_llama_prompt_tokens": null, "start_monotonic_s": 1196328.573048315, "end_monotonic_s": 1196339.046519336, "api_ttft_s": 0.08754590898752213, "api_e2e_s": 10.473471021046862, "api_tail_s": 10.38581522111781, "first_event_tokens": 1, "api_tail_tok_s": 115.44592065936577, "finish_reason": "length", "fresh": true, "usage": { "prompt_tokens": 73, "total_tokens": 1273, "completion_tokens": 1200 }, "engine_metric_isolated": true, "engine_prefill_s": 0.08493559807538986, "engine_decode_s": 10.385799739975482, "engine_computed_prompt_tokens": 73.0, "engine_prefill_tok_s": 859.4747273717237, "engine_decode_tok_s": 115.4460927438247, "engine_timing_note": "prefill=first scheduled to first new token; decode=first to last new token", "metrics_delta": { "vllm:spec_decode_num_drafts_total": 293.0, "vllm:spec_decode_num_draft_tokens_total": 2051.0, "vllm:spec_decode_num_accepted_tokens_total": 909.0, "vllm:spec_decode_num_accepted_tokens_per_pos_total": 909.0, "vllm:estimated_flops_per_gpu_total": 0.0, "vllm:estimated_read_bytes_per_gpu_total": 0.0, "vllm:estimated_write_bytes_per_gpu_total": 0.0, "vllm:prefix_cache_queries_total": 73.0, "vllm:prefix_cache_hits_total": 0.0, "vllm:external_prefix_cache_queries_total": 0.0, "vllm:external_prefix_cache_hits_total": 0.0, "vllm:mm_cache_queries_total": 0.0, "vllm:mm_cache_hits_total": 0.0, "vllm:num_preemptions_total": 0.0, "vllm:prompt_tokens_total": 73.0, "vllm:prompt_tokens_by_source_total": 73.0, "vllm:prompt_tokens_cached_total": 0.0, "vllm:generation_tokens_total": 1200.0, "vllm:request_success_total": 1.0, "vllm:request_prompt_tokens_count": 1.0, "vllm:request_prompt_tokens_sum": 73.0, "vllm:request_generation_tokens_count": 1.0, "vllm:request_generation_tokens_sum": 1200.0, "vllm:iteration_tokens_total_count": 294.0, "vllm:iteration_tokens_total_sum": 1273.0, "vllm:request_max_num_generation_tokens_count": 1.0, "vllm:request_max_num_generation_tokens_sum": 1200.0, "vllm:request_params_n_count": 1.0, "vllm:request_params_n_sum": 1.0, "vllm:request_params_max_tokens_count": 1.0, "vllm:request_params_max_tokens_sum": 1200.0, "vllm:time_to_first_token_seconds_count": 1.0, "vllm:time_to_first_token_seconds_sum": 0.08643460273742676, "vllm:inter_token_latency_seconds_count": 293.0, "vllm:inter_token_latency_seconds_sum": 10.385799739975482, "vllm:request_time_per_output_token_seconds_count": 1.0, "vllm:request_time_per_output_token_seconds_sum": 0.008662051492890288, "vllm:e2e_request_latency_seconds_count": 1.0, "vllm:e2e_request_latency_seconds_sum": 10.47216534614563, "vllm:request_queue_time_seconds_count": 1.0, "vllm:request_queue_time_seconds_sum": 1.581897959113121e-05, "vllm:request_inference_time_seconds_count": 1.0, "vllm:request_inference_time_seconds_sum": 10.470735338050872, "vllm:request_prefill_time_seconds_count": 1.0, "vllm:request_prefill_time_seconds_sum": 0.08493559807538986, "vllm:request_decode_time_seconds_count": 1.0, "vllm:request_decode_time_seconds_sum": 10.385799739975482, "vllm:request_prefill_kv_computed_tokens_count": 1.0, "vllm:request_prefill_kv_computed_tokens_sum": 73.0, "vllm:tool_call_parser_invocations_total": 0.0 }, "repetition": 3 }, { "case": "real-review", "kind": "real", "target_tokens": null, "prompt_sha256": "c0bb3ba0a22668094376e5fb184030cde49af12100e7a7bfb5b3a23a98bc2928", "input_ids_sha256": "a8cfd61ed5bd0fdfc172e8b8ee552b21dce5eb3e89a6f7348a0cb7eb91e3b2d6", "prompt_tokens": 58, "output_tokens": 1200, "expected_llama_prompt_tokens": null, "start_monotonic_s": 1196339.06340178, "end_monotonic_s": 1196356.698465199, "api_ttft_s": 0.05297105899080634, "api_e2e_s": 17.635063418885693, "api_tail_s": 17.58200809592381, "first_event_tokens": 1, "api_tail_tok_s": 68.19471322379692, "finish_reason": "length", "fresh": true, "usage": { "prompt_tokens": 58, "total_tokens": 1258, "completion_tokens": 1200 }, "engine_metric_isolated": true, "engine_prefill_s": 0.05050885211676359, "engine_decode_s": 17.581982095958665, "engine_computed_prompt_tokens": 58.0, "engine_prefill_tok_s": 1148.3135642425368, "engine_decode_tok_s": 68.19481406909168, "engine_timing_note": "prefill=first scheduled to first new token; decode=first to last new token", "metrics_delta": { "vllm:spec_decode_num_drafts_total": 495.0, "vllm:spec_decode_num_draft_tokens_total": 3465.0, "vllm:spec_decode_num_accepted_tokens_total": 705.0, "vllm:spec_decode_num_accepted_tokens_per_pos_total": 705.0, "vllm:estimated_flops_per_gpu_total": 0.0, "vllm:estimated_read_bytes_per_gpu_total": 0.0, "vllm:estimated_write_bytes_per_gpu_total": 0.0, "vllm:prefix_cache_queries_total": 58.0, "vllm:prefix_cache_hits_total": 0.0, "vllm:external_prefix_cache_queries_total": 0.0, "vllm:external_prefix_cache_hits_total": 0.0, "vllm:mm_cache_queries_total": 0.0, "vllm:mm_cache_hits_total": 0.0, "vllm:num_preemptions_total": 0.0, "vllm:prompt_tokens_total": 58.0, "vllm:prompt_tokens_by_source_total": 58.0, "vllm:prompt_tokens_cached_total": 0.0, "vllm:generation_tokens_total": 1200.0, "vllm:request_success_total": 1.0, "vllm:request_prompt_tokens_count": 1.0, "vllm:request_prompt_tokens_sum": 58.0, "vllm:request_generation_tokens_count": 1.0, "vllm:request_generation_tokens_sum": 1200.0, "vllm:iteration_tokens_total_count": 496.0, "vllm:iteration_tokens_total_sum": 1258.0, "vllm:request_max_num_generation_tokens_count": 1.0, "vllm:request_max_num_generation_tokens_sum": 1200.0, "vllm:request_params_n_count": 1.0, "vllm:request_params_n_sum": 1.0, "vllm:request_params_max_tokens_count": 1.0, "vllm:request_params_max_tokens_sum": 1200.0, "vllm:time_to_first_token_seconds_count": 1.0, "vllm:time_to_first_token_seconds_sum": 0.051969289779663086, "vllm:inter_token_latency_seconds_count": 495.0, "vllm:inter_token_latency_seconds_sum": 17.581982095958665, "vllm:request_time_per_output_token_seconds_count": 1.0, "vllm:request_time_per_output_token_seconds_sum": 0.01466387163966526, "vllm:e2e_request_latency_seconds_count": 1.0, "vllm:e2e_request_latency_seconds_sum": 17.633753776550293, "vllm:request_queue_time_seconds_count": 1.0, "vllm:request_queue_time_seconds_sum": 7.400056347250938e-06, "vllm:request_inference_time_seconds_count": 1.0, "vllm:request_inference_time_seconds_sum": 17.63249094807543, "vllm:request_prefill_time_seconds_count": 1.0, "vllm:request_prefill_time_seconds_sum": 0.05050885211676359, "vllm:request_decode_time_seconds_count": 1.0, "vllm:request_decode_time_seconds_sum": 17.581982095958665, "vllm:request_prefill_kv_computed_tokens_count": 1.0, "vllm:request_prefill_kv_computed_tokens_sum": 58.0, "vllm:tool_call_parser_invocations_total": 0.0 }, "repetition": 3 }, { "case": "real-qa", "kind": "real", "target_tokens": null, "prompt_sha256": "d94fa364eca08c2c5166c9545153dee2d53b4ab1ce5ff5edcb6e242d2853683b", "input_ids_sha256": "8be6c502b1c9a99e43226c694ac8e06b80662a5321dbbdd92a17aa8ab1a680e1", "prompt_tokens": 61, "output_tokens": 1200, "expected_llama_prompt_tokens": null, "start_monotonic_s": 1196356.714070568, "end_monotonic_s": 1196370.293770512, "api_ttft_s": 0.053544797003269196, "api_e2e_s": 13.579699944006279, "api_tail_s": 13.52607752289623, "first_event_tokens": 1, "api_tail_tok_s": 88.64358480648924, "finish_reason": "length", "fresh": true, "usage": { "prompt_tokens": 61, "total_tokens": 1261, "completion_tokens": 1200 }, "engine_metric_isolated": true, "engine_prefill_s": 0.05102908704429865, "engine_decode_s": 13.52612899709493, "engine_computed_prompt_tokens": 61.0, "engine_prefill_tok_s": 1195.3966557748827, "engine_decode_tok_s": 88.64324746995352, "engine_timing_note": "prefill=first scheduled to first new token; decode=first to last new token", "metrics_delta": { "vllm:spec_decode_num_drafts_total": 381.0, "vllm:spec_decode_num_draft_tokens_total": 2667.0, "vllm:spec_decode_num_accepted_tokens_total": 818.0, "vllm:spec_decode_num_accepted_tokens_per_pos_total": 818.0, "vllm:estimated_flops_per_gpu_total": 0.0, "vllm:estimated_read_bytes_per_gpu_total": 0.0, "vllm:estimated_write_bytes_per_gpu_total": 0.0, "vllm:prefix_cache_queries_total": 61.0, "vllm:prefix_cache_hits_total": 0.0, "vllm:external_prefix_cache_queries_total": 0.0, "vllm:external_prefix_cache_hits_total": 0.0, "vllm:mm_cache_queries_total": 0.0, "vllm:mm_cache_hits_total": 0.0, "vllm:num_preemptions_total": 0.0, "vllm:prompt_tokens_total": 61.0, "vllm:prompt_tokens_by_source_total": 61.0, "vllm:prompt_tokens_cached_total": 0.0, "vllm:generation_tokens_total": 1200.0, "vllm:request_success_total": 1.0, "vllm:request_prompt_tokens_count": 1.0, "vllm:request_prompt_tokens_sum": 61.0, "vllm:request_generation_tokens_count": 1.0, "vllm:request_generation_tokens_sum": 1200.0, "vllm:iteration_tokens_total_count": 382.0, "vllm:iteration_tokens_total_sum": 1261.0, "vllm:request_max_num_generation_tokens_count": 1.0, "vllm:request_max_num_generation_tokens_sum": 1200.0, "vllm:request_params_n_count": 1.0, "vllm:request_params_n_sum": 1.0, "vllm:request_params_max_tokens_count": 1.0, "vllm:request_params_max_tokens_sum": 1200.0, "vllm:time_to_first_token_seconds_count": 1.0, "vllm:time_to_first_token_seconds_sum": 0.052518367767333984, "vllm:inter_token_latency_seconds_count": 381.0, "vllm:inter_token_latency_seconds_sum": 13.52612899709493, "vllm:request_time_per_output_token_seconds_count": 1.0, "vllm:request_time_per_output_token_seconds_sum": 0.011281175143532052, "vllm:e2e_request_latency_seconds_count": 1.0, "vllm:e2e_request_latency_seconds_sum": 13.57853889465332, "vllm:request_queue_time_seconds_count": 1.0, "vllm:request_queue_time_seconds_sum": 1.5238998457789421e-05, "vllm:request_inference_time_seconds_count": 1.0, "vllm:request_inference_time_seconds_sum": 13.577158084139228, "vllm:request_prefill_time_seconds_count": 1.0, "vllm:request_prefill_time_seconds_sum": 0.05102908704429865, "vllm:request_decode_time_seconds_count": 1.0, "vllm:request_decode_time_seconds_sum": 13.52612899709493, "vllm:request_prefill_kv_computed_tokens_count": 1.0, "vllm:request_prefill_kv_computed_tokens_sum": 61.0, "vllm:tool_call_parser_invocations_total": 0.0 }, "repetition": 3 } ]