[ { "case": "ladder-0", "kind": "ladder", "target_tokens": 0, "prompt_sha256": "3965f79a076c8829b8f9083b6f023262873c2a459645e1f0f979e0f3ddb28460", "input_ids_sha256": "1f563f25a0c1605cb77f6cd9e6f55a0a40c46f09abd4722cf438b1b86a5c1d52", "prompt_tokens": 11, "output_tokens": 1000, "expected_llama_prompt_tokens": 11, "start_monotonic_s": 1196181.337346039, "end_monotonic_s": 1196202.270250352, "api_ttft_s": 0.04355988395400345, "api_e2e_s": 20.9329043130856, "api_tail_s": 20.88927050703205, "first_event_tokens": 1, "api_tail_tok_s": 47.823594398076374, "finish_reason": "length", "fresh": false, "usage": { "prompt_tokens": 11, "total_tokens": 1011, "completion_tokens": 1000 }, "engine_metric_isolated": true, "engine_prefill_s": 0.04102520807646215, "engine_decode_s": 20.889342729002237, "engine_computed_prompt_tokens": 11.0, "engine_prefill_tok_s": 268.1278295895141, "engine_decode_tok_s": 47.82342905471188, "engine_timing_note": "prefill=first scheduled to first new token; decode=first to last new token", "metrics_delta": { "vllm:spec_decode_num_drafts_total": 595.0, "vllm:spec_decode_num_draft_tokens_total": 4165.0, "vllm:spec_decode_num_accepted_tokens_total": 405.0, "vllm:spec_decode_num_accepted_tokens_per_pos_total": 405.0, "vllm:estimated_flops_per_gpu_total": 0.0, "vllm:estimated_read_bytes_per_gpu_total": 0.0, "vllm:estimated_write_bytes_per_gpu_total": 0.0, "vllm:prefix_cache_queries_total": 11.0, "vllm:prefix_cache_hits_total": 0.0, "vllm:external_prefix_cache_queries_total": 0.0, "vllm:external_prefix_cache_hits_total": 0.0, "vllm:mm_cache_queries_total": 0.0, "vllm:mm_cache_hits_total": 0.0, "vllm:num_preemptions_total": 0.0, "vllm:prompt_tokens_total": 11.0, "vllm:prompt_tokens_by_source_total": 11.0, "vllm:prompt_tokens_cached_total": 0.0, "vllm:generation_tokens_total": 1000.0, "vllm:request_success_total": 1.0, "vllm:request_prompt_tokens_count": 1.0, "vllm:request_prompt_tokens_sum": 11.0, "vllm:request_generation_tokens_count": 1.0, "vllm:request_generation_tokens_sum": 1000.0, "vllm:iteration_tokens_total_count": 596.0, "vllm:iteration_tokens_total_sum": 1011.0, "vllm:request_max_num_generation_tokens_count": 1.0, "vllm:request_max_num_generation_tokens_sum": 1000.0, "vllm:request_params_n_count": 1.0, "vllm:request_params_n_sum": 1.0, "vllm:request_params_max_tokens_count": 1.0, "vllm:request_params_max_tokens_sum": 1000.0, "vllm:time_to_first_token_seconds_count": 1.0, "vllm:time_to_first_token_seconds_sum": 0.04266071319580078, "vllm:inter_token_latency_seconds_count": 595.0, "vllm:inter_token_latency_seconds_sum": 20.889342729002237, "vllm:request_time_per_output_token_seconds_count": 1.0, "vllm:request_time_per_output_token_seconds_sum": 0.02091025298198422, "vllm:e2e_request_latency_seconds_count": 1.0, "vllm:e2e_request_latency_seconds_sum": 20.93167519569397, "vllm:request_queue_time_seconds_count": 1.0, "vllm:request_queue_time_seconds_sum": 1.2509990483522415e-05, "vllm:request_inference_time_seconds_count": 1.0, "vllm:request_inference_time_seconds_sum": 20.9303679370787, "vllm:request_prefill_time_seconds_count": 1.0, "vllm:request_prefill_time_seconds_sum": 0.04102520807646215, "vllm:request_decode_time_seconds_count": 1.0, "vllm:request_decode_time_seconds_sum": 20.889342729002237, "vllm:request_prefill_kv_computed_tokens_count": 1.0, "vllm:request_prefill_kv_computed_tokens_sum": 11.0, "vllm:tool_call_parser_invocations_total": 0.0 }, "repetition": 2 }, { "case": "ladder-8000", "kind": "ladder", "target_tokens": 8000, "prompt_sha256": "ef9d16376787f5d95c40f8f6f0ee77d902e03a9173cf110c21007bcdec639e98", "input_ids_sha256": "8adc690f4ef99b0b9b6ff75ad2e742bcf2737662529af5706170a23202bc7753", "prompt_tokens": 7887, "output_tokens": 1000, "expected_llama_prompt_tokens": 7887, "start_monotonic_s": 1196202.298360433, "end_monotonic_s": 1196209.256589988, "api_ttft_s": 2.3524720349814743, "api_e2e_s": 6.958229555049911, "api_tail_s": 4.605681638000533, "first_event_tokens": 1, "api_tail_tok_s": 216.90600404453843, "finish_reason": "length", "fresh": false, "usage": { "prompt_tokens": 7887, "total_tokens": 8887, "completion_tokens": 1000 }, "engine_metric_isolated": true, "engine_prefill_s": 2.3446249309927225, "engine_decode_s": 4.609671717043966, "engine_computed_prompt_tokens": 7887.0, "engine_prefill_tok_s": 3363.864256386891, "engine_decode_tok_s": 216.71825269167465, "engine_timing_note": "prefill=first scheduled to first new token; decode=first to last new token", "metrics_delta": { "vllm:spec_decode_num_drafts_total": 126.0, "vllm:spec_decode_num_draft_tokens_total": 882.0, "vllm:spec_decode_num_accepted_tokens_total": 879.0, "vllm:spec_decode_num_accepted_tokens_per_pos_total": 879.0, "vllm:estimated_flops_per_gpu_total": 0.0, "vllm:estimated_read_bytes_per_gpu_total": 0.0, "vllm:estimated_write_bytes_per_gpu_total": 0.0, "vllm:prefix_cache_queries_total": 7887.0, "vllm:prefix_cache_hits_total": 0.0, "vllm:external_prefix_cache_queries_total": 0.0, "vllm:external_prefix_cache_hits_total": 0.0, "vllm:mm_cache_queries_total": 0.0, "vllm:mm_cache_hits_total": 0.0, "vllm:num_preemptions_total": 0.0, "vllm:prompt_tokens_total": 7887.0, "vllm:prompt_tokens_by_source_total": 7887.0, "vllm:prompt_tokens_cached_total": 0.0, "vllm:generation_tokens_total": 1000.0, "vllm:request_success_total": 1.0, "vllm:request_prompt_tokens_count": 1.0, "vllm:request_prompt_tokens_sum": 7887.0, "vllm:request_generation_tokens_count": 1.0, "vllm:request_generation_tokens_sum": 1000.0, "vllm:iteration_tokens_total_count": 127.0, "vllm:iteration_tokens_total_sum": 8887.0, "vllm:request_max_num_generation_tokens_count": 1.0, "vllm:request_max_num_generation_tokens_sum": 1000.0, "vllm:request_params_n_count": 1.0, "vllm:request_params_n_sum": 1.0, "vllm:request_params_max_tokens_count": 1.0, "vllm:request_params_max_tokens_sum": 1000.0, "vllm:time_to_first_token_seconds_count": 1.0, "vllm:time_to_first_token_seconds_sum": 2.347010850906372, "vllm:inter_token_latency_seconds_count": 126.0, "vllm:inter_token_latency_seconds_sum": 4.609671717043966, "vllm:request_time_per_output_token_seconds_count": 1.0, "vllm:request_time_per_output_token_seconds_sum": 0.004614286003047002, "vllm:e2e_request_latency_seconds_count": 1.0, "vllm:e2e_request_latency_seconds_sum": 6.956257343292236, "vllm:request_queue_time_seconds_count": 1.0, "vllm:request_queue_time_seconds_sum": 1.535983756184578e-05, "vllm:request_inference_time_seconds_count": 1.0, "vllm:request_inference_time_seconds_sum": 6.954296648036689, "vllm:request_prefill_time_seconds_count": 1.0, "vllm:request_prefill_time_seconds_sum": 2.3446249309927225, "vllm:request_decode_time_seconds_count": 1.0, "vllm:request_decode_time_seconds_sum": 4.609671717043966, "vllm:request_prefill_kv_computed_tokens_count": 1.0, "vllm:request_prefill_kv_computed_tokens_sum": 7887.0, "vllm:tool_call_parser_invocations_total": 0.0 }, "repetition": 2 }, { "case": "ladder-16000", "kind": "ladder", "target_tokens": 16000, "prompt_sha256": "17610c10a63b400e07668fc3a875a67548796ea1dd18e67702fb066da804a2b8", "input_ids_sha256": "179aac711afb83a653889e3dbf76a4c9367280452995ef638bbb341c717662da", "prompt_tokens": 16050, "output_tokens": 1000, "expected_llama_prompt_tokens": 16050, "start_monotonic_s": 1196209.316570932, "end_monotonic_s": 1196217.117486243, "api_ttft_s": 3.136147879064083, "api_e2e_s": 7.8009153108578175, "api_tail_s": 4.664693259866908, "first_event_tokens": 1, "api_tail_tok_s": 214.16199187093028, "finish_reason": "length", "fresh": false, "usage": { "prompt_tokens": 16050, "total_tokens": 17050, "completion_tokens": 1000 }, "engine_metric_isolated": true, "engine_prefill_s": 3.1221834709867835, "engine_decode_s": 4.67257089400664, "engine_computed_prompt_tokens": 9890.0, "engine_prefill_tok_s": 3167.6549734837363, "engine_decode_tok_s": 213.8009294372368, "engine_timing_note": "prefill=first scheduled to first new token; decode=first to last new token", "metrics_delta": { "vllm:spec_decode_num_drafts_total": 126.0, "vllm:spec_decode_num_draft_tokens_total": 882.0, "vllm:spec_decode_num_accepted_tokens_total": 879.0, "vllm:spec_decode_num_accepted_tokens_per_pos_total": 879.0, "vllm:estimated_flops_per_gpu_total": 0.0, "vllm:estimated_read_bytes_per_gpu_total": 0.0, "vllm:estimated_write_bytes_per_gpu_total": 0.0, "vllm:prefix_cache_queries_total": 16050.0, "vllm:prefix_cache_hits_total": 6160.0, "vllm:external_prefix_cache_queries_total": 0.0, "vllm:external_prefix_cache_hits_total": 0.0, "vllm:mm_cache_queries_total": 0.0, "vllm:mm_cache_hits_total": 0.0, "vllm:num_preemptions_total": 0.0, "vllm:prompt_tokens_total": 16050.0, "vllm:prompt_tokens_by_source_total": 16050.0, "vllm:prompt_tokens_cached_total": 6160.0, "vllm:generation_tokens_total": 1000.0, "vllm:request_success_total": 1.0, "vllm:request_prompt_tokens_count": 1.0, "vllm:request_prompt_tokens_sum": 16050.0, "vllm:request_generation_tokens_count": 1.0, "vllm:request_generation_tokens_sum": 1000.0, "vllm:iteration_tokens_total_count": 127.0, "vllm:iteration_tokens_total_sum": 10890.0, "vllm:request_max_num_generation_tokens_count": 1.0, "vllm:request_max_num_generation_tokens_sum": 1000.0, "vllm:request_params_n_count": 1.0, "vllm:request_params_n_sum": 1.0, "vllm:request_params_max_tokens_count": 1.0, "vllm:request_params_max_tokens_sum": 1000.0, "vllm:time_to_first_token_seconds_count": 1.0, "vllm:time_to_first_token_seconds_sum": 3.125274896621704, "vllm:inter_token_latency_seconds_count": 126.0, "vllm:inter_token_latency_seconds_sum": 4.67257089400664, "vllm:request_time_per_output_token_seconds_count": 1.0, "vllm:request_time_per_output_token_seconds_sum": 0.004677248142148788, "vllm:e2e_request_latency_seconds_count": 1.0, "vllm:e2e_request_latency_seconds_sum": 7.797501564025879, "vllm:request_queue_time_seconds_count": 1.0, "vllm:request_queue_time_seconds_sum": 2.10499856621027e-05, "vllm:request_inference_time_seconds_count": 1.0, "vllm:request_inference_time_seconds_sum": 7.794754364993423, "vllm:request_prefill_time_seconds_count": 1.0, "vllm:request_prefill_time_seconds_sum": 3.1221834709867835, "vllm:request_decode_time_seconds_count": 1.0, "vllm:request_decode_time_seconds_sum": 4.67257089400664, "vllm:request_prefill_kv_computed_tokens_count": 1.0, "vllm:request_prefill_kv_computed_tokens_sum": 9890.0, "vllm:tool_call_parser_invocations_total": 0.0 }, "repetition": 2 }, { "case": "ladder-32000", "kind": "ladder", "target_tokens": 32000, "prompt_sha256": "d2b2c90bc484f3c1ca1711f0a2bac03a3ea0f5c9234d48231dc746d998114fb8", "input_ids_sha256": "f6305d07154b3068b74b6ca47f8d0755c1fa9943495ffb02c16fa4c36d0a5abf", "prompt_tokens": 32460, "output_tokens": 1000, "expected_llama_prompt_tokens": 32460, "start_monotonic_s": 1196217.189900821, "end_monotonic_s": 1196228.432949787, "api_ttft_s": 6.070800324901938, "api_e2e_s": 11.243048965930939, "api_tail_s": 5.172201300039887, "first_event_tokens": 1, "api_tail_tok_s": 193.14793490197218, "finish_reason": "length", "fresh": false, "usage": { "prompt_tokens": 32460, "total_tokens": 33460, "completion_tokens": 1000 }, "engine_metric_isolated": true, "engine_prefill_s": 6.045140308793634, "engine_decode_s": 5.189917820040137, "engine_computed_prompt_tokens": 17500.0, "engine_prefill_tok_s": 2894.887315443021, "engine_decode_tok_s": 192.48859705302118, "engine_timing_note": "prefill=first scheduled to first new token; decode=first to last new token", "metrics_delta": { "vllm:spec_decode_num_drafts_total": 136.0, "vllm:spec_decode_num_draft_tokens_total": 952.0, "vllm:spec_decode_num_accepted_tokens_total": 865.0, "vllm:spec_decode_num_accepted_tokens_per_pos_total": 865.0, "vllm:estimated_flops_per_gpu_total": 0.0, "vllm:estimated_read_bytes_per_gpu_total": 0.0, "vllm:estimated_write_bytes_per_gpu_total": 0.0, "vllm:prefix_cache_queries_total": 32460.0, "vllm:prefix_cache_hits_total": 14960.0, "vllm:external_prefix_cache_queries_total": 0.0, "vllm:external_prefix_cache_hits_total": 0.0, "vllm:mm_cache_queries_total": 0.0, "vllm:mm_cache_hits_total": 0.0, "vllm:num_preemptions_total": 0.0, "vllm:prompt_tokens_total": 32460.0, "vllm:prompt_tokens_by_source_total": 32460.0, "vllm:prompt_tokens_cached_total": 14960.0, "vllm:generation_tokens_total": 1000.0, "vllm:request_success_total": 1.0, "vllm:request_prompt_tokens_count": 1.0, "vllm:request_prompt_tokens_sum": 32460.0, "vllm:request_generation_tokens_count": 1.0, "vllm:request_generation_tokens_sum": 1000.0, "vllm:iteration_tokens_total_count": 137.0, "vllm:iteration_tokens_total_sum": 18500.0, "vllm:request_max_num_generation_tokens_count": 1.0, "vllm:request_max_num_generation_tokens_sum": 1000.0, "vllm:request_params_n_count": 1.0, "vllm:request_params_n_sum": 1.0, "vllm:request_params_max_tokens_count": 1.0, "vllm:request_params_max_tokens_sum": 1000.0, "vllm:time_to_first_token_seconds_count": 1.0, "vllm:time_to_first_token_seconds_sum": 6.049266338348389, "vllm:inter_token_latency_seconds_count": 136.0, "vllm:inter_token_latency_seconds_sum": 5.189917820040137, "vllm:request_time_per_output_token_seconds_count": 1.0, "vllm:request_time_per_output_token_seconds_sum": 0.005195112932973117, "vllm:e2e_request_latency_seconds_count": 1.0, "vllm:e2e_request_latency_seconds_sum": 11.238527774810791, "vllm:request_queue_time_seconds_count": 1.0, "vllm:request_queue_time_seconds_sum": 1.628999598324299e-05, "vllm:request_inference_time_seconds_count": 1.0, "vllm:request_inference_time_seconds_sum": 11.23505812883377, "vllm:request_prefill_time_seconds_count": 1.0, "vllm:request_prefill_time_seconds_sum": 6.045140308793634, "vllm:request_decode_time_seconds_count": 1.0, "vllm:request_decode_time_seconds_sum": 5.189917820040137, "vllm:request_prefill_kv_computed_tokens_count": 1.0, "vllm:request_prefill_kv_computed_tokens_sum": 17500.0, "vllm:tool_call_parser_invocations_total": 0.0 }, "repetition": 2 }, { "case": "pp512", "kind": "pp0", "target_tokens": 512, "prompt_sha256": "00687b0956e2b6962bfb5759e88b406712148dd82281feffa52d7da81ac07992", "input_ids_sha256": "4a7c2a4ee25dbaec6b96571f42a173757ab2d90e3a9d05d47a1832357c2a9db9", "prompt_tokens": 512, "output_tokens": 64, "expected_llama_prompt_tokens": null, "start_monotonic_s": 1196228.454878311, "end_monotonic_s": 1196228.949706597, "api_ttft_s": 0.1774438340216875, "api_e2e_s": 0.4948282861150801, "api_tail_s": 0.3173055301886052, "first_event_tokens": 1, "api_tail_tok_s": 198.546807433684, "finish_reason": "length", "fresh": true, "usage": { "prompt_tokens": 512, "total_tokens": 576, "completion_tokens": 64 }, "engine_metric_isolated": true, "engine_prefill_s": 0.1736438910011202, "engine_decode_s": 0.31792341312393546, "engine_computed_prompt_tokens": 512.0, "engine_prefill_tok_s": 2948.5632753800537, "engine_decode_tok_s": 198.1609324741391, "engine_timing_note": "prefill=first scheduled to first new token; decode=first to last new token", "metrics_delta": { "vllm:spec_decode_num_drafts_total": 9.0, "vllm:spec_decode_num_draft_tokens_total": 63.0, "vllm:spec_decode_num_accepted_tokens_total": 59.0, "vllm:spec_decode_num_accepted_tokens_per_pos_total": 59.0, "vllm:estimated_flops_per_gpu_total": 0.0, "vllm:estimated_read_bytes_per_gpu_total": 0.0, "vllm:estimated_write_bytes_per_gpu_total": 0.0, "vllm:prefix_cache_queries_total": 512.0, "vllm:prefix_cache_hits_total": 0.0, "vllm:external_prefix_cache_queries_total": 0.0, "vllm:external_prefix_cache_hits_total": 0.0, "vllm:mm_cache_queries_total": 0.0, "vllm:mm_cache_hits_total": 0.0, "vllm:num_preemptions_total": 0.0, "vllm:prompt_tokens_total": 512.0, "vllm:prompt_tokens_by_source_total": 512.0, "vllm:prompt_tokens_cached_total": 0.0, "vllm:generation_tokens_total": 64.0, "vllm:request_success_total": 1.0, "vllm:request_prompt_tokens_count": 1.0, "vllm:request_prompt_tokens_sum": 512.0, "vllm:request_generation_tokens_count": 1.0, "vllm:request_generation_tokens_sum": 64.0, "vllm:iteration_tokens_total_count": 10.0, "vllm:iteration_tokens_total_sum": 576.0, "vllm:request_max_num_generation_tokens_count": 1.0, "vllm:request_max_num_generation_tokens_sum": 64.0, "vllm:request_params_n_count": 1.0, "vllm:request_params_n_sum": 1.0, "vllm:request_params_max_tokens_count": 1.0, "vllm:request_params_max_tokens_sum": 64.0, "vllm:time_to_first_token_seconds_count": 1.0, "vllm:time_to_first_token_seconds_sum": 0.17576313018798828, "vllm:inter_token_latency_seconds_count": 9.0, "vllm:inter_token_latency_seconds_sum": 0.31792341312393546, "vllm:request_time_per_output_token_seconds_count": 1.0, "vllm:request_time_per_output_token_seconds_sum": 0.0050464033829196175, "vllm:e2e_request_latency_seconds_count": 1.0, "vllm:e2e_request_latency_seconds_sum": 0.493391752243042, "vllm:request_queue_time_seconds_count": 1.0, "vllm:request_queue_time_seconds_sum": 1.8630875274538994e-05, "vllm:request_inference_time_seconds_count": 1.0, "vllm:request_inference_time_seconds_sum": 0.49156730412505567, "vllm:request_prefill_time_seconds_count": 1.0, "vllm:request_prefill_time_seconds_sum": 0.1736438910011202, "vllm:request_decode_time_seconds_count": 1.0, "vllm:request_decode_time_seconds_sum": 0.31792341312393546, "vllm:request_prefill_kv_computed_tokens_count": 1.0, "vllm:request_prefill_kv_computed_tokens_sum": 512.0, "vllm:tool_call_parser_invocations_total": 0.0 }, "repetition": 2 }, { "case": "pp2048", "kind": "pp0", "target_tokens": 2048, "prompt_sha256": "c21aa67b768e16f833142c7aad2014477820ee970ac9e41290a35eaa3f105b72", "input_ids_sha256": "c0fd1fc7b2d01c93a8897da47cb5aca81bb77a25869d459434bccef5236dc0c0", "prompt_tokens": 1978, "output_tokens": 64, "expected_llama_prompt_tokens": null, "start_monotonic_s": 1196228.967702494, "end_monotonic_s": 1196229.93440984, "api_ttft_s": 0.6386040279176086, "api_e2e_s": 0.9667073460295796, "api_tail_s": 0.3280226660426706, "first_event_tokens": 1, "api_tail_tok_s": 192.05989866506567, "finish_reason": "length", "fresh": true, "usage": { "prompt_tokens": 1978, "total_tokens": 2042, "completion_tokens": 64 }, "engine_metric_isolated": true, "engine_prefill_s": 0.6339886360801756, "engine_decode_s": 0.32958722999319434, "engine_computed_prompt_tokens": 1978.0, "engine_prefill_tok_s": 3119.9297391662676, "engine_decode_tok_s": 191.1481825351695, "engine_timing_note": "prefill=first scheduled to first new token; decode=first to last new token", "metrics_delta": { "vllm:spec_decode_num_drafts_total": 9.0, "vllm:spec_decode_num_draft_tokens_total": 63.0, "vllm:spec_decode_num_accepted_tokens_total": 59.0, "vllm:spec_decode_num_accepted_tokens_per_pos_total": 59.0, "vllm:estimated_flops_per_gpu_total": 0.0, "vllm:estimated_read_bytes_per_gpu_total": 0.0, "vllm:estimated_write_bytes_per_gpu_total": 0.0, "vllm:prefix_cache_queries_total": 1978.0, "vllm:prefix_cache_hits_total": 0.0, "vllm:external_prefix_cache_queries_total": 0.0, "vllm:external_prefix_cache_hits_total": 0.0, "vllm:mm_cache_queries_total": 0.0, "vllm:mm_cache_hits_total": 0.0, "vllm:num_preemptions_total": 0.0, "vllm:prompt_tokens_total": 1978.0, "vllm:prompt_tokens_by_source_total": 1978.0, "vllm:prompt_tokens_cached_total": 0.0, "vllm:generation_tokens_total": 64.0, "vllm:request_success_total": 1.0, "vllm:request_prompt_tokens_count": 1.0, "vllm:request_prompt_tokens_sum": 1978.0, "vllm:request_generation_tokens_count": 1.0, "vllm:request_generation_tokens_sum": 64.0, "vllm:iteration_tokens_total_count": 10.0, "vllm:iteration_tokens_total_sum": 2042.0, "vllm:request_max_num_generation_tokens_count": 1.0, "vllm:request_max_num_generation_tokens_sum": 64.0, "vllm:request_params_n_count": 1.0, "vllm:request_params_n_sum": 1.0, "vllm:request_params_max_tokens_count": 1.0, "vllm:request_params_max_tokens_sum": 64.0, "vllm:time_to_first_token_seconds_count": 1.0, "vllm:time_to_first_token_seconds_sum": 0.6358683109283447, "vllm:inter_token_latency_seconds_count": 9.0, "vllm:inter_token_latency_seconds_sum": 0.32958722999319434, "vllm:request_time_per_output_token_seconds_count": 1.0, "vllm:request_time_per_output_token_seconds_sum": 0.005231543333225314, "vllm:e2e_request_latency_seconds_count": 1.0, "vllm:e2e_request_latency_seconds_sum": 0.9651498794555664, "vllm:request_queue_time_seconds_count": 1.0, "vllm:request_queue_time_seconds_sum": 1.546088606119156e-05, "vllm:request_inference_time_seconds_count": 1.0, "vllm:request_inference_time_seconds_sum": 0.96357586607337, "vllm:request_prefill_time_seconds_count": 1.0, "vllm:request_prefill_time_seconds_sum": 0.6339886360801756, "vllm:request_decode_time_seconds_count": 1.0, "vllm:request_decode_time_seconds_sum": 0.32958722999319434, "vllm:request_prefill_kv_computed_tokens_count": 1.0, "vllm:request_prefill_kv_computed_tokens_sum": 1978.0, "vllm:tool_call_parser_invocations_total": 0.0 }, "repetition": 2 }, { "case": "pp8192", "kind": "pp0", "target_tokens": 8192, "prompt_sha256": "a29d179294fb683902960258e22071ae39bfbf7ebc69c21818c88bda08aa04bc", "input_ids_sha256": "737621140e7867e679c4d71011d3dc9f26ccc84238c657bf1348a8b9d0ebaa2e", "prompt_tokens": 8077, "output_tokens": 64, "expected_llama_prompt_tokens": null, "start_monotonic_s": 1196229.962439608, "end_monotonic_s": 1196232.897341246, "api_ttft_s": 2.422477585962042, "api_e2e_s": 2.9349016379565, "api_tail_s": 0.5123184099793434, "first_event_tokens": 1, "api_tail_tok_s": 122.97040038545589, "finish_reason": "length", "fresh": true, "usage": { "prompt_tokens": 8077, "total_tokens": 8141, "completion_tokens": 64 }, "engine_metric_isolated": true, "engine_prefill_s": 2.4144134370144457, "engine_decode_s": 0.5166354349348694, "engine_computed_prompt_tokens": 8077.0, "engine_prefill_tok_s": 3345.3259811159983, "engine_decode_tok_s": 121.94285513525067, "engine_timing_note": "prefill=first scheduled to first new token; decode=first to last new token", "metrics_delta": { "vllm:spec_decode_num_drafts_total": 14.0, "vllm:spec_decode_num_draft_tokens_total": 98.0, "vllm:spec_decode_num_accepted_tokens_total": 49.0, "vllm:spec_decode_num_accepted_tokens_per_pos_total": 49.0, "vllm:estimated_flops_per_gpu_total": 0.0, "vllm:estimated_read_bytes_per_gpu_total": 0.0, "vllm:estimated_write_bytes_per_gpu_total": 0.0, "vllm:prefix_cache_queries_total": 8077.0, "vllm:prefix_cache_hits_total": 0.0, "vllm:external_prefix_cache_queries_total": 0.0, "vllm:external_prefix_cache_hits_total": 0.0, "vllm:mm_cache_queries_total": 0.0, "vllm:mm_cache_hits_total": 0.0, "vllm:num_preemptions_total": 0.0, "vllm:prompt_tokens_total": 8077.0, "vllm:prompt_tokens_by_source_total": 8077.0, "vllm:prompt_tokens_cached_total": 0.0, "vllm:generation_tokens_total": 64.0, "vllm:request_success_total": 1.0, "vllm:request_prompt_tokens_count": 1.0, "vllm:request_prompt_tokens_sum": 8077.0, "vllm:request_generation_tokens_count": 1.0, "vllm:request_generation_tokens_sum": 64.0, "vllm:iteration_tokens_total_count": 15.0, "vllm:iteration_tokens_total_sum": 8141.0, "vllm:request_max_num_generation_tokens_count": 1.0, "vllm:request_max_num_generation_tokens_sum": 64.0, "vllm:request_params_n_count": 1.0, "vllm:request_params_n_sum": 1.0, "vllm:request_params_max_tokens_count": 1.0, "vllm:request_params_max_tokens_sum": 64.0, "vllm:time_to_first_token_seconds_count": 1.0, "vllm:time_to_first_token_seconds_sum": 2.416694402694702, "vllm:inter_token_latency_seconds_count": 14.0, "vllm:inter_token_latency_seconds_sum": 0.5166354349348694, "vllm:request_time_per_output_token_seconds_count": 1.0, "vllm:request_time_per_output_token_seconds_sum": 0.008200562459283633, "vllm:e2e_request_latency_seconds_count": 1.0, "vllm:e2e_request_latency_seconds_sum": 2.9329190254211426, "vllm:request_queue_time_seconds_count": 1.0, "vllm:request_queue_time_seconds_sum": 1.5821075066924095e-05, "vllm:request_inference_time_seconds_count": 1.0, "vllm:request_inference_time_seconds_sum": 2.931048871949315, "vllm:request_prefill_time_seconds_count": 1.0, "vllm:request_prefill_time_seconds_sum": 2.4144134370144457, "vllm:request_decode_time_seconds_count": 1.0, "vllm:request_decode_time_seconds_sum": 0.5166354349348694, "vllm:request_prefill_kv_computed_tokens_count": 1.0, "vllm:request_prefill_kv_computed_tokens_sum": 8077.0, "vllm:tool_call_parser_invocations_total": 0.0 }, "repetition": 2 }, { "case": "real-design", "kind": "real", "target_tokens": null, "prompt_sha256": "e6c65ab2c1586c13c0e3e223551dbd804d5f0ae382f7cf3c665c297b3d18b291", "input_ids_sha256": "6af826ccc823c0456c53eddecd1e695f6dcfe2337aa5d71a24a30aa7f2413d1a", "prompt_tokens": 73, "output_tokens": 1200, "expected_llama_prompt_tokens": null, "start_monotonic_s": 1196232.912831198, "end_monotonic_s": 1196241.546806131, "api_ttft_s": 0.08721214300021529, "api_e2e_s": 8.633974933065474, "api_tail_s": 8.546661727130413, "first_event_tokens": 1, "api_tail_tok_s": 140.2886926241517, "finish_reason": "length", "fresh": true, "usage": { "prompt_tokens": 73, "total_tokens": 1273, "completion_tokens": 1200 }, "engine_metric_isolated": true, "engine_prefill_s": 0.08443758194334805, "engine_decode_s": 8.547019785037264, "engine_computed_prompt_tokens": 73.0, "engine_prefill_tok_s": 864.5439426365632, "engine_decode_tok_s": 140.28281554922978, "engine_timing_note": "prefill=first scheduled to first new token; decode=first to last new token", "metrics_delta": { "vllm:spec_decode_num_drafts_total": 241.0, "vllm:spec_decode_num_draft_tokens_total": 1687.0, "vllm:spec_decode_num_accepted_tokens_total": 959.0, "vllm:spec_decode_num_accepted_tokens_per_pos_total": 959.0, "vllm:estimated_flops_per_gpu_total": 0.0, "vllm:estimated_read_bytes_per_gpu_total": 0.0, "vllm:estimated_write_bytes_per_gpu_total": 0.0, "vllm:prefix_cache_queries_total": 73.0, "vllm:prefix_cache_hits_total": 0.0, "vllm:external_prefix_cache_queries_total": 0.0, "vllm:external_prefix_cache_hits_total": 0.0, "vllm:mm_cache_queries_total": 0.0, "vllm:mm_cache_hits_total": 0.0, "vllm:num_preemptions_total": 0.0, "vllm:prompt_tokens_total": 73.0, "vllm:prompt_tokens_by_source_total": 73.0, "vllm:prompt_tokens_cached_total": 0.0, "vllm:generation_tokens_total": 1200.0, "vllm:request_success_total": 1.0, "vllm:request_prompt_tokens_count": 1.0, "vllm:request_prompt_tokens_sum": 73.0, "vllm:request_generation_tokens_count": 1.0, "vllm:request_generation_tokens_sum": 1200.0, "vllm:iteration_tokens_total_count": 242.0, "vllm:iteration_tokens_total_sum": 1273.0, "vllm:request_max_num_generation_tokens_count": 1.0, "vllm:request_max_num_generation_tokens_sum": 1200.0, "vllm:request_params_n_count": 1.0, "vllm:request_params_n_sum": 1.0, "vllm:request_params_max_tokens_count": 1.0, "vllm:request_params_max_tokens_sum": 1200.0, "vllm:time_to_first_token_seconds_count": 1.0, "vllm:time_to_first_token_seconds_sum": 0.08603048324584961, "vllm:inter_token_latency_seconds_count": 241.0, "vllm:inter_token_latency_seconds_sum": 8.547019785037264, "vllm:request_time_per_output_token_seconds_count": 1.0, "vllm:request_time_per_output_token_seconds_sum": 0.007128456868254612, "vllm:e2e_request_latency_seconds_count": 1.0, "vllm:e2e_request_latency_seconds_sum": 8.632781028747559, "vllm:request_queue_time_seconds_count": 1.0, "vllm:request_queue_time_seconds_sum": 1.437007449567318e-05, "vllm:request_inference_time_seconds_count": 1.0, "vllm:request_inference_time_seconds_sum": 8.631457366980612, "vllm:request_prefill_time_seconds_count": 1.0, "vllm:request_prefill_time_seconds_sum": 0.08443758194334805, "vllm:request_decode_time_seconds_count": 1.0, "vllm:request_decode_time_seconds_sum": 8.547019785037264, "vllm:request_prefill_kv_computed_tokens_count": 1.0, "vllm:request_prefill_kv_computed_tokens_sum": 73.0, "vllm:tool_call_parser_invocations_total": 0.0 }, "repetition": 2 }, { "case": "real-review", "kind": "real", "target_tokens": null, "prompt_sha256": "c0bb3ba0a22668094376e5fb184030cde49af12100e7a7bfb5b3a23a98bc2928", "input_ids_sha256": "a8cfd61ed5bd0fdfc172e8b8ee552b21dce5eb3e89a6f7348a0cb7eb91e3b2d6", "prompt_tokens": 58, "output_tokens": 1200, "expected_llama_prompt_tokens": null, "start_monotonic_s": 1196241.562012306, "end_monotonic_s": 1196259.238789429, "api_ttft_s": 0.05375650501810014, "api_e2e_s": 17.676777123007923, "api_tail_s": 17.622930916026235, "first_event_tokens": 1, "api_tail_tok_s": 68.03635591112902, "finish_reason": "length", "fresh": true, "usage": { "prompt_tokens": 58, "total_tokens": 1258, "completion_tokens": 1200 }, "engine_metric_isolated": true, "engine_prefill_s": 0.05089226202107966, "engine_decode_s": 17.62301445682533, "engine_computed_prompt_tokens": 58.0, "engine_prefill_tok_s": 1139.6624495876465, "engine_decode_tok_s": 68.03603338903416, "engine_timing_note": "prefill=first scheduled to first new token; decode=first to last new token", "metrics_delta": { "vllm:spec_decode_num_drafts_total": 496.0, "vllm:spec_decode_num_draft_tokens_total": 3472.0, "vllm:spec_decode_num_accepted_tokens_total": 704.0, "vllm:spec_decode_num_accepted_tokens_per_pos_total": 704.0, "vllm:estimated_flops_per_gpu_total": 0.0, "vllm:estimated_read_bytes_per_gpu_total": 0.0, "vllm:estimated_write_bytes_per_gpu_total": 0.0, "vllm:prefix_cache_queries_total": 58.0, "vllm:prefix_cache_hits_total": 0.0, "vllm:external_prefix_cache_queries_total": 0.0, "vllm:external_prefix_cache_hits_total": 0.0, "vllm:mm_cache_queries_total": 0.0, "vllm:mm_cache_hits_total": 0.0, "vllm:num_preemptions_total": 0.0, "vllm:prompt_tokens_total": 58.0, "vllm:prompt_tokens_by_source_total": 58.0, "vllm:prompt_tokens_cached_total": 0.0, "vllm:generation_tokens_total": 1200.0, "vllm:request_success_total": 1.0, "vllm:request_prompt_tokens_count": 1.0, "vllm:request_prompt_tokens_sum": 58.0, "vllm:request_generation_tokens_count": 1.0, "vllm:request_generation_tokens_sum": 1200.0, "vllm:iteration_tokens_total_count": 497.0, "vllm:iteration_tokens_total_sum": 1258.0, "vllm:request_max_num_generation_tokens_count": 1.0, "vllm:request_max_num_generation_tokens_sum": 1200.0, "vllm:request_params_n_count": 1.0, "vllm:request_params_n_sum": 1.0, "vllm:request_params_max_tokens_count": 1.0, "vllm:request_params_max_tokens_sum": 1200.0, "vllm:time_to_first_token_seconds_count": 1.0, "vllm:time_to_first_token_seconds_sum": 0.052437782287597656, "vllm:inter_token_latency_seconds_count": 496.0, "vllm:inter_token_latency_seconds_sum": 17.62301445682533, "vllm:request_time_per_output_token_seconds_count": 1.0, "vllm:request_time_per_output_token_seconds_sum": 0.01469809379218126, "vllm:e2e_request_latency_seconds_count": 1.0, "vllm:e2e_request_latency_seconds_sum": 17.67529320716858, "vllm:request_queue_time_seconds_count": 1.0, "vllm:request_queue_time_seconds_sum": 1.6489997506141663e-05, "vllm:request_inference_time_seconds_count": 1.0, "vllm:request_inference_time_seconds_sum": 17.67390671884641, "vllm:request_prefill_time_seconds_count": 1.0, "vllm:request_prefill_time_seconds_sum": 0.05089226202107966, "vllm:request_decode_time_seconds_count": 1.0, "vllm:request_decode_time_seconds_sum": 17.62301445682533, "vllm:request_prefill_kv_computed_tokens_count": 1.0, "vllm:request_prefill_kv_computed_tokens_sum": 58.0, "vllm:tool_call_parser_invocations_total": 0.0 }, "repetition": 2 }, { "case": "real-qa", "kind": "real", "target_tokens": null, "prompt_sha256": "d94fa364eca08c2c5166c9545153dee2d53b4ab1ce5ff5edcb6e242d2853683b", "input_ids_sha256": "8be6c502b1c9a99e43226c694ac8e06b80662a5321dbbdd92a17aa8ab1a680e1", "prompt_tokens": 61, "output_tokens": 1200, "expected_llama_prompt_tokens": null, "start_monotonic_s": 1196259.256081629, "end_monotonic_s": 1196272.726434134, "api_ttft_s": 0.0538298359606415, "api_e2e_s": 13.47035250486806, "api_tail_s": 13.416451558005065, "first_event_tokens": 1, "api_tail_tok_s": 89.3678924577195, "finish_reason": "length", "fresh": true, "usage": { "prompt_tokens": 61, "total_tokens": 1261, "completion_tokens": 1200 }, "engine_metric_isolated": true, "engine_prefill_s": 0.05107556493021548, "engine_decode_s": 13.416722763096914, "engine_computed_prompt_tokens": 61.0, "engine_prefill_tok_s": 1194.3088653712255, "engine_decode_tok_s": 89.36608597875215, "engine_timing_note": "prefill=first scheduled to first new token; decode=first to last new token", "metrics_delta": { "vllm:spec_decode_num_drafts_total": 378.0, "vllm:spec_decode_num_draft_tokens_total": 2646.0, "vllm:spec_decode_num_accepted_tokens_total": 826.0, "vllm:spec_decode_num_accepted_tokens_per_pos_total": 826.0, "vllm:estimated_flops_per_gpu_total": 0.0, "vllm:estimated_read_bytes_per_gpu_total": 0.0, "vllm:estimated_write_bytes_per_gpu_total": 0.0, "vllm:prefix_cache_queries_total": 61.0, "vllm:prefix_cache_hits_total": 0.0, "vllm:external_prefix_cache_queries_total": 0.0, "vllm:external_prefix_cache_hits_total": 0.0, "vllm:mm_cache_queries_total": 0.0, "vllm:mm_cache_hits_total": 0.0, "vllm:num_preemptions_total": 0.0, "vllm:prompt_tokens_total": 61.0, "vllm:prompt_tokens_by_source_total": 61.0, "vllm:prompt_tokens_cached_total": 0.0, "vllm:generation_tokens_total": 1200.0, "vllm:request_success_total": 1.0, "vllm:request_prompt_tokens_count": 1.0, "vllm:request_prompt_tokens_sum": 61.0, "vllm:request_generation_tokens_count": 1.0, "vllm:request_generation_tokens_sum": 1200.0, "vllm:iteration_tokens_total_count": 379.0, "vllm:iteration_tokens_total_sum": 1261.0, "vllm:request_max_num_generation_tokens_count": 1.0, "vllm:request_max_num_generation_tokens_sum": 1200.0, "vllm:request_params_n_count": 1.0, "vllm:request_params_n_sum": 1.0, "vllm:request_params_max_tokens_count": 1.0, "vllm:request_params_max_tokens_sum": 1200.0, "vllm:time_to_first_token_seconds_count": 1.0, "vllm:time_to_first_token_seconds_sum": 0.05268359184265137, "vllm:inter_token_latency_seconds_count": 378.0, "vllm:inter_token_latency_seconds_sum": 13.416722763096914, "vllm:request_time_per_output_token_seconds_count": 1.0, "vllm:request_time_per_output_token_seconds_sum": 0.01118992724194906, "vllm:e2e_request_latency_seconds_count": 1.0, "vllm:e2e_request_latency_seconds_sum": 13.469149827957153, "vllm:request_queue_time_seconds_count": 1.0, "vllm:request_queue_time_seconds_sum": 1.53400469571352e-05, "vllm:request_inference_time_seconds_count": 1.0, "vllm:request_inference_time_seconds_sum": 13.46779832802713, "vllm:request_prefill_time_seconds_count": 1.0, "vllm:request_prefill_time_seconds_sum": 0.05107556493021548, "vllm:request_decode_time_seconds_count": 1.0, "vllm:request_decode_time_seconds_sum": 13.416722763096914, "vllm:request_prefill_kv_computed_tokens_count": 1.0, "vllm:request_prefill_kv_computed_tokens_sum": 61.0, "vllm:tool_call_parser_invocations_total": 0.0 }, "repetition": 2 } ]