{ "title": "Qwen3.8-Flash-Next on one MI350P", "date": "2026-10-30", "concurrency": [ 1, 2, 4, 8, 16, 32, 64 ], "scope": "Speed and serving: one MI350P, configured GPU power limit 450 W. Full-stack comparison with different quantized checkpoints. The Paiton CDNA4 engine is proprietary and not released. Accuracy BF16 control is on MI355X.", "engines": [ { "id": "paiton", "label": "Paiton CDNA4", "source": "lmx-paiton.json", "checkpoint": "EliovpAI/Qwen3.8-Flash-Next-W4A8-Paiton-CDNA4" }, { "id": "vllm", "label": "vLLM + AITER", "source": "lmx-vllm.json", "checkpoint": "amd/Qwen3.8-Flash-Next-Quark-MXFP4-PLEFP8" }, { "id": "sglang16", "label": "SGLang MTP · 16 seqs", "source": "lmx-sglang-mtp16.json", "checkpoint": "amd/Qwen3.8-Flash-Next-Quark-MXFP4-PLEFP8" }, { "id": "sglang8", "label": "SGLang MTP · 8 seqs", "source": "lmx-sglang-mtp8.json", "checkpoint": "amd/Qwen3.8-Flash-Next-Quark-MXFP4-PLEFP8" }, { "id": "sglangNoMtp", "label": "SGLang · no MTP", "source": "lmx-sglang-no-mtp.json", "checkpoint": "amd/Qwen3.8-Flash-Next-Quark-MXFP4-PLEFP8" } ], "sources": [ { "file": "lmx-paiton.json", "originalPath": "p161-paiton-native-rc13/lmx/inference-matrices/EliovpAI-Qwen3.8-Flash-Next-W4A8-Paiton-CDNA4/matrix-20261030T010342Z.json", "sha256": "5ec7aac68c0a85e0eeb5a49cda85ef56f8fb8ec43eb536fa0f2520ae76e64133" }, { "file": "lmx-vllm.json", "originalPath": "a2-vllm-aiter/lmx/inference-matrices/amd-Qwen3.8-Flash-Next-Quark-MXFP4-PLEFP8/matrix-20261026T025114Z.json", "sha256": "f46e9feb093931fd495aeae2178a161826806d4b8a26a137e06b1c46359af327" }, { "file": "lmx-vllm-c3264.json", "originalPath": "a2i-c3264/lmx/inference-matrices/amd-Qwen3.8-Flash-Next-Quark-MXFP4-PLEFP8/matrix-20261029T091557Z.json", "sha256": "69d72b1186750b8a8178d06430f7afe0815d044778bdb696ac28a3ebc0bd939e" }, { "file": "lmx-sglang-mtp8.json", "originalPath": "s3-sglang-mtp3/lmx/inference-matrices/amd-Qwen3.8-Flash-Next-Quark-MXFP4-PLEFP8/matrix-20261025T212626Z.json", "sha256": "830d4d84f151de0556b6e161cb9ef1929fccf194b1d62ac8efb19b2b47eaced3" }, { "file": "lmx-sglang-mtp16.json", "originalPath": "s3b-sglang-mtp3-c16/lmx/inference-matrices/amd-Qwen3.8-Flash-Next-Quark-MXFP4-PLEFP8/matrix-20261026T034823Z.json", "sha256": "d927a7caf752529b7f0744366518bfe773ec1cdd2ac4a4db91081add121dc6a2" }, { "file": "lmx-sglang-no-mtp.json", "originalPath": "s5i-c3264/lmx/inference-matrices/amd-Qwen3.8-Flash-Next-Quark-MXFP4-PLEFP8/matrix-20261029T094637Z.json", "sha256": "ccc195fd79c281c084e2dcbdc64a917309933878772922d8f9cfbdd02677a1d4" }, { "file": "sweep-paiton.jsonl", "originalPath": "p161-paiton-native-rc13/sweep-final2/cells.jsonl", "sha256": "74702f79403a46bd6d105bdd3062823b875cff139379f967a5ee9c2706ec1bba" }, { "file": "sweep-paiton-c3264.jsonl", "originalPath": "p161-paiton-native-rc13/sweep-final2-c3264/cells.jsonl", "sha256": "76b237a3593516cdcf7df3016772c7c6fedf18401634d4942ade5c68ee7b0f44" }, { "file": "sweep-vllm.jsonl", "originalPath": "a2-vllm-aiter/sweep-final2/cells.jsonl", "sha256": "8ef249aa26237bbe584041dea21519e52b77d289eb9656d31decca0c1695dfe6" }, { "file": "sweep-vllm-c3264.jsonl", "originalPath": "a2-vllm-aiter/sweep-final2-c3264/cells.jsonl", "sha256": "bd28d65b0d6da7762b6c8602a5213ee4d5601f0318bd6a2c10b59d2bedc71c9d" }, { "file": "sweep-sglang-mtp16.jsonl", "originalPath": "s3b-sglang-mtp3-c16/sweep-final2/cells.jsonl", "sha256": "edae2e965c18190c4e10eb61e20e607892fafeaba5712843884786ba8e878c45" }, { "file": "sweep-sglang-mtp8.jsonl", "originalPath": "s3-sglang-mtp3/sweep-final2/cells.jsonl", "sha256": "9583772f470412b92eb7ea8f8b6c0d42fc53a3561201afd25114dcfdc98fd376" }, { "file": "sweep-sglang-no-mtp.jsonl", "originalPath": "s5-sglang-nomtp-c64/sweep-final2-c3264/cells.jsonl", "sha256": "0e2b5d9b9f44b201d9108e754019b742f9fca42e455c3899e46ebad464a6180d" }, { "file": "accuracy-paiton.log", "originalPath": "accuracy/paiton-a5-harness-kl.log", "sha256": "a2ce0647db44c092d13957a43c653fe69d96025d9c6bb9f43629eb870467bcff" }, { "file": "accuracy-quark.json", "originalPath": "accuracy/klq-quark-mxfp4/kl_served.json", "sha256": "b58d2e0be1a4eac1d7b8bca4e8ebf054f2af8e8f346fa9530c5b9bfa81153159" }, { "file": "accuracy-bf16.json", "originalPath": "accuracy/klbf16-ctl3-mi355/kl_served.json", "sha256": "a89f057109f4e7c4f30ae88855d16461ea934856e90d9eb11b403dff5923eeaf" }, { "file": "weight-model-card.md", "url": "https://huggingface.co/EliovpAI/Qwen3.8-Flash-Next-W4A8-Paiton-CDNA4/raw/main/README.md", "retrieved": "2026-10-30", "sha256": "a259addcb459e1689b219d389e3f9bad0a9457bece103b0b915343eac581765d" } ], "decode": { "unit": "aggregate output tokens/s", "method": "LocalMaxxing 0.1.54 / llm-inference-bench 0.7.7; empty context; 30-second cells; up to 8192 output tokens. Selected whole median-c1 run of three where available; c32/64 comparator extensions are separate runs. Null means not measured, never zero.", "rows": [ { "id": "paiton", "label": "Paiton CDNA4", "values": [ 273.071554221675, 453.37796270139864, 676.5358452552488, 839.6350758287525, 1265.8309257468627, 1886.7405904566629, 2508.6422309896725 ], "sources": [ "lmx-paiton.json" ] }, { "id": "vllm", "label": "vLLM + AITER", "values": [ 122.06160390947484, 195.3843121247914, 384.72664349139023, 602.7865944613228, 1029.0272364481477, 1544.073581515523, 2185.1747014833904 ], "sources": [ "lmx-vllm.json", "lmx-vllm-c3264.json" ] }, { "id": "sglang16", "label": "SGLang MTP · 16 seqs", "values": [ 119.60659852164058, 219.00534407363273, 403.47940340831076, 629.8803271477349, 829.3427892365852, null, null ], "sources": [ "lmx-sglang-mtp16.json" ] }, { "id": "sglang8", "label": "SGLang MTP · 8 seqs", "values": [ 118.55223501714063, 223.05100922412242, 397.0209419813495, 639.1802204621197, 634.3976960427909, null, null ], "sources": [ "lmx-sglang-mtp8.json" ] }, { "id": "sglangNoMtp", "label": "SGLang · no MTP", "values": [ null, null, null, null, null, 1088.6304487429068, 1092.9729308758801 ], "sources": [ "lmx-sglang-no-mtp.json" ] } ] }, "prefill": { "unit": "input tokens/s", "nominalTokens": [ 8192, 65536 ], "rows": [ { "id": "paiton", "source": "lmx-paiton.json", "measurements": [ { "nominalTokens": 8192, "client_tok_per_sec": 13709, "client_ttft_seconds": 0.598, "hardware_summary": {}, "method": "scout_only", "prefill_seconds": 0.598, "prompt_tokens": 8197, "samples": 1, "server_validation": { "cached_tokens": 0, "invalid_reason": "", "method": "", "prefill_seconds": 0, "prompt_tokens": 0, "request_prompt_tokens": 0, "samples": 0, "tok_per_sec": 0, "token_source": "" }, "tok_per_sec": 13709, "ttft_seconds": 0.598 }, { "nominalTokens": 65536, "client_tok_per_sec": 14286, "client_ttft_seconds": 4.489, "hardware_summary": {}, "method": "scout_only", "prefill_seconds": 4.489, "prompt_tokens": 64123, "samples": 1, "server_validation": { "cached_tokens": 0, "invalid_reason": "", "method": "", "prefill_seconds": 0, "prompt_tokens": 0, "request_prompt_tokens": 0, "samples": 0, "tok_per_sec": 0, "token_source": "" }, "tok_per_sec": 14286, "ttft_seconds": 4.489 } ] }, { "id": "vllm", "source": "lmx-vllm.json", "measurements": [ { "nominalTokens": 8192, "client_tok_per_sec": 7009, "client_ttft_seconds": 1.169, "hardware_summary": {}, "method": "scout_only", "prefill_seconds": 1.169, "prompt_tokens": 8196, "samples": 1, "server_validation": { "cached_tokens": 0, "invalid_reason": "", "method": "", "prefill_seconds": 0, "prompt_tokens": 0, "request_prompt_tokens": 0, "samples": 0, "tok_per_sec": 0, "token_source": "" }, "tok_per_sec": 7009, "ttft_seconds": 1.169 }, { "nominalTokens": 65536, "client_tok_per_sec": 7224, "client_ttft_seconds": 8.877, "hardware_summary": {}, "method": "scout_only", "prefill_seconds": 8.877, "prompt_tokens": 64122, "samples": 1, "server_validation": { "cached_tokens": 0, "invalid_reason": "", "method": "", "prefill_seconds": 0, "prompt_tokens": 0, "request_prompt_tokens": 0, "samples": 0, "tok_per_sec": 0, "token_source": "" }, "tok_per_sec": 7224, "ttft_seconds": 8.877 } ] }, { "id": "sglang16", "source": "lmx-sglang-mtp16.json", "measurements": [ { "nominalTokens": 8192, "client_tok_per_sec": 11535, "client_ttft_seconds": 0.711, "hardware_summary": {}, "method": "scout_only", "prefill_seconds": 0.711, "prompt_tokens": 8196, "samples": 1, "server_validation": { "cached_tokens": 0, "invalid_reason": "", "method": "", "prefill_seconds": 0, "prompt_tokens": 0, "request_prompt_tokens": 0, "samples": 0, "tok_per_sec": 0, "token_source": "" }, "tok_per_sec": 11535, "ttft_seconds": 0.711 }, { "nominalTokens": 65536, "client_tok_per_sec": 13827, "client_ttft_seconds": 4.638, "hardware_summary": {}, "method": "scout_only", "prefill_seconds": 4.638, "prompt_tokens": 64122, "samples": 1, "server_validation": { "cached_tokens": 0, "invalid_reason": "", "method": "", "prefill_seconds": 0, "prompt_tokens": 0, "request_prompt_tokens": 0, "samples": 0, "tok_per_sec": 0, "token_source": "" }, "tok_per_sec": 13827, "ttft_seconds": 4.638 } ] }, { "id": "sglang8", "source": "lmx-sglang-mtp8.json", "measurements": [ { "nominalTokens": 8192, "client_tok_per_sec": 12069, "client_ttft_seconds": 0.679, "hardware_summary": {}, "method": "scout_only", "prefill_seconds": 0.679, "prompt_tokens": 8195, "samples": 1, "server_validation": { "cached_tokens": 0, "invalid_reason": "", "method": "", "prefill_seconds": 0, "prompt_tokens": 0, "request_prompt_tokens": 0, "samples": 0, "tok_per_sec": 0, "token_source": "" }, "tok_per_sec": 12069, "ttft_seconds": 0.679 }, { "nominalTokens": 65536, "client_tok_per_sec": 11585, "client_ttft_seconds": 5.535, "hardware_summary": {}, "method": "scout_only", "prefill_seconds": 5.535, "prompt_tokens": 64121, "samples": 1, "server_validation": { "cached_tokens": 0, "invalid_reason": "", "method": "", "prefill_seconds": 0, "prompt_tokens": 0, "request_prompt_tokens": 0, "samples": 0, "tok_per_sec": 0, "token_source": "" }, "tok_per_sec": 11585, "ttft_seconds": 5.535 } ] }, { "id": "sglangNoMtp", "source": "lmx-sglang-no-mtp.json", "measurements": [ { "nominalTokens": 8192, "client_tok_per_sec": 12184, "client_ttft_seconds": 0.673, "hardware_summary": {}, "method": "scout_only", "prefill_seconds": 0.673, "prompt_tokens": 8197, "samples": 1, "server_validation": { "cached_tokens": 0, "invalid_reason": "", "method": "", "prefill_seconds": 0, "prompt_tokens": 0, "request_prompt_tokens": 0, "samples": 0, "tok_per_sec": 0, "token_source": "" }, "tok_per_sec": 12184, "ttft_seconds": 0.673 }, { "nominalTokens": 65536, "client_tok_per_sec": 13401, "client_ttft_seconds": 4.785, "hardware_summary": {}, "method": "scout_only", "prefill_seconds": 4.785, "prompt_tokens": 64123, "samples": 1, "server_validation": { "cached_tokens": 0, "invalid_reason": "", "method": "", "prefill_seconds": 0, "prompt_tokens": 0, "request_prompt_tokens": 0, "samples": 0, "tok_per_sec": 0, "token_source": "" }, "tok_per_sec": 13401, "ttft_seconds": 4.785 } ] } ] }, "cache": { "unit": "GPU KV tokens", "configuredContextTokens": 262144, "divisorTokens": 32768, "method": "Startup-reported cache. Dividing by 32768 is a memory-only illustration, not measured simultaneous-chat capacity. Request-state, output growth and configured running-sequence limits also apply.", "rows": [ { "id": "paiton", "source": "lmx-paiton.json", "tokens": 2190540, "memoryOnly32k": 66 }, { "id": "vllm", "source": "lmx-vllm.json", "tokens": 1795019, "memoryOnly32k": 54 }, { "id": "sglang8", "source": "lmx-sglang-mtp8.json", "tokens": 367424, "memoryOnly32k": 11 }, { "id": "sglang16", "source": "lmx-sglang-mtp16.json", "tokens": 64704, "memoryOnly32k": 1 } ] }, "serving": { "aggregateUnit": "output tokens/s per GPU", "userUnit": "tokens/s = reciprocal median TPOT after the first token; not aggregate/concurrency or end-to-end rate", "powerUnit": "mean GPU SOCKET_POWER in W; excludes host, PSU and cooling", "energyUnit": "output tokens/s per kW of mean GPU power", "sampleLimits": "2–128 requests per cell. Sample p99 is not an SLA. Up to max(8,2C) request starts, stopping new starts after 100 seconds; in-flight work finishes. Two 16-output-token warmup rounds.", "selection": "vLLM 1K/1K c32 uses earlier sweep-final2 (1169.2 output tokens/s); c64 and other c32/64 cells use later sweep-final2-c3264. SGLang MTP16 for 1K/1K; MTP8 for long inputs/outputs; no-MTP c32/64 is a different setup and is not joined to MTP.", "workloads": [ { "inputTokens": 1024, "outputTokens": 1024, "rows": [ { "id": "paiton", "cells": [ { "t_start": 1793323646.2, "t_end": 1793323670.9, "isl": 1024, "osl": 1024, "conc": 1, "n": 8, "errors": 0, "wall_s": 24.7, "short": 0, "out_tok_s": 331.4, "user_tok_s": 337.6, "gpus": 1, "power_w": 308.1, "tok_s_per_kw": 1075.7, "ttft_p50_ms": 97.8, "ttft_p99_ms": 100.6, "tpot_p50_ms": 3, "tpot_p99_ms": 3.5, "e2e_p50_ms": 3126.8, "e2e_p99_ms": 3728.2, "itl_p50_ms": 9.11, "itl_p99_ms": 9.53, "source": "sweep-paiton.jsonl" }, { "t_start": 1793323671.3, "t_end": 1793323688.3, "isl": 1024, "osl": 1024, "conc": 2, "n": 8, "errors": 0, "wall_s": 17, "short": 0, "out_tok_s": 482.2, "user_tok_s": 247.9, "gpus": 1, "power_w": 328.7, "tok_s_per_kw": 1467.1, "ttft_p50_ms": 118.7, "ttft_p99_ms": 153, "tpot_p50_ms": 4, "tpot_p99_ms": 4.7, "e2e_p50_ms": 4245.8, "e2e_p99_ms": 4919.5, "itl_p50_ms": 11.18, "itl_p99_ms": 11.88, "source": "sweep-paiton.jsonl" }, { "t_start": 1793323689.2, "t_end": 1793323700.2, "isl": 1024, "osl": 1024, "conc": 4, "n": 8, "errors": 0, "wall_s": 11, "short": 0, "out_tok_s": 746.3, "user_tok_s": 193.8, "gpus": 1, "power_w": 364.1, "tok_s_per_kw": 2049.8, "ttft_p50_ms": 150.3, "ttft_p99_ms": 331.5, "tpot_p50_ms": 5.2, "tpot_p99_ms": 5.7, "e2e_p50_ms": 5429.8, "e2e_p99_ms": 6176.9, "itl_p50_ms": 15.46, "itl_p99_ms": 16.26, "source": "sweep-paiton.jsonl" }, { "t_start": 1793323701.9, "t_end": 1793323723.6, "isl": 1024, "osl": 1024, "conc": 8, "n": 16, "errors": 0, "wall_s": 21.7, "short": 0, "out_tok_s": 754.6, "user_tok_s": 97.4, "gpus": 1, "power_w": 377.6, "tok_s_per_kw": 1998.7, "ttft_p50_ms": 335.8, "ttft_p99_ms": 668, "tpot_p50_ms": 10.3, "tpot_p99_ms": 10.5, "e2e_p50_ms": 10849.7, "e2e_p99_ms": 11167.8, "itl_p50_ms": 9.91, "itl_p99_ms": 15.89, "source": "sweep-paiton.jsonl" }, { "t_start": 1793323726.6, "t_end": 1793323758, "isl": 1024, "osl": 1024, "conc": 16, "n": 32, "errors": 0, "wall_s": 31.3, "short": 0, "out_tok_s": 1046.3, "user_tok_s": 67.6, "gpus": 1, "power_w": 406, "tok_s_per_kw": 2576.9, "ttft_p50_ms": 524.7, "ttft_p99_ms": 1299.8, "tpot_p50_ms": 14.8, "tpot_p99_ms": 15.1, "e2e_p50_ms": 15626.5, "e2e_p99_ms": 16476.6, "itl_p50_ms": 14.01, "itl_p99_ms": 19.72, "source": "sweep-paiton.jsonl" }, { "t_start": 1793325068.3, "t_end": 1793325113.7, "isl": 1024, "osl": 1024, "conc": 32, "n": 64, "errors": 0, "wall_s": 45.4, "short": 0, "out_tok_s": 1444.1, "user_tok_s": 46.4, "gpus": 1, "power_w": 437.6, "tok_s_per_kw": 3299.8, "ttft_p50_ms": 632.4, "ttft_p99_ms": 2571.6, "tpot_p50_ms": 21.6, "tpot_p99_ms": 21.9, "e2e_p50_ms": 22579.8, "e2e_p99_ms": 24647.3, "itl_p50_ms": 19.74, "itl_p99_ms": 112.26, "source": "sweep-paiton-c3264.jsonl" }, { "t_start": 1793325124.9, "t_end": 1793325198.2, "isl": 1024, "osl": 1024, "conc": 64, "n": 128, "errors": 0, "wall_s": 73.2, "short": 0, "out_tok_s": 1790.4, "user_tok_s": 28.9, "gpus": 1, "power_w": 445.2, "tok_s_per_kw": 4021.3, "ttft_p50_ms": 861, "ttft_p99_ms": 5530.5, "tpot_p50_ms": 34.6, "tpot_p99_ms": 35.6, "e2e_p50_ms": 36330.8, "e2e_p99_ms": 41110.2, "itl_p50_ms": 30.72, "itl_p99_ms": 165.91, "source": "sweep-paiton-c3264.jsonl" } ] }, { "id": "vllm", "cells": [ { "isl": 1024, "osl": 1024, "conc": 1, "n": 8, "errors": 0, "wall_s": 67.1, "short": 0, "out_tok_s": 122.1, "user_tok_s": 125.6, "power_w": 295.6, "tok_s_per_kw": 413.1, "ttft_p50_ms": 239.4, "ttft_p99_ms": 262.7, "tpot_p50_ms": 8, "tpot_p99_ms": 8, "e2e_p50_ms": 8384.4, "e2e_p99_ms": 8409.7, "itl_p50_ms": 7.96, "itl_p99_ms": 9.43, "source": "sweep-vllm.jsonl" }, { "isl": 1024, "osl": 1024, "conc": 2, "n": 8, "errors": 0, "wall_s": 42.8, "short": 0, "out_tok_s": 191.5, "user_tok_s": 99.9, "power_w": 300.3, "tok_s_per_kw": 637.6, "ttft_p50_ms": 454, "ttft_p99_ms": 456.7, "tpot_p50_ms": 10, "tpot_p99_ms": 10, "e2e_p50_ms": 10695.6, "e2e_p99_ms": 10705.8, "itl_p50_ms": 10.01, "itl_p99_ms": 11.2, "source": "sweep-vllm.jsonl" }, { "isl": 1024, "osl": 1024, "conc": 4, "n": 8, "errors": 0, "wall_s": 22.4, "short": 0, "out_tok_s": 365.4, "user_tok_s": 96.9, "power_w": 360.3, "tok_s_per_kw": 1014, "ttft_p50_ms": 653.5, "ttft_p99_ms": 663.5, "tpot_p50_ms": 10.3, "tpot_p99_ms": 10.5, "e2e_p50_ms": 11212.7, "e2e_p99_ms": 11223.1, "itl_p50_ms": 10.3, "itl_p99_ms": 12.24, "source": "sweep-vllm.jsonl" }, { "isl": 1024, "osl": 1024, "conc": 8, "n": 16, "errors": 0, "wall_s": 30.3, "short": 0, "out_tok_s": 539.9, "user_tok_s": 73, "power_w": 355.2, "tok_s_per_kw": 1520, "ttft_p50_ms": 1163.2, "ttft_p99_ms": 1173.5, "tpot_p50_ms": 13.7, "tpot_p99_ms": 14.4, "e2e_p50_ms": 15170.1, "e2e_p99_ms": 15185.1, "itl_p50_ms": 13.69, "itl_p99_ms": 15.65, "source": "sweep-vllm.jsonl" }, { "isl": 1024, "osl": 1024, "conc": 16, "n": 32, "errors": 0, "wall_s": 38.5, "short": 0, "out_tok_s": 851.5, "user_tok_s": 56.8, "power_w": 401.7, "tok_s_per_kw": 2119.7, "ttft_p50_ms": 1782.2, "ttft_p99_ms": 2240.8, "tpot_p50_ms": 17.6, "tpot_p99_ms": 18.4, "e2e_p50_ms": 19274.9, "e2e_p99_ms": 19740.2, "itl_p50_ms": 16.63, "itl_p99_ms": 19.57, "source": "sweep-vllm.jsonl" }, { "isl": 1024, "osl": 1024, "conc": 32, "n": 64, "errors": 0, "wall_s": 56.1, "short": 0, "out_tok_s": 1169.2, "user_tok_s": 39.8, "power_w": 441.9, "tok_s_per_kw": 2645.9, "ttft_p50_ms": 2299.6, "ttft_p99_ms": 4415.3, "tpot_p50_ms": 25.1, "tpot_p99_ms": 26.7, "e2e_p50_ms": 28140.2, "e2e_p99_ms": 30281.2, "itl_p50_ms": 23.13, "itl_p99_ms": 27.8, "source": "sweep-vllm.jsonl" }, { "t_start": 1793206964.7, "t_end": 1793207052.7, "isl": 1024, "osl": 1024, "conc": 64, "n": 128, "errors": 0, "wall_s": 88, "short": 0, "out_tok_s": 1490.1, "user_tok_s": 24.9, "power_w": 444.2, "tok_s_per_kw": 3354.5, "ttft_p50_ms": 2810.8, "ttft_p99_ms": 8611.9, "tpot_p50_ms": 40.1, "tpot_p99_ms": 42.3, "e2e_p50_ms": 43924, "e2e_p99_ms": 49697.3, "itl_p50_ms": 34.63, "itl_p99_ms": 40.5, "source": "sweep-vllm-c3264.jsonl" } ] }, { "id": "sglang16", "cells": [ { "isl": 1024, "osl": 1024, "conc": 1, "n": 8, "errors": 0, "wall_s": 48.2, "short": 0, "out_tok_s": 170, "user_tok_s": 172.7, "power_w": 263.5, "tok_s_per_kw": 644.9, "ttft_p50_ms": 184.4, "ttft_p99_ms": 187.5, "tpot_p50_ms": 5.8, "tpot_p99_ms": 6.6, "e2e_p50_ms": 6108, "e2e_p99_ms": 6941.3, "itl_p50_ms": 17.91, "itl_p99_ms": 18.86, "source": "sweep-sglang-mtp16.jsonl" }, { "isl": 1024, "osl": 1024, "conc": 2, "n": 8, "errors": 0, "wall_s": 29.6, "short": 0, "out_tok_s": 277.2, "user_tok_s": 143.1, "power_w": 289.9, "tok_s_per_kw": 956.3, "ttft_p50_ms": 199.9, "ttft_p99_ms": 208.6, "tpot_p50_ms": 7, "tpot_p99_ms": 7.8, "e2e_p50_ms": 7347.3, "e2e_p99_ms": 8157.8, "itl_p50_ms": 19.71, "itl_p99_ms": 21.56, "source": "sweep-sglang-mtp16.jsonl" }, { "isl": 1024, "osl": 1024, "conc": 4, "n": 8, "errors": 0, "wall_s": 17.1, "short": 0, "out_tok_s": 479.4, "user_tok_s": 124.3, "power_w": 322.1, "tok_s_per_kw": 1488.4, "ttft_p50_ms": 275.6, "ttft_p99_ms": 276, "tpot_p50_ms": 8, "tpot_p99_ms": 8.6, "e2e_p50_ms": 8503.2, "e2e_p99_ms": 8980.1, "itl_p50_ms": 22.46, "itl_p99_ms": 24.17, "source": "sweep-sglang-mtp16.jsonl" }, { "isl": 1024, "osl": 1024, "conc": 8, "n": 16, "errors": 0, "wall_s": 22.3, "short": 0, "out_tok_s": 736, "user_tok_s": 111.3, "power_w": 348.4, "tok_s_per_kw": 2112.3, "ttft_p50_ms": 564.4, "ttft_p99_ms": 580, "tpot_p50_ms": 9, "tpot_p99_ms": 12.4, "e2e_p50_ms": 9417, "e2e_p99_ms": 13229.8, "itl_p50_ms": 29.01, "itl_p99_ms": 176.47, "source": "sweep-sglang-mtp16.jsonl" }, { "isl": 1024, "osl": 1024, "conc": 16, "n": 32, "errors": 0, "wall_s": 31.3, "short": 0, "out_tok_s": 1047, "user_tok_s": 76.8, "power_w": 384.9, "tok_s_per_kw": 2720.2, "ttft_p50_ms": 613.9, "ttft_p99_ms": 1028.9, "tpot_p50_ms": 13, "tpot_p99_ms": 17.6, "e2e_p50_ms": 13848.2, "e2e_p99_ms": 18993.6, "itl_p50_ms": 37.94, "itl_p99_ms": 196.47, "source": "sweep-sglang-mtp16.jsonl" }, null, null ] }, { "id": "sglangNoMtp", "cells": [ null, null, null, null, null, { "t_start": 1793208791.1, "t_end": 1793208863.9, "isl": 1024, "osl": 1024, "conc": 32, "n": 64, "errors": 0, "wall_s": 72.8, "short": 0, "out_tok_s": 900.7, "user_tok_s": 41.8, "power_w": 359.1, "tok_s_per_kw": 2507.9, "ttft_p50_ms": 1415.9, "ttft_p99_ms": 26763.2, "tpot_p50_ms": 23.9, "tpot_p99_ms": 24.4, "e2e_p50_ms": 25946.7, "e2e_p99_ms": 51694.3, "itl_p50_ms": 23.7, "itl_p99_ms": 25.08, "source": "sweep-sglang-no-mtp.jsonl" }, { "t_start": 1793208873.5, "t_end": 1793209018.5, "isl": 1024, "osl": 1024, "conc": 64, "n": 128, "errors": 0, "wall_s": 145, "short": 0, "out_tok_s": 904.1, "user_tok_s": 41.5, "power_w": 362.6, "tok_s_per_kw": 2493.3, "ttft_p50_ms": 27371.5, "ttft_p99_ms": 53244, "tpot_p50_ms": 24.1, "tpot_p99_ms": 24.6, "e2e_p50_ms": 51941.5, "e2e_p99_ms": 78016.4, "itl_p50_ms": 23.9, "itl_p99_ms": 25.86, "source": "sweep-sglang-no-mtp.jsonl" } ] } ] }, { "inputTokens": 1024, "outputTokens": 8192, "rows": [ { "id": "paiton", "cells": [ { "t_start": 1793323943.1, "t_end": 1793324056.9, "isl": 1024, "osl": 8192, "conc": 1, "n": 5, "errors": 0, "wall_s": 113.8, "short": 0, "out_tok_s": 359.9, "user_tok_s": 364.3, "gpus": 1, "power_w": 324, "tok_s_per_kw": 1110.6, "ttft_p50_ms": 99.3, "ttft_p99_ms": 103.5, "tpot_p50_ms": 2.7, "tpot_p99_ms": 3.1, "e2e_p50_ms": 22584.7, "e2e_p99_ms": 25212, "itl_p50_ms": 9.38, "itl_p99_ms": 9.72, "source": "sweep-paiton.jsonl" }, { "t_start": 1793324057.4, "t_end": 1793324176.7, "isl": 1024, "osl": 8192, "conc": 2, "n": 8, "errors": 0, "wall_s": 119.3, "short": 0, "out_tok_s": 549.1, "user_tok_s": 277.8, "gpus": 1, "power_w": 337, "tok_s_per_kw": 1629.6, "ttft_p50_ms": 119.4, "ttft_p99_ms": 155.4, "tpot_p50_ms": 3.6, "tpot_p99_ms": 3.9, "e2e_p50_ms": 29600.7, "e2e_p99_ms": 31806.8, "itl_p50_ms": 11.6, "itl_p99_ms": 12.01, "source": "sweep-paiton.jsonl" }, { "t_start": 1793324177.7, "t_end": 1793324265.5, "isl": 1024, "osl": 8192, "conc": 4, "n": 8, "errors": 0, "wall_s": 87.9, "short": 0, "out_tok_s": 746, "user_tok_s": 198, "gpus": 1, "power_w": 364.1, "tok_s_per_kw": 2049, "ttft_p50_ms": 136.7, "ttft_p99_ms": 351.7, "tpot_p50_ms": 5, "tpot_p99_ms": 5.4, "e2e_p50_ms": 41498.4, "e2e_p99_ms": 44800.4, "itl_p50_ms": 16.03, "itl_p99_ms": 16.66, "source": "sweep-paiton.jsonl" }, { "t_start": 1793324267.2, "t_end": 1793324434.3, "isl": 1024, "osl": 8192, "conc": 8, "n": 16, "errors": 0, "wall_s": 167, "short": 0, "out_tok_s": 784.7, "user_tok_s": 98.6, "gpus": 1, "power_w": 384, "tok_s_per_kw": 2043.7, "ttft_p50_ms": 427.4, "ttft_p99_ms": 655.2, "tpot_p50_ms": 10.1, "tpot_p99_ms": 10.2, "e2e_p50_ms": 83501.1, "e2e_p99_ms": 83650.5, "itl_p50_ms": 10.11, "itl_p99_ms": 10.62, "source": "sweep-paiton.jsonl" }, { "t_start": 1793324437.3, "t_end": 1793324556.7, "isl": 1024, "osl": 8192, "conc": 16, "n": 16, "errors": 0, "wall_s": 119.3, "short": 0, "out_tok_s": 1098.8, "user_tok_s": 69.1, "gpus": 1, "power_w": 412.6, "tok_s_per_kw": 2663.2, "ttft_p50_ms": 898.4, "ttft_p99_ms": 1323.3, "tpot_p50_ms": 14.5, "tpot_p99_ms": 14.5, "e2e_p50_ms": 119246.4, "e2e_p99_ms": 119281.3, "itl_p50_ms": 14.4, "itl_p99_ms": 15.77, "source": "sweep-paiton.jsonl" }, { "t_start": 1793325547.1, "t_end": 1793325719.6, "isl": 1024, "osl": 8192, "conc": 32, "n": 32, "errors": 0, "wall_s": 172.4, "short": 0, "out_tok_s": 1520.6, "user_tok_s": 47.9, "gpus": 1, "power_w": 445.3, "tok_s_per_kw": 3414.5, "ttft_p50_ms": 1563.6, "ttft_p99_ms": 2710.6, "tpot_p50_ms": 20.9, "tpot_p99_ms": 21, "e2e_p50_ms": 172301.1, "e2e_p99_ms": 172379.6, "itl_p50_ms": 20.7, "itl_p99_ms": 41.11, "source": "sweep-paiton-c3264.jsonl" }, { "t_start": 1793325731.2, "t_end": 1793326002, "isl": 1024, "osl": 8192, "conc": 64, "n": 64, "errors": 0, "wall_s": 270.6, "short": 0, "out_tok_s": 1937.3, "user_tok_s": 30.6, "gpus": 1, "power_w": 447.9, "tok_s_per_kw": 4325.7, "ttft_p50_ms": 2879.3, "ttft_p99_ms": 5674.1, "tpot_p50_ms": 32.7, "tpot_p99_ms": 32.9, "e2e_p50_ms": 270354.6, "e2e_p99_ms": 270597.5, "itl_p50_ms": 32.47, "itl_p99_ms": 39.97, "source": "sweep-paiton-c3264.jsonl" } ] }, { "id": "vllm", "cells": [ { "isl": 1024, "osl": 8192, "conc": 1, "n": 2, "errors": 0, "wall_s": 134.2, "short": 0, "out_tok_s": 122.1, "user_tok_s": 122.6, "power_w": 302.4, "tok_s_per_kw": 403.6, "ttft_p50_ms": 232.9, "ttft_p99_ms": 242.6, "tpot_p50_ms": 8.2, "tpot_p99_ms": 8.2, "e2e_p50_ms": 67049.2, "e2e_p99_ms": 67181.8, "itl_p50_ms": 8.07, "itl_p99_ms": 12.74, "source": "sweep-vllm.jsonl" }, { "isl": 1024, "osl": 8192, "conc": 2, "n": 4, "errors": 0, "wall_s": 167.1, "short": 0, "out_tok_s": 196.1, "user_tok_s": 98.4, "power_w": 302.5, "tok_s_per_kw": 648.3, "ttft_p50_ms": 438.8, "ttft_p99_ms": 441.1, "tpot_p50_ms": 10.2, "tpot_p99_ms": 10.2, "e2e_p50_ms": 83665.5, "e2e_p99_ms": 83667.1, "itl_p50_ms": 10.12, "itl_p99_ms": 11.93, "source": "sweep-vllm.jsonl" }, { "isl": 1024, "osl": 8192, "conc": 4, "n": 4, "errors": 0, "wall_s": 142, "short": 0, "out_tok_s": 230.7, "user_tok_s": 57.9, "power_w": 257.1, "tok_s_per_kw": 897.2, "ttft_p50_ms": 655.9, "ttft_p99_ms": 656.2, "tpot_p50_ms": 17.3, "tpot_p99_ms": 17.3, "e2e_p50_ms": 142025.7, "e2e_p99_ms": 142026.2, "itl_p50_ms": 10.42, "itl_p99_ms": 11.06, "source": "sweep-vllm.jsonl" }, { "isl": 1024, "osl": 8192, "conc": 8, "n": 8, "errors": 0, "wall_s": 114.5, "short": 0, "out_tok_s": 572.5, "user_tok_s": 72.3, "power_w": 349.3, "tok_s_per_kw": 1638.7, "ttft_p50_ms": 1156.5, "ttft_p99_ms": 1157.5, "tpot_p50_ms": 13.8, "tpot_p99_ms": 13.9, "e2e_p50_ms": 114474.6, "e2e_p99_ms": 114476.6, "itl_p50_ms": 13.79, "itl_p99_ms": 16.27, "source": "sweep-vllm.jsonl" }, { "isl": 1024, "osl": 8192, "conc": 16, "n": 16, "errors": 0, "wall_s": 141.1, "short": 0, "out_tok_s": 929.1, "user_tok_s": 58.6, "power_w": 397.8, "tok_s_per_kw": 2335.9, "ttft_p50_ms": 2088.8, "ttft_p99_ms": 2236.4, "tpot_p50_ms": 17.1, "tpot_p99_ms": 17.2, "e2e_p50_ms": 141051.7, "e2e_p99_ms": 141058.6, "itl_p50_ms": 16.93, "itl_p99_ms": 21.11, "source": "sweep-vllm.jsonl" }, { "t_start": 1793207496.2, "t_end": 1793207694.8, "isl": 1024, "osl": 8192, "conc": 32, "n": 32, "errors": 0, "wall_s": 198.5, "short": 0, "out_tok_s": 1320.6, "user_tok_s": 41.9, "power_w": 447.7, "tok_s_per_kw": 2949.7, "ttft_p50_ms": 3067.5, "ttft_p99_ms": 4436.4, "tpot_p50_ms": 23.9, "tpot_p99_ms": 24.2, "e2e_p50_ms": 198471.9, "e2e_p99_ms": 198491.2, "itl_p50_ms": 23.68, "itl_p99_ms": 25.51, "source": "sweep-vllm-c3264.jsonl" }, { "t_start": 1793207713.4, "t_end": 1793208015.5, "isl": 1024, "osl": 8192, "conc": 64, "n": 64, "errors": 0, "wall_s": 301.8, "short": 0, "out_tok_s": 1737.2, "user_tok_s": 27.6, "power_w": 448.7, "tok_s_per_kw": 3872, "ttft_p50_ms": 4985, "ttft_p99_ms": 8773.3, "tpot_p50_ms": 36.2, "tpot_p99_ms": 36.7, "e2e_p50_ms": 301704.5, "e2e_p99_ms": 301769.5, "itl_p50_ms": 35.87, "itl_p99_ms": 38.33, "source": "sweep-vllm-c3264.jsonl" } ] }, { "id": "sglang8", "cells": [ { "isl": 1024, "osl": 8192, "conc": 1, "n": 3, "errors": 0, "wall_s": 132.1, "short": 0, "out_tok_s": 186, "user_tok_s": 190.8, "power_w": 263.7, "tok_s_per_kw": 705.4, "ttft_p50_ms": 186.9, "ttft_p99_ms": 188, "tpot_p50_ms": 5.2, "tpot_p99_ms": 5.7, "e2e_p50_ms": 43110.4, "e2e_p99_ms": 47266.6, "itl_p50_ms": 18.38, "itl_p99_ms": 19.13, "source": "sweep-sglang-mtp8.jsonl" }, { "isl": 1024, "osl": 8192, "conc": 2, "n": 5, "errors": 0, "wall_s": 146.7, "short": 0, "out_tok_s": 279.2, "user_tok_s": 167.3, "power_w": 279.5, "tok_s_per_kw": 998.9, "ttft_p50_ms": 210.4, "ttft_p99_ms": 332.2, "tpot_p50_ms": 6, "tpot_p99_ms": 6.8, "e2e_p50_ms": 49174.1, "e2e_p99_ms": 55530, "itl_p50_ms": 20.26, "itl_p99_ms": 21.13, "source": "sweep-sglang-mtp8.jsonl" }, { "isl": 1024, "osl": 8192, "conc": 4, "n": 8, "errors": 0, "wall_s": 137.7, "short": 0, "out_tok_s": 476, "user_tok_s": 137.5, "power_w": 311.8, "tok_s_per_kw": 1526.4, "ttft_p50_ms": 276.8, "ttft_p99_ms": 277.4, "tpot_p50_ms": 7.3, "tpot_p99_ms": 9.5, "e2e_p50_ms": 59766.2, "e2e_p99_ms": 77919, "itl_p50_ms": 23.2, "itl_p99_ms": 24.44, "source": "sweep-sglang-mtp8.jsonl" }, { "isl": 1024, "osl": 8192, "conc": 8, "n": 16, "errors": 0, "wall_s": 155.1, "short": 0, "out_tok_s": 845.3, "user_tok_s": 117.8, "power_w": 359.9, "tok_s_per_kw": 2348.8, "ttft_p50_ms": 567.8, "ttft_p99_ms": 582.7, "tpot_p50_ms": 8.5, "tpot_p99_ms": 10.8, "e2e_p50_ms": 70136.2, "e2e_p99_ms": 89021.2, "itl_p50_ms": 27.87, "itl_p99_ms": 30.55, "source": "sweep-sglang-mtp8.jsonl" }, { "isl": 1024, "osl": 8192, "conc": 16, "n": 24, "errors": 0, "wall_s": 226.5, "short": 0, "out_tok_s": 868, "user_tok_s": 120.2, "power_w": 373.5, "tok_s_per_kw": 2324.4, "ttft_p50_ms": 64352.3, "ttft_p99_ms": 87906.1, "tpot_p50_ms": 8.3, "tpot_p99_ms": 11.2, "e2e_p50_ms": 133585, "e2e_p99_ms": 172005.4, "itl_p50_ms": 27.8, "itl_p99_ms": 30.44, "source": "sweep-sglang-mtp8.jsonl" }, null, null ] }, { "id": "sglangNoMtp", "cells": [ null, null, null, null, null, { "t_start": 1793209403.5, "t_end": 1793209746.4, "isl": 1024, "osl": 8192, "conc": 32, "n": 32, "errors": 0, "wall_s": 342.7, "short": 0, "out_tok_s": 764.9, "user_tok_s": 40.4, "power_w": 348.6, "tok_s_per_kw": 2194.4, "ttft_p50_ms": 1486.2, "ttft_p99_ms": 204686.1, "tpot_p50_ms": 24.8, "tpot_p99_ms": 24.9, "e2e_p50_ms": 204335.5, "e2e_p99_ms": 342707.7, "itl_p50_ms": 24.73, "itl_p99_ms": 26.11, "source": "sweep-sglang-no-mtp.jsonl" }, { "t_start": 1793209756, "t_end": 1793210330.8, "isl": 1024, "osl": 8192, "conc": 64, "n": 64, "errors": 0, "wall_s": 574.5, "short": 0, "out_tok_s": 912.5, "user_tok_s": 40.3, "power_w": 368.7, "tok_s_per_kw": 2474.7, "ttft_p50_ms": 205568.6, "ttft_p99_ms": 410269.5, "tpot_p50_ms": 24.8, "tpot_p99_ms": 24.9, "e2e_p50_ms": 409523.2, "e2e_p99_ms": 574520.2, "itl_p50_ms": 24.78, "itl_p99_ms": 26.34, "source": "sweep-sglang-no-mtp.jsonl" } ] } ] }, { "inputTokens": 8192, "outputTokens": 1024, "rows": [ { "id": "paiton", "cells": [ { "t_start": 1793323759.2, "t_end": 1793323785.2, "isl": 8192, "osl": 1024, "conc": 1, "n": 8, "errors": 0, "wall_s": 26, "short": 0, "out_tok_s": 315, "user_tok_s": 390.8, "gpus": 1, "power_w": 332.2, "tok_s_per_kw": 948.2, "ttft_p50_ms": 535.3, "ttft_p99_ms": 537.4, "tpot_p50_ms": 2.6, "tpot_p99_ms": 3.2, "e2e_p50_ms": 3152.8, "e2e_p99_ms": 3798.6, "itl_p50_ms": 9.46, "itl_p99_ms": 9.89, "source": "sweep-paiton.jsonl" }, { "t_start": 1793323787.7, "t_end": 1793323807.6, "isl": 8192, "osl": 1024, "conc": 2, "n": 8, "errors": 0, "wall_s": 19.9, "short": 0, "out_tok_s": 411.9, "user_tok_s": 232.6, "gpus": 1, "power_w": 367.1, "tok_s_per_kw": 1122.1, "ttft_p50_ms": 707, "ttft_p99_ms": 1220.9, "tpot_p50_ms": 4.3, "tpot_p99_ms": 4.7, "e2e_p50_ms": 5104.4, "e2e_p99_ms": 5684.5, "itl_p50_ms": 11.69, "itl_p99_ms": 153, "source": "sweep-paiton.jsonl" }, { "t_start": 1793323812.8, "t_end": 1793323828.7, "isl": 8192, "osl": 1024, "conc": 4, "n": 8, "errors": 0, "wall_s": 15.8, "short": 0, "out_tok_s": 517.9, "user_tok_s": 151.8, "gpus": 1, "power_w": 388.5, "tok_s_per_kw": 1332.8, "ttft_p50_ms": 1251.4, "ttft_p99_ms": 2605.4, "tpot_p50_ms": 6.6, "tpot_p99_ms": 8.1, "e2e_p50_ms": 7991.7, "e2e_p99_ms": 10276, "itl_p50_ms": 15.99, "itl_p99_ms": 173.46, "source": "sweep-paiton.jsonl" }, { "t_start": 1793323839.3, "t_end": 1793323870.6, "isl": 8192, "osl": 1024, "conc": 8, "n": 16, "errors": 0, "wall_s": 31.2, "short": 0, "out_tok_s": 524.7, "user_tok_s": 71.3, "gpus": 1, "power_w": 401.7, "tok_s_per_kw": 1306.4, "ttft_p50_ms": 1325.2, "ttft_p99_ms": 5361.7, "tpot_p50_ms": 14, "tpot_p99_ms": 14.5, "e2e_p50_ms": 15601.3, "e2e_p99_ms": 19782, "itl_p50_ms": 10.12, "itl_p99_ms": 166.93, "source": "sweep-paiton.jsonl" }, { "t_start": 1793323892.1, "t_end": 1793323942.8, "isl": 8192, "osl": 1024, "conc": 16, "n": 32, "errors": 0, "wall_s": 50.7, "short": 0, "out_tok_s": 646.4, "user_tok_s": 43, "gpus": 1, "power_w": 425.9, "tok_s_per_kw": 1517.7, "ttft_p50_ms": 1355.1, "ttft_p99_ms": 10867.9, "tpot_p50_ms": 23.3, "tpot_p99_ms": 23.9, "e2e_p50_ms": 25186.7, "e2e_p99_ms": 34961.3, "itl_p50_ms": 14.48, "itl_p99_ms": 172.94, "source": "sweep-paiton.jsonl" }, { "t_start": 1793325241.8, "t_end": 1793325327.3, "isl": 8192, "osl": 1024, "conc": 32, "n": 64, "errors": 0, "wall_s": 85.5, "short": 0, "out_tok_s": 766.1, "user_tok_s": 25.3, "gpus": 1, "power_w": 445.1, "tok_s_per_kw": 1721.1, "ttft_p50_ms": 1755.3, "ttft_p99_ms": 21862.3, "tpot_p50_ms": 39.5, "tpot_p99_ms": 40.1, "e2e_p50_ms": 41864.2, "e2e_p99_ms": 62713, "itl_p50_ms": 20.67, "itl_p99_ms": 180.23, "source": "sweep-paiton-c3264.jsonl" }, { "t_start": 1793325414.7, "t_end": 1793325541.2, "isl": 8192, "osl": 1024, "conc": 64, "n": 101, "errors": 0, "wall_s": 126.4, "short": 0, "out_tok_s": 818.5, "user_tok_s": 15.4, "gpus": 1, "power_w": 444.8, "tok_s_per_kw": 1840.2, "ttft_p50_ms": 9611.7, "ttft_p99_ms": 47099.7, "tpot_p50_ms": 65.1, "tpot_p99_ms": 71.2, "e2e_p50_ms": 81770.2, "e2e_p99_ms": 105400.9, "itl_p50_ms": 31.98, "itl_p99_ms": 185.38, "source": "sweep-paiton-c3264.jsonl" } ] }, { "id": "vllm", "cells": [ { "isl": 8192, "osl": 1024, "conc": 1, "n": 8, "errors": 0, "wall_s": 75.3, "short": 0, "out_tok_s": 108.8, "user_tok_s": 123.5, "power_w": 322.1, "tok_s_per_kw": 337.9, "ttft_p50_ms": 1133.1, "ttft_p99_ms": 1148.1, "tpot_p50_ms": 8.1, "tpot_p99_ms": 8.1, "e2e_p50_ms": 9411.6, "e2e_p99_ms": 9422.7, "itl_p50_ms": 8.08, "itl_p99_ms": 10.35, "source": "sweep-vllm.jsonl" }, { "isl": 8192, "osl": 1024, "conc": 2, "n": 8, "errors": 0, "wall_s": 50.3, "short": 0, "out_tok_s": 163, "user_tok_s": 98.1, "power_w": 321, "tok_s_per_kw": 507.6, "ttft_p50_ms": 2170.8, "ttft_p99_ms": 2181.5, "tpot_p50_ms": 10.2, "tpot_p99_ms": 10.2, "e2e_p50_ms": 12579, "e2e_p99_ms": 12583.9, "itl_p50_ms": 10.16, "itl_p99_ms": 12.63, "source": "sweep-vllm.jsonl" }, { "isl": 8192, "osl": 1024, "conc": 4, "n": 8, "errors": 0, "wall_s": 30, "short": 0, "out_tok_s": 272.7, "user_tok_s": 85.1, "power_w": 383.8, "tok_s_per_kw": 710.6, "ttft_p50_ms": 3294, "ttft_p99_ms": 4340.8, "tpot_p50_ms": 11.7, "tpot_p99_ms": 12.6, "e2e_p50_ms": 15011.1, "e2e_p99_ms": 16060.4, "itl_p50_ms": 10.46, "itl_p99_ms": 12.45, "source": "sweep-vllm.jsonl" }, { "isl": 8192, "osl": 1024, "conc": 8, "n": 16, "errors": 0, "wall_s": 45.7, "short": 0, "out_tok_s": 358.8, "user_tok_s": 55.6, "power_w": 385.4, "tok_s_per_kw": 931.1, "ttft_p50_ms": 4482, "ttft_p99_ms": 8707.5, "tpot_p50_ms": 18, "tpot_p99_ms": 20.4, "e2e_p50_ms": 22778.9, "e2e_p99_ms": 28554.3, "itl_p50_ms": 13.86, "itl_p99_ms": 18.53, "source": "sweep-vllm.jsonl" }, { "isl": 8192, "osl": 1024, "conc": 16, "n": 32, "errors": 0, "wall_s": 69.2, "short": 0, "out_tok_s": 473.8, "user_tok_s": 34.2, "power_w": 421.7, "tok_s_per_kw": 1123.5, "ttft_p50_ms": 4563.2, "ttft_p99_ms": 17418.6, "tpot_p50_ms": 29.2, "tpot_p99_ms": 31.8, "e2e_p50_ms": 34454.3, "e2e_p99_ms": 48348.4, "itl_p50_ms": 16.92, "itl_p99_ms": 789.12, "source": "sweep-vllm.jsonl" }, { "t_start": 1793207123.1, "t_end": 1793207240.6, "isl": 8192, "osl": 1024, "conc": 32, "n": 64, "errors": 0, "wall_s": 117.4, "short": 0, "out_tok_s": 558.2, "user_tok_s": 19, "power_w": 439.3, "tok_s_per_kw": 1270.6, "ttft_p50_ms": 4590, "ttft_p99_ms": 34920.6, "tpot_p50_ms": 52.7, "tpot_p99_ms": 54.9, "e2e_p50_ms": 58389.9, "e2e_p99_ms": 88796.2, "itl_p50_ms": 23.58, "itl_p99_ms": 889.63, "source": "sweep-vllm-c3264.jsonl" }, { "t_start": 1793207380.7, "t_end": 1793207486.6, "isl": 8192, "osl": 1024, "conc": 64, "n": 64, "errors": 0, "wall_s": 105.8, "short": 0, "out_tok_s": 619.4, "user_tok_s": 14.8, "power_w": 438.8, "tok_s_per_kw": 1411.5, "ttft_p50_ms": 37633.7, "ttft_p99_ms": 70120.9, "tpot_p50_ms": 67.8, "tpot_p99_ms": 98.6, "e2e_p50_ms": 105084.2, "e2e_p99_ms": 105727.8, "itl_p50_ms": 35.52, "itl_p99_ms": 902.74, "source": "sweep-vllm-c3264.jsonl" } ] }, { "id": "sglang8", "cells": [ { "isl": 8192, "osl": 1024, "conc": 1, "n": 8, "errors": 0, "wall_s": 48, "short": 0, "out_tok_s": 170.5, "user_tok_s": 198.9, "power_w": 279.1, "tok_s_per_kw": 611.1, "ttft_p50_ms": 515.1, "ttft_p99_ms": 520, "tpot_p50_ms": 5, "tpot_p99_ms": 7.8, "e2e_p50_ms": 5659.6, "e2e_p99_ms": 8507.7, "itl_p50_ms": 18.39, "itl_p99_ms": 19.43, "source": "sweep-sglang-mtp8.jsonl" }, { "isl": 8192, "osl": 1024, "conc": 2, "n": 8, "errors": 0, "wall_s": 33, "short": 0, "out_tok_s": 248.5, "user_tok_s": 123.2, "power_w": 298.7, "tok_s_per_kw": 831.9, "ttft_p50_ms": 531.5, "ttft_p99_ms": 1005.8, "tpot_p50_ms": 8.1, "tpot_p99_ms": 9.5, "e2e_p50_ms": 9170.7, "e2e_p99_ms": 10260.4, "itl_p50_ms": 20.32, "itl_p99_ms": 22.74, "source": "sweep-sglang-mtp8.jsonl" }, { "isl": 8192, "osl": 1024, "conc": 4, "n": 8, "errors": 0, "wall_s": 21.3, "short": 0, "out_tok_s": 383.7, "user_tok_s": 117.3, "power_w": 328.3, "tok_s_per_kw": 1168.9, "ttft_p50_ms": 1025.2, "ttft_p99_ms": 2000.7, "tpot_p50_ms": 8.5, "tpot_p99_ms": 11.2, "e2e_p50_ms": 9343.3, "e2e_p99_ms": 12137.8, "itl_p50_ms": 23.3, "itl_p99_ms": 29.4, "source": "sweep-sglang-mtp8.jsonl" }, { "isl": 8192, "osl": 1024, "conc": 8, "n": 16, "errors": 0, "wall_s": 29.6, "short": 0, "out_tok_s": 554.4, "user_tok_s": 83.5, "power_w": 373.5, "tok_s_per_kw": 1484.3, "ttft_p50_ms": 1007.6, "ttft_p99_ms": 4001.5, "tpot_p50_ms": 12, "tpot_p99_ms": 18.8, "e2e_p50_ms": 13829.4, "e2e_p99_ms": 20238.2, "itl_p50_ms": 27.86, "itl_p99_ms": 516.91, "source": "sweep-sglang-mtp8.jsonl" }, { "isl": 8192, "osl": 1024, "conc": 16, "n": 32, "errors": 0, "wall_s": 56.9, "short": 0, "out_tok_s": 575.5, "user_tok_s": 79.2, "power_w": 378.6, "tok_s_per_kw": 1519.9, "ttft_p50_ms": 14067, "ttft_p99_ms": 20375.1, "tpot_p50_ms": 12.6, "tpot_p99_ms": 16.7, "e2e_p50_ms": 27521.7, "e2e_p99_ms": 33309.5, "itl_p50_ms": 27.86, "itl_p99_ms": 516.16, "source": "sweep-sglang-mtp8.jsonl" }, null, null ] }, { "id": "sglangNoMtp", "cells": [ null, null, null, null, null, { "t_start": 1793209050.5, "t_end": 1793209152.6, "isl": 8192, "osl": 1024, "conc": 32, "n": 64, "errors": 0, "wall_s": 102.1, "short": 0, "out_tok_s": 641.7, "user_tok_s": 35.2, "power_w": 392.6, "tok_s_per_kw": 1634.3, "ttft_p50_ms": 8586.4, "ttft_p99_ms": 40964.2, "tpot_p50_ms": 28.4, "tpot_p99_ms": 35.4, "e2e_p50_ms": 37495.6, "e2e_p99_ms": 74647.9, "itl_p50_ms": 24.67, "itl_p99_ms": 26.21, "source": "sweep-sglang-no-mtp.jsonl" }, { "t_start": 1793209218, "t_end": 1793209398.5, "isl": 8192, "osl": 1024, "conc": 64, "n": 114, "errors": 0, "wall_s": 180.4, "short": 0, "out_tok_s": 647.1, "user_tok_s": 33.8, "power_w": 388.9, "tok_s_per_kw": 1664.1, "ttft_p50_ms": 49352.2, "ttft_p99_ms": 84806.2, "tpot_p50_ms": 29.6, "tpot_p99_ms": 37.2, "e2e_p50_ms": 78152.7, "e2e_p99_ms": 115527.1, "itl_p50_ms": 24.76, "itl_p99_ms": 26.44, "source": "sweep-sglang-no-mtp.jsonl" } ] } ] } ] }, "accuracy": { "positions": 130, "method": "Two long rows (r24/r25), 65 teacher-forced positions each. Top-1024-plus-tail KL against the BF16 reference. Short rows excluded after BF16 control inconsistency. Paiton measured through its harness; Quark through vLLM+AITER. Not task accuracy or the separate full-vocabulary weight-emulation KL.", "rows": [ { "id": "bf16", "hardware": "MI355X", "kl": 0.007694863545343617, "top1": 0.9538461538461538, "rows": [ "r24", "r25" ], "source": "accuracy-bf16.json" }, { "id": "paiton", "hardware": "MI350P", "kl": 0.0219415, "top1": 0.9307695, "rows": [ "r24", "r25" ], "source": "accuracy-paiton.log" }, { "id": "quark", "hardware": "MI350P", "kl": 0.11406719017357203, "top1": 0.8076923076923077, "rows": [ "r24", "r25" ], "source": "accuracy-quark.json" } ] }, "weightAccuracy": { "source": "weight-model-card.md", "heldOutTokens": 294912, "method": "Full-vocabulary KL from the original BF16 next-token distribution, averaged over text and assistant positions of the held-out set. Emulation of served arithmetic; free routing. Top-1 is next-token agreement, not correct answers. Separate from the 130-position served check. No significant superiority over INT4 is established; no INT4 confidence interval is reported.", "rows": [ { "id": "a5", "label": "Paiton A5 · CDNA4 FP4", "kl": 0.0265, "kl95CI": [ 0.0239, 0.0292 ], "routingLockedKl": 0.0131, "top1Percent": 93.78 }, { "id": "int4", "label": "INT4-g128 · RDNA4 recipe", "kl": 0.0288, "kl95CI": null, "routingLockedKl": 0.0135, "top1Percent": 93.51 } ], "retrieval": [ { "matched": 8, "total": 8, "contextLabel": "131K" }, { "matched": 4, "total": 4, "contextLabel": "258K" } ], "limits": "Teacher-forced long-context tests, not serving-load validation. Task benchmarks, multilingual quality and tool-calling quality were not measured for this release. Public weights require the proprietary CDNA4 runtime; not an all-hardware validation." } }