LLM Decoding on LLM Decoding
40,000cuBLASLt ThroughputcuBLASLt
Evaluation results
| Method | Links | |||||
|---|---|---|---|---|---|---|
| cuBLASLtGPU=B200, Model=Llama3.2-1B, Concurrency=256, Precision=INT82026.03 | 40,000 | — | — | — | — | |
| cuBLASLtGPU=B200, Model=Llama3.2-1B, Concurrency=512, Precision=INT82026.03 | 33,300 | — | — | — | — | |
| cuBLASLtGPU=B200, Model=Llama3.2-3B, Concurrency=512, Precision=INT82026.03 | 30,400 | — | — | — | — | |
| cuBLASLtGPU=H100, Model=Llama3.2-1B, Concurrency=512, Precision=INT82026.03 | 29,900 | — | — | — | — | |
| cuBLASLtGPU=H100, Model=Llama3.2-1B, Concurrency=256, Precision=INT82026.03 | 29,100 | — | — | — | — | |
| cuBLASLtGPU=B200, Model=BitNet-2B, Concurrency=256, Precision=INT82026.03 | 26,500 | — | — | — | — | |
| cuBLASLtGPU=B200, Model=Qwen2.5-7B, Concurrency=512, Precision=INT82026.03 | 26,500 | — | — | — | — | |
| cuBLASLtGPU=B200, Model=BitNet-2B, Concurrency=512, Precision=INT82026.03 | 26,200 | — | — | — | — | |
| cuBLASLtGPU=B200, Model=Llama3.2-3B, Concurrency=256, Precision=INT82026.03 | 25,900 | — | — | — | — | |
| cuBLASLtGPU=B200, Model=Llama3.2-1B, Concurrency=128, Precision=INT82026.03 | 25,400 | — | — | — | — | |
| cuBLASLtGPU=B200, Model=Qwen2.5-7B, Concurrency=256, Precision=INT82026.03 | 24,700 | — | — | — | — | |
| cuBLASLtGPU=A100, Model=Llama3.2-1B, Concurrency=256, Precision=INT82026.03 | 23,000 | — | — | — | — | |
| cuBLASLtGPU=A100, Model=Llama3.2-1B, Concurrency=512, Precision=INT82026.03 | 22,900 | — | — | — | — | |
| cuBLASLtGPU=H100, Model=Llama3.2-1B, Concurrency=128, Precision=INT82026.03 | 21,300 | — | — | — | — | |
| cuBLASLtGPU=H100, Model=BitNet-2B, Concurrency=512, Precision=INT82026.03 | 20,700 | — | — | — | — | |
| cuBLASLtGPU=H100, Model=BitNet-2B, Concurrency=256, Precision=INT82026.03 | 20,300 | — | — | — | — | |
| cuBLASLtGPU=RTX4090, Model=Llama3.2-1B, Concurrency=256, Precision=INT82026.03 | 20,200 | — | — | — | — | |
| cuBLASLtGPU=RTX4090, Model=Llama3.2-1B, Concurrency=512, Precision=INT82026.03 | 20,100 | — | — | — | — | |
| cuBLASLtGPU=H100, Model=Llama3.2-3B, Concurrency=512, Precision=INT82026.03 | 18,100 | — | — | — | — | |
| cuBLASLtGPU=H100, Model=Llama3.2-3B, Concurrency=256, Precision=INT82026.03 | 18,000 | — | — | — | — | |
| cuBLASLtGPU=B200, Model=BitNet-2B, Concurrency=128, Precision=INT82026.03 | 17,200 | — | — | — | — | |
| cuBLASLtGPU=B200, Model=Llama3.2-3B, Concurrency=128, Precision=INT82026.03 | 17,200 | — | — | — | — | |
| cuBLASLtGPU=B200, Model=Llama3.2-1B, Concurrency=64, Precision=INT82026.03 | 17,100 | — | — | — | — | |
| cuBLASLtGPU=A100, Model=BitNet-2B, Concurrency=256, Precision=INT82026.03 | 16,000 | — | — | — | — | |
| cuBLASLtGPU=A100, Model=Llama3.2-1B, Concurrency=128, Precision=INT82026.03 | 15,800 | — | — | — | — | |
| cuBLASLtGPU=B200, Model=Qwen2.5-7B, Concurrency=128, Precision=INT82026.03 | 15,400 | — | — | — | — | |
| cuBLASLtGPU=A100, Model=BitNet-2B, Concurrency=512, Precision=INT82026.03 | 15,000 | — | — | — | — | |
| cuBLASLtGPU=H100, Model=Qwen2.5-7B, Concurrency=512, Precision=INT82026.03 | 14,700 | — | — | — | — | |
| cuBLASLtGPU=RTX4090, Model=BitNet-2B, Concurrency=256, Precision=INT82026.03 | 14,600 | — | — | — | — | |
| cuBLASLtGPU=H100, Model=Qwen2.5-7B, Concurrency=256, Precision=INT82026.03 | 14,600 | — | — | — | — | |
| cuBLASLtGPU=A100, Model=Llama3.2-3B, Concurrency=256, Precision=INT82026.03 | 14,200 | — | — | — | — | |
| cuBLASLtGPU=A100, Model=Llama3.2-3B, Concurrency=512, Precision=INT82026.03 | 14,000 | — | — | — | — | |
| cuBLASLtGPU=RTX4090, Model=Llama3.2-1B, Concurrency=128, Precision=INT82026.03 | 14,000 | — | — | — | — | |
| cuBLASLtGPU=H100, Model=BitNet-2B, Concurrency=128, Precision=INT82026.03 | 13,300 | — | — | — | — | |
| cuBLASLtGPU=RTX4090, Model=BitNet-2B, Concurrency=512, Precision=INT82026.03 | 13,200 | — | — | — | — | |
| cuBLASLtGPU=H100, Model=Llama3.2-1B, Concurrency=64, Precision=INT82026.03 | 13,100 | — | — | — | — | |
| cuBLASLtGPU=RTX4090, Model=Llama3.2-3B, Concurrency=512, Precision=INT82026.03 | 11,600 | — | — | — | — | |
| cuBLASLtGPU=RTX4090, Model=Llama3.2-3B, Concurrency=256, Precision=INT82026.03 | 11,500 | — | — | — | — | |
| cuBLASLtGPU=H100, Model=Llama3.2-3B, Concurrency=128, Precision=INT82026.03 | 11,400 | — | — | — | — | |
| cuBLASLtGPU=A100, Model=Llama3.2-1B, Concurrency=64, Precision=INT82026.03 | 11,200 | — | — | — | — | |
| cuBLASLtGPU=B200, Model=Llama3.2-3B, Concurrency=64, Precision=INT82026.03 | 11,100 | — | — | — | — | |
| cuBLASLtGPU=A100, Model=BitNet-2B, Concurrency=128, Precision=INT82026.03 | 10,900 | — | — | — | — | |
| cuBLASLtGPU=RTX4090, Model=BitNet-2B, Concurrency=128, Precision=INT82026.03 | 10,600 | — | — | — | — | |
| cuBLASLtGPU=A100, Model=Qwen2.5-7B, Concurrency=256, Precision=INT82026.03 | 10,300 | — | — | — | — | |
| cuBLASLtGPU=A100, Model=Llama3.2-3B, Concurrency=128, Precision=INT82026.03 | 10,100 | — | — | — | — | |
| cuBLASLtGPU=B200, Model=Qwen2.5-14B, Concurrency=128, Precision=INT82026.03 | 10,000 | — | — | — | — | |
| cuBLASLtGPU=B200, Model=BitNet-2B, Concurrency=64, Precision=INT82026.03 | 9,970 | — | — | — | — | |
| cuBLASLtGPU=H100, Model=Qwen2.5-7B, Concurrency=128, Precision=INT82026.03 | 9,930 | — | — | — | — | |
| cuBLASLtGPU=A100, Model=Qwen2.5-7B, Concurrency=512, Precision=INT82026.03 | 9,820 | — | — | — | — | |
| cuBLASLtGPU=B200, Model=Qwen2.5-7B, Concurrency=64, Precision=INT82026.03 | 9,570 | — | — | — | — | |
| cuBLASLtGPU=RTX4090, Model=Llama3.2-1B, Concurrency=64, Precision=INT82026.03 | 9,370 | — | — | — | — | |
| cuBLASLtGPU=RTX4090, Model=Qwen2.5-7B, Concurrency=512, Precision=INT82026.03 | 8,700 | — | — | — | — | |
| cuBLASLtGPU=RTX4090, Model=Qwen2.5-7B, Concurrency=256, Precision=INT82026.03 | 8,690 | — | — | — | — | |
| cuBLASLtGPU=H100, Model=Qwen2.5-14B, Concurrency=256, Precision=INT82026.03 | 8,230 | — | — | — | — | |
| cuBLASLtGPU=RTX4090, Model=Llama3.2-3B, Concurrency=128, Precision=INT82026.03 | 8,190 | — | — | — | — | |
| cuBLASLtGPU=A100, Model=Qwen2.5-7B, Concurrency=128, Precision=INT82026.03 | 8,170 | — | — | — | — | |
| cuBLASLtGPU=H100, Model=BitNet-2B, Concurrency=64, Precision=INT82026.03 | 8,110 | — | — | — | — | |
| cuBLASLtGPU=H100, Model=Qwen2.5-14B, Concurrency=512, Precision=INT82026.03 | 8,080 | — | — | — | — | |
| cuBLASLtGPU=H100, Model=Llama3.2-3B, Concurrency=64, Precision=INT82026.03 | 7,340 | — | — | — | — | |
| cuBLASLtGPU=A100, Model=BitNet-2B, Concurrency=64, Precision=INT82026.03 | 7,090 | — | — | — | — | |
| cuBLASLtGPU=RTX4090, Model=BitNet-2B, Concurrency=64, Precision=INT82026.03 | 6,990 | — | — | — | — | |
| cuBLASLtGPU=A100, Model=Llama3.2-3B, Concurrency=64, Precision=INT82026.03 | 6,690 | — | — | — | — | |
| cuBLASLtGPU=RTX4090, Model=Qwen2.5-7B, Concurrency=128, Precision=INT82026.03 | 6,130 | — | — | — | — | |
| cuBLASLtGPU=RTX4090, Model=Llama3.2-3B, Concurrency=64, Precision=INT82026.03 | 6,080 | — | — | — | — | |
| cuBLASLtGPU=A100, Model=Qwen2.5-14B, Concurrency=256, Precision=INT82026.03 | 5,770 | — | — | — | — | |
| cuBLASLtGPU=H100, Model=Qwen2.5-7B, Concurrency=64, Precision=INT82026.03 | 5,720 | — | — | — | — | |
| cuBLASLtGPU=H100, Model=Qwen2.5-14B, Concurrency=128, Precision=INT82026.03 | 5,670 | — | — | — | — | |
| cuBLASLtGPU=B200, Model=Qwen2.5-14B, Concurrency=64, Precision=INT82026.03 | 5,670 | — | — | — | — | |
| cuBLASLtGPU=A100, Model=Qwen2.5-14B, Concurrency=512, Precision=INT82026.03 | 5,390 | — | — | — | — | |
| cuBLASLtGPU=A100, Model=Qwen2.5-7B, Concurrency=64, Precision=INT82026.03 | 5,000 | — | — | — | — | |
| cuBLASLtGPU=A100, Model=Qwen2.5-14B, Concurrency=128, Precision=INT82026.03 | 4,840 | — | — | — | — | |
| cuBLASLtGPU=RTX4090, Model=Qwen2.5-7B, Concurrency=64, Precision=INT82026.03 | 3,990 | — | — | — | — | |
| cuBLASLtGPU=H100, Model=Qwen2.5-14B, Concurrency=64, Precision=INT82026.03 | 3,310 | — | — | — | — | |
| cuBLASLtGPU=A100, Model=Qwen2.5-14B, Concurrency=64, Precision=INT82026.03 | 3,080 | — | — | — | — | |
| cuBLASLtGPU=RTX4090, Model=Qwen2.5-14B, Concurrency=128, Precision=INT82026.03 | 3,030 | — | — | — | — | |
| cuBLASLtGPU=RTX4090, Model=Qwen2.5-14B, Concurrency=256, Precision=INT82026.03 | 2,430 | — | — | — | — | |
| cuBLASLtGPU=RTX4090, Model=Qwen2.5-14B, Concurrency=64, Precision=INT82026.03 | 1,530 | — | — | — | — | |
| cuBLASLtGPU=RTX4090, Model=Qwen2.5-14B, Concurrency=512, Precision=INT82026.03 | 1,370 | — | — | — | — | |
| SlideSparseGPU=A100, Model=Llama3.2-1B, Concurrency=64, Precision=INT82026.03 | — | 1.1 | 1 | 0.91 | 0.96 | |
| SlideSparseGPU=A100, Model=Llama3.2-1B, Concurrency=128, Precision=INT82026.03 | — | 1.05 | 0.98 | 1.01 | 0.97 | |
| SlideSparseGPU=A100, Model=Llama3.2-1B, Concurrency=256, Precision=INT82026.03 | — | 1.07 | 1.03 | 1.01 | 0.99 | |
| SlideSparseGPU=A100, Model=Llama3.2-1B, Concurrency=512, Precision=INT82026.03 | — | 1.09 | 1.06 | 1.06 | 1.02 | |
| SlideSparseGPU=A100, Model=BitNet-2B, Concurrency=64, Precision=INT82026.03 | — | 1.16 | 1.11 | 1.1 | 1.07 | |
| SlideSparseGPU=A100, Model=BitNet-2B, Concurrency=128, Precision=INT82026.03 | — | 1.14 | 1.07 | 1.05 | 1.03 | |
| SlideSparseGPU=A100, Model=BitNet-2B, Concurrency=256, Precision=INT82026.03 | — | 1.14 | 1.06 | 1.05 | 1.03 | |
| SlideSparseGPU=A100, Model=BitNet-2B, Concurrency=512, Precision=INT82026.03 | — | 1.21 | 1.14 | 1.13 | 1.07 | |
| SlideSparseGPU=A100, Model=Llama3.2-3B, Concurrency=64, Precision=INT82026.03 | — | 1.09 | 1.04 | 1.01 | 0.99 | |
| SlideSparseGPU=A100, Model=Llama3.2-3B, Concurrency=128, Precision=INT82026.03 | — | 1.16 | 1.08 | 1.06 | 1.05 | |
| SlideSparseGPU=A100, Model=Llama3.2-3B, Concurrency=256, Precision=INT82026.03 | — | 1.13 | 1.06 | 1.05 | 1.03 | |
| SlideSparseGPU=A100, Model=Llama3.2-3B, Concurrency=512, Precision=INT82026.03 | — | 1.18 | 1.1 | 1.07 | 1.04 | |
| SlideSparseGPU=A100, Model=Qwen2.5-7B, Concurrency=64, Precision=INT82026.03 | — | 1.25 | 1.14 | 1.12 | 1.08 | |
| SlideSparseGPU=A100, Model=Qwen2.5-7B, Concurrency=128, Precision=INT82026.03 | — | 1.21 | 1.09 | 1.05 | 1.03 | |
| SlideSparseGPU=A100, Model=Qwen2.5-7B, Concurrency=256, Precision=INT82026.03 | — | 1.26 | 1.12 | 1.08 | 1.04 | |
| SlideSparseGPU=A100, Model=Qwen2.5-7B, Concurrency=512, Precision=INT82026.03 | — | 1.31 | 1.16 | 1.12 | 1.09 | |
| SlideSparseGPU=A100, Model=Qwen2.5-14B, Concurrency=64, Precision=INT82026.03 | — | 1.28 | 1.14 | 1.1 | 1.08 | |
| SlideSparseGPU=A100, Model=Qwen2.5-14B, Concurrency=128, Precision=INT82026.03 | — | 1.24 | 1.12 | 1.09 | 1.04 | |
| SlideSparseGPU=A100, Model=Qwen2.5-14B, Concurrency=256, Precision=INT82026.03 | — | 1.35 | 1.19 | 1.13 | 1.1 | |
| SlideSparseGPU=A100, Model=Qwen2.5-14B, Concurrency=512, Precision=INT82026.03 | — | 1.4 | 1.23 | 1.17 | 1.14 | |
| SlideSparseGPU=RTX4090, Model=Llama3.2-1B, Concurrency=64, Precision=INT82026.03 | — | 1.15 | 1.06 | 0.93 | 0.94 | |
| SlideSparseGPU=RTX4090, Model=Llama3.2-1B, Concurrency=128, Precision=INT82026.03 | — | 1.03 | 1.01 | 1 | 0.96 |