Zero-shot Reasoning on (OpenbookQA, ARC-e, ARC-c, WinoGrande, HellaSwag, PIQA, MathQA)
69.99Average AccuracyMeta-Llama-3-8B Dense
Evaluation Results
| Method | Links | |||||||||||
|---|---|---|---|---|---|---|---|---|---|---|---|---|
| Meta-Llama-3-8B DenseConfig=Dense, Fine-tuning Protocol=None, Base Model=Meta-Llama-3-8B2026.03 | 69.99 | 45 | 77.69 | 53.33 | 72.85 | 79.17 | 80.79 | — | — | 69.68 | 81.44 | |
| LFT-Hassle-free-ALPS-TMConfig=4:8+64LR, Fine-tuning Protocol=LFT, Base Model=Meta-Llama-3-8B2026.03 | 67 | 40.8 | 73.86 | 48.29 | 71.19 | 74.92 | 78.84 | — | — | 68.59 | 79.51 | |
| LFT-Hassle-free-SparseGPT-TMConfig=4:8+64LR, Fine-tuning Protocol=LFT, Base Model=Meta-Llama-3-8B2026.03 | 66.27 | 40.8 | 74.12 | 45.65 | 70.48 | 74.29 | 78.78 | — | — | 66.79 | 79.24 | |
| LFT-3BASiL-TMConfig=4:8+64LR, Fine-tuning Protocol=LFT, Base Model=Meta-Llama-3-8B2026.03 | 66.23 | 41.6 | 73.23 | 48.72 | 71.9 | 74.4 | 79.33 | — | — | 61.73 | 78.9 | |
| LFT-3BASiLConfig=4:8+64LR, Fine-tuning Protocol=LFT, Base Model=Meta-Llama-3-8B2026.03 | 65.68 | 43.4 | 71 | 45.56 | 72.45 | 74.85 | 78.51 | — | — | 61.73 | 77.92 | |
| LFT-Hassle-free-SparseGPTConfig=4:8+64LR, Fine-tuning Protocol=LFT, Base Model=Meta-Llama-3-8B2026.03 | 64.99 | 41.4 | 70.45 | 43.86 | 71.35 | 73.45 | 78.18 | — | — | 62.45 | 78.81 | |
| LFT-3BASiLConfig=2:4+64LR, Fine-tuning Protocol=LFT, Base Model=Meta-Llama-3-8B2026.03 | 64.94 | 41.2 | 69.19 | 43.52 | 71.98 | 72.9 | 77.37 | — | — | 67.15 | 76.24 | |
| LFT-Hassle-free-ALPSConfig=2:4+64LR, Fine-tuning Protocol=LFT, Base Model=Meta-Llama-3-8B2026.03 | 64.32 | 39.8 | 67.09 | 42.92 | 70.56 | 72.4 | 77.48 | — | — | 67.15 | 77.13 | |
| LFT-OATSConfig=4:8+64LR, Fine-tuning Protocol=LFT, Base Model=Meta-Llama-3-8B2026.03 | 64.22 | 40.4 | 70.92 | 45.39 | 70.72 | 72.31 | 77.8 | — | — | 60.65 | 75.6 | |
| LFT-Hassle-free-SparseGPT-TMConfig=2:4+64LR, Fine-tuning Protocol=LFT, Base Model=Meta-Llama-3-8B2026.03 | 64.12 | 40.4 | 67.8 | 43.6 | 70.01 | 72.57 | 79 | — | — | 63.9 | 75.69 | |
| LFT-Hassle-free-ALPSConfig=4:8+64LR, Fine-tuning Protocol=LFT, Base Model=Meta-Llama-3-8B2026.03 | 63.91 | 40.2 | 71.04 | 44.45 | 72.77 | 71.19 | 76.82 | — | — | 56.68 | 78.13 | |
| LFT-OATS-TMConfig=2:4+64LR, Fine-tuning Protocol=LFT, Base Model=Meta-Llama-3-8B2026.03 | 63.81 | 40.6 | 69.99 | 44.28 | 69.77 | 71.25 | 77.69 | — | — | 58.84 | 78.07 | |
| LFT-Hassle-free-ALPS-TMConfig=2:4+64LR, Fine-tuning Protocol=LFT, Base Model=Meta-Llama-3-8B2026.03 | 63.67 | 40.4 | 67.38 | 41.89 | 70.88 | 72.51 | 78.24 | — | — | 62.45 | 75.63 | |
| LFT-3BASiL-TMConfig=2:4+64LR, Fine-tuning Protocol=LFT, Base Model=Meta-Llama-3-8B2026.03 | 63.64 | 39.2 | 68.14 | 41.89 | 71.98 | 73.17 | 78.73 | — | — | 59.57 | 76.42 | |
| Base modelBackbone=Mixtral-8x7B, Pruned Ratio=0%2026.06 | 63.29 | 36 | 84 | 57 | 76 | 65 | 82 | 43 | — | — | — | |
| FlexMoEBackbone=Phi-3.5-MoE, Pruned Ratio=20%2026.06 | 63.29 | — | — | — | — | — | — | — | — | — | — | |
| LFT-Hassle-free-SparseGPTConfig=2:4+64LR, Fine-tuning Protocol=LFT, Base Model=Meta-Llama-3-8B2026.03 | 63.25 | 39.4 | 68.86 | 41.72 | 70.01 | 71.62 | 77.04 | — | — | 61.37 | 76.02 | |
| LFT-OATSConfig=2:4+64LR, Fine-tuning Protocol=LFT, Base Model=Meta-Llama-3-8B2026.03 | 62.54 | 38.8 | 65.66 | 40.7 | 68.27 | 69.54 | 77.15 | — | — | 66.43 | 73.73 | |
| Base modelBackbone=Phi-3.5-MoE, Pruned Ratio=0%2026.06 | 62 | — | — | — | — | — | — | — | — | — | — | |
| FlexMoEBackbone=Mixtral-8x7B, Pruned Ratio=20%2026.06 | 61.86 | 35 | 82 | 54 | 77 | 64 | 81 | 40 | — | — | — | |
| TD-MoEBackbone=Mixtral-8x7B, Pruned Ratio=20%2026.06 | 61.71 | 33 | 83 | 53 | 77 | 64 | 82 | 40 | — | — | — | |
| LFT-Hassle-free-SparseGPT-TMConfig=3:8+64LR, Fine-tuning Protocol=LFT, Base Model=Meta-Llama-3-8B2026.03 | 61.42 | 35.2 | 62.04 | 37.54 | 67.25 | 66.5 | 75.68 | — | — | 70.4 | 76.79 | |
| LFT-3BASiL-TMConfig=3:8+64LR, Fine-tuning Protocol=LFT, Base Model=Meta-Llama-3-8B2026.03 | 61.36 | 37.6 | 63.17 | 39.08 | 68.03 | 68.15 | 75.41 | — | — | 62.09 | 77.34 | |
| MoE-SVDBackbone=Phi-3.5-MoE, Pruned Ratio=20%, Protocol=fine-tuned2026.06 | 61.14 | — | — | — | — | — | — | — | — | — | — | |
| TD-MoEBackbone=Phi-3.5-MoE, Pruned Ratio=20%2026.06 | 61 | — | — | — | — | — | — | — | — | — | — | |
| LFT-Hassle-free-ALPSConfig=3:8+64LR, Fine-tuning Protocol=LFT, Base Model=Meta-Llama-3-8B2026.03 | 60.52 | 37.2 | 60.35 | 36.43 | 68.67 | 66.86 | 74.59 | — | — | 65.34 | 74.71 | |
| MoE-SVDBackbone=Mixtral-8x7B, Pruned Ratio=20%, Protocol=fine-tuned2026.06 | 60.14 | 33 | 80 | 55 | 73 | 61 | 81 | 38 | — | — | — | |
| LFT-Hassle-free-ALPS-TMConfig=3:8+64LR, Fine-tuning Protocol=LFT, Base Model=Meta-Llama-3-8B2026.03 | 59.99 | 36.2 | 61.66 | 37.03 | 68.98 | 68.34 | 75.9 | — | — | 62.45 | 69.33 | |
| LFT-3BASiLConfig=3:8+64LR, Fine-tuning Protocol=LFT, Base Model=Meta-Llama-3-8B2026.03 | 59.87 | 38.2 | 63.55 | 38.65 | 67.48 | 67.47 | 74.7 | — | — | 55.6 | 73.27 | |
| FlexMoEBackbone=Phi-3.5-MoE, Pruned Ratio=40%2026.06 | 59.43 | — | — | — | — | — | — | — | — | — | — | |
| LFT-OATS-TMConfig=3:8+64LR, Fine-tuning Protocol=LFT, Base Model=Meta-Llama-3-8B2026.03 | 59.2 | 36.8 | 60.65 | 37.46 | 66.54 | 65.43 | 73.61 | — | — | 57.4 | 75.72 | |
| FlexMoEBackbone=Qwen2-57B-A14B, Pruned Ratio=40%, Fine-tuning=False2026.06 | 58.86 | — | — | — | — | — | — | — | — | — | — | |
| Base modelBackbone=Qwen2-57B-A14B, Pruned Ratio=0%2026.06 | 58.71 | — | — | — | — | — | — | — | — | — | — | |
| FlexMoEBackbone=Qwen2-57B-A14B, Pruned Ratio=60%, Fine-tuning=False2026.06 | 58.57 | — | — | — | — | — | — | — | — | — | — | |
| FlexMoEBackbone=Qwen2-57B-A14B, Pruned Ratio=20%, Fine-tuning=False2026.06 | 58.43 | — | — | — | — | — | — | — | — | — | — | |
| LFT-Hassle-free-SparseGPTConfig=3:8+64LR, Fine-tuning Protocol=LFT, Base Model=Meta-Llama-3-8B2026.03 | 58.3 | 35 | 58 | 37.2 | 66.69 | 64.77 | 73.94 | — | — | 61.01 | 69.79 | |
| TD-MoEBackbone=Qwen2-57B-A14B, Pruned Ratio=20%2026.06 | 58.29 | — | — | — | — | — | — | — | — | — | — | |
| FlexMoEBackbone=Mixtral-8x7B, Pruned Ratio=40%2026.06 | 58 | 33 | 77 | 49 | 73 | 60 | 80 | 34 | — | — | — | |
| TD-MoEBackbone=Phi-3.5-MoE, Pruned Ratio=40%2026.06 | 57.86 | — | — | — | — | — | — | — | — | — | — | |
| NAEEBackbone=Mixtral-8x7B, Pruned Ratio=20%2026.06 | 57.71 | 32 | 76 | 47 | 72 | 58 | 79 | 40 | — | — | — | |
| MoE-I²Backbone=Mixtral-8x7B, Pruned Ratio=20%2026.06 | 57.57 | 32 | 79 | 48 | 74 | 55 | 78 | 37 | — | — | — | |
| NAEEBackbone=Phi-3.5-MoE, Pruned Ratio=40%2026.06 | 57.57 | — | — | — | — | — | — | — | — | — | — | |
| TD-MoEBackbone=Mixtral-8x7B, Pruned Ratio=40%2026.06 | 57 | 28 | 77 | 47 | 76 | 57 | 79 | 35 | — | — | — | |
| MoE-SVDBackbone=Qwen2-57B-A14B, Pruned Ratio=20%2026.06 | 56.57 | — | — | — | — | — | — | — | — | — | — | |
| LFT-OATSConfig=3:8+64LR, Fine-tuning Protocol=LFT, Base Model=Meta-Llama-3-8B2026.03 | 56.15 | 35 | 57.45 | 35.58 | 63.93 | 61.2 | 72.36 | — | — | 53.07 | 70.58 | |
| NAEEBackbone=Qwen2-57B-A14B, Pruned Ratio=20%2026.06 | 55.86 | — | — | — | — | — | — | — | — | — | — | |
| MoE-SVDBackbone=Phi-3.5-MoE, Pruned Ratio=40%2026.06 | 55.86 | — | — | — | — | — | — | — | — | — | — | |
| TD-MoEBackbone=Qwen2-57B-A14B, Pruned Ratio=40%2026.06 | 55.57 | — | — | — | — | — | — | — | — | — | — | |
| LittleBitModel=Llama2-7B, Data=196.00M, GPU Hours=123.6, Quantization=1-bit2026.02 | 54.92 | — | — | — | — | — | — | — | — | — | — | |
| DBFModel=Llama2-7B, Data=1.38B, GPU Hours=37.6, Quantization=1-bit2026.02 | 54.24 | — | — | — | — | — | — | — | — | — | — | |
| NAEEBackbone=Qwen2-57B-A14B, Pruned Ratio=40%2026.06 | 53.14 | — | — | — | — | — | — | — | — | — | — | |
| FlexMoEBackbone=Phi-3.5-MoE, Pruned Ratio=60%2026.06 | 53 | — | — | — | — | — | — | — | — | — | — | |
| DBFModel=Qwen3-4B, Data=1.19B, GPU Hours=25.3, Quantization=1-bit2026.02 | 52.3 | — | — | — | — | — | — | — | — | — | — | |
| BaselineBackbone=LLaMA-7B, Ratio=0.0, Fine-tuning=false, Mixed-rank=false2026.02 | 52 | 28 | 67 | 38 | 67 | 56 | 78 | 27 | 0 | — | — | |
| TD-MoEBackbone=Qwen2-57B-A14B, Pruned Ratio=60%2026.06 | 51.57 | — | — | — | — | — | — | — | — | — | — | |
| LFT-3BASiL-TMConfig=2:8+64LR, Fine-tuning Protocol=LFT, Base Model=Meta-Llama-3-8B2026.03 | 51.33 | 32.2 | 45.92 | 26.96 | 60.69 | 52.97 | 68.12 | — | — | 53.43 | 70.34 | |
| NANOQUANTModel=Llama2-7B, Data=1.05M, GPU Hours=2.1, Quantization=1-bit2026.02 | 51.01 | — | — | — | — | — | — | — | — | — | — | |
| NANOQUANTModel=Qwen3-4B, Data=1.05M, GPU Hours=2.3, Quantization=1-bit2026.02 | 50.63 | — | — | — | — | — | — | — | — | — | — | |
| LFT-Hassle-free-ALPS-TMConfig=2:8+64LR, Fine-tuning Protocol=LFT, Base Model=Meta-Llama-3-8B2026.03 | 50.2 | 31.2 | 46.84 | 27.99 | 60.54 | 52.18 | 67.95 | — | — | 53.43 | 61.44 | |
| LFT-3BASiLConfig=2:8+64LR, Fine-tuning Protocol=LFT, Base Model=Meta-Llama-3-8B2026.03 | 50.15 | 30.6 | 44.02 | 26.37 | 60.06 | 49.63 | 66.81 | — | — | 55.6 | 68.1 | |
| SAES-SVDBackbone=LLaMA-7B, Ratio=0.2, Fine-tuning=false, Mixed-rank=false2026.02 | 50 | 29 | 68 | 36 | 65 | 45 | 75 | 25 | 390 | — | — | |
| MoE-SVDBackbone=Mixtral-8x7B, Pruned Ratio=40%2026.06 | 50 | 27 | 72 | 38 | 67 | 43 | 71 | 32 | — | — | — | |
| TD-MoEBackbone=Phi-3.5-MoE, Pruned Ratio=60%2026.06 | 49.86 | — | — | — | — | — | — | — | — | — | — | |
| FlexMoEBackbone=Mixtral-8x7B, Pruned Ratio=60%2026.06 | 49.57 | 23 | 65 | 35 | 71 | 51 | 74 | 28 | — | — | — | |
| MoE-I² (P+F)Backbone=Mixtral-8x7B, Pruned Ratio=40%2026.06 | 49.14 | 26 | 71 | 38 | 66 | 43 | 69 | 31 | — | — | — | |
| LFT-Hassle-free-SparseGPT-TMConfig=2:8+64LR, Fine-tuning Protocol=LFT, Base Model=Meta-Llama-3-8B2026.03 | 48.96 | 29.8 | 43.77 | 26.37 | 58.25 | 48.49 | 66 | — | — | 52.71 | 66.27 | |
| NAEEBackbone=Mixtral-8x7B, Pruned Ratio=40%2026.06 | 48.71 | 25 | 63 | 36 | 64 | 46 | 72 | 35 | — | — | — | |
| MoE-SVDBackbone=Phi-3.5-MoE, Pruned Ratio=60%2026.06 | 48.57 | — | — | — | — | — | — | — | — | — | — | |
| MoE-SVDBackbone=Qwen2-57B-A14B, Pruned Ratio=40%2026.06 | 48.14 | — | — | — | — | — | — | — | — | — | — | |
| LittleBitModel=Qwen3-4B, Data=169.50M, GPU Hours=92.5, Quantization=1-bit2026.02 | 47.32 | — | — | — | — | — | — | — | — | — | — | |
| LFT-OATS-TMConfig=2:8+64LR, Fine-tuning Protocol=LFT, Base Model=Meta-Llama-3-8B2026.03 | 47.11 | 30.6 | 42.09 | 25.26 | 52.64 | 43.81 | 65.61 | — | — | 52.71 | 64.13 | |
| Dip-SVDBackbone=LLaMA-7B, Ratio=0.2, Fine-tuning=false, Mixed-rank=true2026.02 | 47 | 27 | 63 | 33 | 64 | 45 | 71 | 24 | 920 | — | — | |
| MoE-SVDBackbone=Qwen2-57B-A14B, Pruned Ratio=60%2026.06 | 46.86 | — | — | — | — | — | — | — | — | — | — | |
| LFT-Hassle-free-ALPSConfig=2:8+64LR, Fine-tuning Protocol=LFT, Base Model=Meta-Llama-3-8B2026.03 | 46.85 | 29.6 | 43.73 | 26.62 | 59.43 | 48.42 | 66.38 | — | — | 53.43 | 47.16 | |
| LFT-Hassle-free-SparseGPTConfig=2:8+64LR, Fine-tuning Protocol=LFT, Base Model=Meta-Llama-3-8B2026.03 | 46.58 | 29.4 | 40.28 | 24.91 | 54.85 | 43.3 | 62.35 | — | — | 53.07 | 64.46 | |
| Dobi-SVDBackbone=LLaMA-7B, Ratio=0.2, Fine-tuning=true, Mixed-rank=true2026.02 | 46 | 26 | 59 | 31 | 66 | 44 | 70 | 23 | 1,080 | — | — | |
| MoE-I²Backbone=Phi-3.5-MoE, Pruned Ratio=40%2026.06 | 45.29 | — | — | — | — | — | — | — | — | — | — | |
| LFT-OATSConfig=2:8+64LR, Fine-tuning Protocol=LFT, Base Model=Meta-Llama-3-8B2026.03 | 44.49 | 27.2 | 38.72 | 24.23 | 51.93 | 36.88 | 62.24 | — | — | 52.71 | 62.02 | |
| SVD-LLMBackbone=LLaMA-7B, Ratio=0.2, Fine-tuning=true, Mixed-rank=false2026.02 | 44 | 22 | 58 | 29 | 63 | 43 | 69 | 24 | 1,470 | — | — | |
| NAEEBackbone=Qwen2-57B-A14B, Pruned Ratio=60%2026.06 | 44 | — | — | — | — | — | — | — | — | — | — | |
| ASVDBackbone=LLaMA-7B, Ratio=0.2, Fine-tuning=true, Mixed-rank=true2026.02 | 43 | 25 | 53 | 27 | 64 | 41 | 68 | 24 | 1,670 | — | — | |
| TD-MoEBackbone=Mixtral-8x7B, Pruned Ratio=60%2026.06 | 41.86 | 21 | 55 | 28 | 62 | 38 | 65 | 24 | — | — | — | |
| SAES-SVDBackbone=LLaMA-7B, Ratio=0.4, Fine-tuning=false, Mixed-rank=false2026.02 | 41 | 23 | 50 | 29 | 62 | 36 | 65 | 23 | 2,110 | — | — | |
| Dip-SVDBackbone=LLaMA-7B, Ratio=0.4, Fine-tuning=false, Mixed-rank=true2026.02 | 40 | 22 | 50 | 30 | 61 | 36 | 64 | 22 | 2,280 | — | — | |
| Dobi-SVDBackbone=LLaMA-7B, Ratio=0.4, Fine-tuning=true, Mixed-rank=true2026.02 | 38 | 22 | 41 | 27 | 58 | 34 | 61 | 23 | 2,630 | — | — | |
| MoE-SVDBackbone=Mixtral-8x7B, Pruned Ratio=60%2026.06 | 37.43 | 19 | 45 | 23 | 55 | 33 | 62 | 25 | — | — | — | |
| SVD-LLMBackbone=LLaMA-7B, Ratio=0.4, Fine-tuning=true, Mixed-rank=false2026.02 | 37 | 19 | 42 | 25 | 58 | 33 | 60 | 21 | 2,830 | — | — | |
| NAEEBackbone=Mixtral-8x7B, Pruned Ratio=60%2026.06 | 36.86 | 17 | 42 | 23 | 55 | 33 | 62 | 26 | — | — | — | |
| MoE-I²Backbone=Mixtral-8x7B, Pruned Ratio=60%2026.06 | 36 | 18 | 44 | 22 | 55 | 32 | 58 | 23 | — | — | — | |
| SAES-SVDBackbone=LLaMA-7B, Ratio=0.6, Fine-tuning=false, Mixed-rank=false2026.02 | 34 | 16 | 33 | 25 | 52 | 30 | 54 | 23 | 3,410 | — | — | |
| Dobi-SVDBackbone=LLaMA-7B, Ratio=0.6, Fine-tuning=true, Mixed-rank=true2026.02 | 32 | 15 | 31 | 20 | 52 | 28 | 54 | 22 | 3,800 | — | — | |
| SVD-LLMBackbone=LLaMA-7B, Ratio=0.6, Fine-tuning=true, Mixed-rank=false2026.02 | 31 | 14 | 28 | 22 | 50 | 27 | 55 | 21 | 3,990 | — | — | |
| ASVDBackbone=LLaMA-7B, Ratio=0.4, Fine-tuning=true, Mixed-rank=true2026.02 | 30 | 13 | 28 | 22 | 48 | 26 | 55 | 19 | 4,180 | — | — | |
| ASVDBackbone=LLaMA-7B, Ratio=0.6, Fine-tuning=true, Mixed-rank=true2026.02 | 29 | 12 | 26 | 21 | 49 | 26 | 53 | 18 | 4,380 | — | — | |
| FWSVDBackbone=LLaMA-7B, Ratio=0.2, Fine-tuning=false, Mixed-rank=false2026.02 | 2 | 9 | 11 | 6 | 5 | 8 | 10 | 5 | 9,600 | — | — | |
| FWSVDBackbone=LLaMA-7B, Ratio=0.4, Fine-tuning=false, Mixed-rank=false2026.02 | 2 | 6 | 5 | 2 | 2 | 0 | 5 | 3 | 9,660 | — | — | |
| FWSVDBackbone=LLaMA-7B, Ratio=0.6, Fine-tuning=false, Mixed-rank=false2026.02 | 1 | 6 | 1 | 0 | 0 | 1 | 1 | 0 | 9,780 | — | — |