Language Modeling on LM Evaluation Harness (test)
74.11WG (Winograd Schema)Llama3.1-Mamba2-DPO
Evaluation results
| Method | Links | ||||||||||||||||
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| Llama3.1-Mamba2-DPOpercentage_attention_layers=50%, evaluation_mode=zero-shot2024.08 | 74.11 | 80.03 | 79.69 | 84.81 | 59.73 | 59.74 | 44 | 50.22 | 74.6 | 46.12 | 65.31 | — | — | — | — | — | |
| Llama-3.1-8B-Instructevaluation_mode=zero-shot2024.08 | 73.88 | 80.79 | 79.21 | 81.78 | 55.2 | 68.12 | 43.2 | 42.67 | 75.2 | 44.78 | 64.48 | — | — | — | — | — | |
| Llama3.1-Mamba-DPOpercentage_attention_layers=50%, evaluation_mode=zero-shot2024.08 | 73.8 | 80.41 | 77.36 | 84.01 | 56.57 | 63.5 | 44.2 | 46.07 | 74.4 | 43.44 | 64.38 | — | — | — | — | — | |
| Llama3.1-Mambapercentage_attention_layers=50%, evaluation_mode=zero-shot2024.08 | 72.77 | 79.33 | 75.91 | 82.24 | 53.84 | 62.13 | 42.8 | 40.02 | 72 | 42.11 | 62.32 | — | — | — | — | — | |
| Llama3.1-Mamba2percentage_attention_layers=50%, evaluation_mode=zero-shot2024.08 | 71.74 | 78.89 | 75.36 | 82.2 | 52.65 | 61.01 | 41.6 | 40.31 | 72.6 | 42.11 | 61.85 | — | — | — | — | — | |
| Mamba2-Llama3percentage_attention_layers=50%, evaluation_mode=zero-shot2024.08 | 71.51 | 81.45 | 79.47 | 78.83 | 58.19 | 55.7 | 44.2 | 57.74 | 72.4 | 38.85 | 63.84 | — | — | — | — | — | |
| TRI Mamba-7Bevaluation_mode=zero-shot2024.08 | 71.42 | 81.01 | 77.93 | 77.53 | 46.67 | 33.39 | 46.2 | 32.09 | 72.3 | 37.99 | 57.65 | — | — | — | — | — | |
| Nvidia Hybrid Mamba-8Bevaluation_mode=zero-shot2024.08 | 71.27 | 79.65 | 77.68 | 77.23 | 47.7 | 51.46 | 42.8 | 38.72 | 69.8 | 39.71 | 59.6 | — | — | — | — | — | |
| Mamba-Llama3percentage_attention_layers=50%, evaluation_mode=zero-shot2024.08 | 68.98 | 78.02 | 78.43 | 74.45 | 51.96 | 57.81 | 44 | 47.69 | 73 | 38.56 | 61.3 | — | — | — | — | — | |
| Mamba-Zephyrpercentage_attention_layers=50%, evaluation_mode=zero-shot2024.08 | 68.82 | 80.36 | 76.91 | 81.4 | 55.63 | 55.43 | 42.6 | 41.99 | 72.6 | 42.2 | 61.79 | — | — | — | — | — | |
| Llama-3.2-3B-Instructevaluation_mode=zero-shot2024.08 | 67.48 | 75.68 | 70.43 | 74.07 | 45.9 | 60.43 | 36 | 38.01 | 69.6 | 40.67 | 57.83 | — | — | — | — | — | |
| Llama3.2-Mamba-DPOpercentage_attention_layers=50%, evaluation_mode=zero-shot2024.08 | 67.4 | 77.31 | 72.56 | 79.97 | 52.65 | 55.09 | 41.6 | 48.53 | 70 | 43.64 | 60.88 | — | — | — | — | — | |
| Llama3.2-Mambapercentage_attention_layers=50%, evaluation_mode=zero-shot2024.08 | 67.32 | 77.31 | 70.37 | 77.65 | 48.38 | 54.48 | 39.4 | 42.02 | 66.4 | 40.29 | 58.36 | — | — | — | — | — | |
| Llama3.2-Mamba2-DPOpercentage_attention_layers=50%, evaluation_mode=zero-shot2024.08 | 67.32 | 77.69 | 74.45 | 80.26 | 54.1 | 52.47 | 42.4 | 50.28 | 65.4 | 43.44 | 60.78 | — | — | — | — | — | |
| Llama3.2-Mamba2percentage_attention_layers=50%, evaluation_mode=zero-shot2024.08 | 66.06 | 76.01 | 69.13 | 76.68 | 46.67 | 53.12 | 38.8 | 34.78 | 63.8 | 39.81 | 56.49 | — | — | — | — | — | |
| Mamba2-Llama3percentage_attention_layers=25%, evaluation_mode=zero-shot2024.08 | 64.8 | 78.73 | 77.7 | 76.35 | 52.47 | 53.71 | 42.4 | 55.33 | 64.8 | 39.23 | 60.55 | — | — | — | — | — | |
| Qwen3-1.7BAttention Mechanism=Global-Attn2026.01 | 64.48 | 75.68 | 66.48 | — | — | 60.48 | — | — | — | — | 71 | — | — | — | 62.95 | 95.9 | |
| ROSA-TuningAttention Mechanism=Window-Attn + ROSA, Base Model=Qwen3-1.7B2026.01 | 63.93 | 75.19 | 65.58 | — | — | 60.33 | — | — | — | — | 70.5 | — | — | — | 62.56 | 95.4 | |
| Mamba2-Llama3percentage_attention_layers=12.5%, evaluation_mode=zero-shot2024.08 | 63.38 | 76.82 | 73.14 | 75.84 | 50.26 | 50.78 | 39.6 | 50 | 65.8 | 36.46 | 58.21 | — | — | — | — | — | |
| Mamba-Llama3percentage_attention_layers=25%, evaluation_mode=zero-shot2024.08 | 62.83 | 78.07 | 75 | 74.28 | 47.35 | 53.5 | 40 | 43.64 | 65.4 | 36.94 | 57.7 | — | — | — | — | — | |
| Mamba-Llama3percentage_attention_layers=12.5%, evaluation_mode=zero-shot2024.08 | 59.75 | 75.08 | 71.71 | 70.58 | 43.6 | 49.81 | 41.4 | 41.41 | 62.4 | 34.45 | 55.02 | — | — | — | — | — | |
| Mamba2-Llama3percentage_attention_layers=0%, evaluation_mode=zero-shot2024.08 | 58.56 | 76.82 | 70.75 | 74.12 | 47.95 | 45.19 | 39 | 40.2 | 62.2 | 32.63 | 54.74 | — | — | — | — | — |