Machine Translation on FLORES xx→en (test)
-65.05Score (de→en)OPUS
Evaluation results
| Method | Links | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| OPUSnoise_type=swap, beam_size=5, training_data=open2024.03 | -65.05 | -65.41 | -36.18 | -63.44 | -57.52 | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | |
| OPUSnoise_type=key, beam_size=5, training_data=open2024.03 | -63.78 | -65.4 | -38.48 | -65.5 | -58.29 | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | |
| OPUSnoise_type=drop, beam_size=5, training_data=open2024.03 | -53.61 | -49.92 | -29.54 | -52.48 | -46.39 | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | |
| OPUSnoise_type=dupe, beam_size=5, training_data=open2024.03 | -34.31 | -31.83 | -6.92 | -34.58 | -26.91 | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | |
| TInoise_type=key, beam_size=5, training_data=closed2024.03 | -29.15 | -32.18 | -19.8 | -34.15 | -28.82 | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | |
| TInoise_type=swap, beam_size=5, training_data=closed2024.03 | -27.61 | -27.01 | -23.45 | -25.54 | -25.9 | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | |
| NLLBnoise_type=key, beam_size=5, training_data=open2024.03 | -20.66 | -21.86 | -28.01 | -23.6 | -23.53 | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | |
| TInoise_type=drop, beam_size=5, training_data=closed2024.03 | -19.62 | -18.01 | -17.33 | -17.71 | -18.17 | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | |
| NLLBnoise_type=swap, beam_size=5, training_data=open2024.03 | -18.14 | -20.97 | -23.79 | -20.81 | -20.93 | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | |
| NLLBnoise_type=drop, beam_size=5, training_data=open2024.03 | -16.5 | -17.27 | -21.11 | -19.04 | -18.48 | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | |
| GPT-3.5noise_type=key, temperature=0, training_data=closed2024.03 | -9.17 | -8.63 | -16.31 | -10.27 | -11.09 | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | |
| GPT-3.5noise_type=drop, temperature=0, training_data=closed2024.03 | -6.55 | -5.68 | -17.81 | -7.09 | -9.28 | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | |
| GPT-3.5noise_type=swap, temperature=0, training_data=closed2024.03 | -4.36 | -5.85 | -20.89 | -6.78 | -9.47 | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | |
| NLLBnoise_type=dupe, beam_size=5, training_data=open2024.03 | -4.07 | -5.31 | -4.36 | -4.58 | -4.58 | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | |
| TInoise_type=dupe, beam_size=5, training_data=closed2024.03 | -3.37 | -4.54 | -1.82 | -3.62 | -3.38 | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | |
| GPT-3.5noise_type=dupe, temperature=0, training_data=closed2024.03 | -1.36 | -1.42 | -5.64 | -1.44 | -2.47 | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | |
| ALMA-13bParameters=13b2024.02 | 27.7 | 27.4 | — | — | 18 | 10.8 | — | 12.1 | 18.1 | 10.2 | — | 19.6 | 14.2 | — | 22.7 | — | 16.9 | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | |
| Whisper-Large V2Pipeline=End-to-end ST, Source=Reproduced2024.02 | 34.6 | 32.2 | — | 38.1 | 23.5 | 25.5 | 13 | 23.7 | 23.3 | 19.6 | 22 | 23.6 | 18.9 | 9.2 | 29.4 | 20.4 | 18.4 | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | |
| SeamlessM4T-LargePipeline=End-to-end ST, Source=Reproduced2024.02 | 35.8 | 33.1 | — | 38.9 | 27.1 | 32.8 | 31.7 | 25.6 | 25 | 28.2 | 26.3 | 25 | 17 | 16 | 30.2 | 21.6 | 19.8 | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | |
| BigTranslate2024.02 | 35.9 | 38.7 | — | — | 22.8 | 18.6 | — | 9.5 | 29 | 1.4 | — | 29 | 16.9 | — | 25.9 | — | 23 | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | |
| SeamlessM4T-Large-V2Pipeline=End-to-end ST, Source=Reproduced (Latest Baseline)2024.02 | 37.1 | 33.7 | — | 38.5 | 29.4 | 34.7 | 34.9 | 27.3 | 25.4 | 30.3 | 28.5 | 26.5 | 19.5 | 22.1 | 33.2 | 25.7 | 23 | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | |
| AudioPaLM2Pipeline=End-to-end ST, Source=Reported2024.02 | 38.7 | 36.5 | — | 38.4 | 24 | 29 | 7.2 | 18.8 | 26.9 | 25.7 | 21.7 | 27.8 | 11.1 | 15 | 26.9 | 15.6 | 21.3 | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | |
| SeamlessM4T-V2 (ASR+MT)Pipeline=Cascaded ASR+MT, Source=Reproduced (Latest Baseline)2024.02 | 39.1 | 35.7 | — | 43.2 | 32.3 | 39.2 | 36.8 | 29.4 | 26.7 | 33.9 | 32.9 | 29.3 | 22.5 | 25.4 | 34.8 | 29.7 | 25.9 | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | |
| GenTranslatePipeline=End-to-end ST, Source=Reproduced2024.02 | 39.2 | 37 | — | 43 | 30.1 | 34.6 | 33.6 | 29.4 | 29.8 | 30.5 | 28.3 | 29.7 | 18.6 | 17.4 | 33.9 | 24.1 | 21.7 | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | |
| SeamlessM4T (ASR+MT)Pipeline=Cascaded ASR+MT, Source=Reproduced2024.02 | 39.7 | 37.7 | — | 23.7 | 31.9 | 38.9 | 37 | 29 | 27.7 | 34.1 | 33.9 | 21.7 | 42.3 | 34 | 24.9 | 24.4 | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | |
| Whisper + NLLB-3.3bPipeline=Cascaded ASR+MT, Source=Reproduced2024.02 | 40.5 | 39.7 | — | 44.3 | 31.2 | 35.5 | 29.6 | 31.1 | 30.9 | 28.2 | 26.7 | 30 | 24.7 | 20 | 35.3 | 26.4 | 25.4 | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | |
| GenTranslate-V2Pipeline=End-to-end ST, Source=Reproduced2024.02 | 40.7 | 37.8 | — | 43 | 32.3 | 37.6 | 36.8 | 31.5 | 29.9 | 33.4 | 30.4 | 31.2 | 21 | 23.4 | 36.2 | 27.2 | 25 | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | |
| GenTranslate-V2Pipeline=Cascaded ASR+MT, Source=Reproduced2024.02 | 40.9 | 40 | — | 44.2 | 34.2 | 40 | 39.1 | 33.8 | 30 | 35.4 | 33 | 31.6 | 23.7 | 26.4 | 37.1 | 30.9 | 26.9 | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | |
| GenTranslatePipeline=Cascaded ASR+MT, Source=Reproduced2024.02 | 41.6 | 40.6 | — | 45 | 34 | 39.9 | 39.4 | 32.8 | 31.2 | 35.9 | 34.9 | 32.1 | 22.8 | 24.1 | 36.9 | 27.4 | 25.7 | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | |
| SeamlessM4T-Large-V22024.02 | 44.1 | 45.5 | — | — | 36.1 | 41.5 | — | 35.6 | 29.9 | 37.6 | — | 33.5 | 25.5 | — | 39 | — | 29 | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | |
| GenTranslate-V2Backbone=SeamlessM4T-Large-V22024.02 | 44.5 | 46.7 | — | — | 37.3 | 42 | — | 36.6 | 34.4 | 38.1 | — | 35.1 | 26.7 | — | 39.3 | — | 29.9 | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | |
| NLLB-3.3bParameters=3.3b2024.02 | 44.6 | 46.2 | — | — | 37.5 | 43 | — | 37.7 | 32.2 | 38.7 | — | 34.6 | 28.1 | — | 40.8 | — | 29.5 | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | |
| OPUSMetric=BLEU, nrefs=1, case=mixed, eff=no, tok=flores200, smooth=exp, version=2.5.12024.03 | 44.7 | 49.7 | 30.7 | 49.3 | 43.6 | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | |
| SeamlessM4T-Large2024.02 | 45.1 | 45.1 | — | — | 37.5 | 43.7 | — | 37.7 | 31.5 | 39 | — | 35.2 | 26.1 | — | 41.2 | — | 29.9 | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | |
| GenTranslateBackbone=SeamlessM4T-Large2024.02 | 45.3 | 46.4 | — | — | 38.5 | 43.9 | — | 38.5 | 35.5 | 39.4 | — | 36.6 | 26.7 | — | 41.8 | — | 30.5 | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | |
| GPT-3.5Metric=BLEU, nrefs=1, case=mixed, eff=no, tok=flores200, smooth=exp, version=2.5.12024.03 | 48.8 | 49.1 | 32.5 | 53.6 | 46 | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | |
| NLLBMetric=BLEU, nrefs=1, case=mixed, eff=no, tok=flores200, smooth=exp, version=2.5.12024.03 | 49.3 | 50.3 | 34.8 | 55.1 | 47.4 | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | |
| TIMetric=BLEU, nrefs=1, case=mixed, eff=no, tok=flores200, smooth=exp, version=2.5.12024.03 | 50.6 | 52.1 | 37 | 56.7 | 49.1 | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | |
| Granite 4 MicroVariant=Base2026.01 | — | — | — | — | — | — | 49.7 | — | — | — | — | — | — | — | — | — | — | 18.8 | 59.3 | 45.2 | 60.7 | 38.2 | 53.5 | 45.8 | 31.1 | 26.6 | 46.7 | 51.2 | 43.3 | 63.5 | 28.2 | 42.3 | 34 | 34.3 | 36.6 | 44.9 | 26.3 | 38.3 | 34.1 | 18.4 | 56.9 | 50.4 | 27.3 | 27.8 | 29.4 | 58.7 | 31.8 | 27.6 | 52.1 | 31.5 | 25.3 | 25.9 | 31.3 | 27.6 | 41 | 29.3 | 32.2 | 39.2 | 26.2 | 43.5 | 41 | 38.7 | 39.9 | 54.9 | 16.6 | 34.1 | 31.9 | 31 | 32.5 | 29.5 | |
| KakugoVariant=GenReas+Tran (full)2026.01 | — | — | — | — | — | — | 62.7 | — | — | — | — | — | — | — | — | — | — | 31.6 | 60.1 | 47.6 | 61 | 45.8 | 51.1 | 57.8 | 41.3 | 23.8 | 61.8 | 52.8 | 51.1 | 63.2 | 35.4 | 55.8 | 45 | 40.9 | 48.7 | 53.4 | 43.5 | 43.1 | 38.8 | 19.4 | 62.2 | 64.7 | 38.9 | 28.4 | 42.8 | 57.4 | 39.7 | 37.7 | 64.4 | 45 | 35.5 | 40.2 | 44.3 | 40.7 | 53.1 | 41.7 | 34.7 | 49.4 | 41.5 | 53 | 52.8 | 49.6 | 48.3 | 54.2 | 22.8 | 45.6 | 42.2 | 43.1 | 36.9 | 46.7 |