Speech Translation Metric Evaluation on Europarl-ST (test)
0.9978Average CorrelationASRsmlss
Evaluation results
| Method | Links | ||||||||||||||
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| ASRsmlssEvaluation Metric=COMET2025.11 | 0.9978 | 0.9972 | 0.9971 | 0.9988 | 0.9985 | 0.9982 | 0.9969 | 86.83 | 86.25 | 90.92 | 89.08 | 89.05 | 85.37 | 87.55 | |
| ASRwhsprEvaluation Metric=COMET2025.11 | 0.9977 | 0.997 | 0.9971 | 0.9987 | 0.9985 | 0.998 | 0.997 | 85.68 | 85.91 | 90.14 | 89.06 | 87.86 | 86.01 | 87.1 | |
| BTmdldEvaluation Metric=COMET2025.11 | 0.9951 | 0.9944 | 0.9939 | 0.9965 | 0.9964 | 0.9958 | 0.9938 | 73.39 | 70.71 | 73.02 | 73.54 | 74 | 70.72 | 72.39 | |
| BTnllbEvaluation Metric=COMET2025.11 | 0.9944 | 0.9936 | 0.993 | 0.9961 | 0.9958 | 0.9952 | 0.9929 | 69.84 | 66.36 | 69.42 | 69.48 | 70.27 | 66.29 | 68.41 | |
| ASRowsmEvaluation Metric=COMET2025.11 | 0.9909 | 0.9846 | 0.9887 | 0.9967 | 0.9952 | 0.992 | 0.9881 | 27.02 | 45.69 | 73.97 | 64.83 | 50.45 | 43.94 | 48.24 | |
| COMET-refEvaluation Metric=COMET2025.11 | 0.9857 | 0.9842 | 0.9841 | 0.9851 | 0.9894 | 0.9873 | 0.984 | — | — | — | — | — | — | — | |
| shuffEvaluation Metric=COMET2025.11 | 0.9824 | 0.9789 | 0.9791 | 0.9872 | 0.9863 | 0.9838 | 0.9788 | — | — | — | — | — | — | — | |
| whspr-mdldSource=owsm+mdld2025.11 | 0.9674 | — | — | — | — | — | — | — | — | — | — | — | — | 92.55 | |
| whspr-nllbSource=owsm+mdld2025.11 | 0.966 | — | — | — | — | — | — | — | — | — | — | — | — | 92.22 | |
| smlss-mdldSource=whsprST2025.11 | 0.9616 | — | — | — | — | — | — | — | — | — | — | — | — | 91.79 | |
| smlss-mdldSource=owsm+mdld2025.11 | 0.9614 | — | — | — | — | — | — | — | — | — | — | — | — | 91.16 | |
| ASRsmlssEvaluation Metric=MetricX2025.11 | 0.9609 | 0.9693 | 0.9572 | 0.959 | 0.9806 | 0.9646 | 0.9349 | 93.42 | 92.35 | 90.11 | 95.56 | 92.82 | 88.77 | 92.06 | |
| smlss-nllbSource=whsprST2025.11 | 0.9607 | — | — | — | — | — | — | — | — | — | — | — | — | 91.6 | |
| smlss-nllbSource=owsm+mdld2025.11 | 0.9594 | — | — | — | — | — | — | — | — | — | — | — | — | 90.72 | |
| whspr-mdldSource=whsprST2025.11 | 0.9593 | — | — | — | — | — | — | — | — | — | — | — | — | 91.3 | |
| whspr-nllbSource=whsprST2025.11 | 0.9585 | — | — | — | — | — | — | — | — | — | — | — | — | 91.13 | |
| BTMdld*Source=owsm+mdld2025.11 | 0.9584 | — | — | — | — | — | — | — | — | — | — | — | — | 90.48 | |
| ASRwhsprEvaluation Metric=MetricX2025.11 | 0.9566 | 0.9576 | 0.9317 | 0.9545 | 0.9792 | 0.9661 | 0.9503 | 90.94 | 87.79 | 89.04 | 95.24 | 93.13 | 91.43 | 91.17 | |
| MetricX-ref*Source=owsm+mdld2025.11 | 0.9543 | — | — | — | — | — | — | — | — | — | — | — | — | — | |
| BTNllb*Source=owsm+mdld2025.11 | 0.9524 | — | — | — | — | — | — | — | — | — | — | — | — | 89.12 | |
| whspr-mdldSource=smlssST2025.11 | 0.9523 | — | — | — | — | — | — | — | — | — | — | — | — | 90.32 | |
| BTMdld*Source=whsprST2025.11 | 0.9523 | — | — | — | — | — | — | — | — | — | — | — | — | 89.8 | |
| whspr-nllbSource=smlssST2025.11 | 0.9507 | — | — | — | — | — | — | — | — | — | — | — | — | 89.98 | |
| BTNllb*Source=whsprST2025.11 | 0.9487 | — | — | — | — | — | — | — | — | — | — | — | — | 89.02 | |
| smlss-mdldSource=smlssST2025.11 | 0.9451 | — | — | — | — | — | — | — | — | — | — | — | — | 88.85 | |
| whspr-mdldSource=average2025.11 | 0.9448 | — | — | — | — | — | — | — | — | — | — | — | — | 88.78 | |
| whspr-mdldSource=owsmST2025.11 | 0.9442 | — | — | — | — | — | — | — | — | — | — | — | — | 86.54 | |
| whspr-nllbSource=owsmST2025.11 | 0.9433 | — | — | — | — | — | — | — | — | — | — | — | — | 86.33 | |
| smlss-nllbSource=smlssST2025.11 | 0.9429 | — | — | — | — | — | — | — | — | — | — | — | — | 88.41 | |
| whspr-nllbSource=average2025.11 | 0.9428 | — | — | — | — | — | — | — | — | — | — | — | — | 88.38 | |
| smlss-mdldSource=average2025.11 | 0.9395 | — | — | — | — | — | — | — | — | — | — | — | — | 87.71 | |
| smlss-mdldSource=owsmST2025.11 | 0.939 | — | — | — | — | — | — | — | — | — | — | — | — | 85.3 | |
| smlss-nllbSource=owsmST2025.11 | 0.9378 | — | — | — | — | — | — | — | — | — | — | — | — | 85 | |
| smlss-nllbSource=average2025.11 | 0.9372 | — | — | — | — | — | — | — | — | — | — | — | — | 87.24 | |
| BTMdld*Source=smlssST2025.11 | 0.9366 | — | — | — | — | — | — | — | — | — | — | — | — | 87.14 | |
| MetricX-ref*Source=smlssST2025.11 | 0.9327 | — | — | — | — | — | — | — | — | — | — | — | — | — | |
| BTNllb*Source=smlssST2025.11 | 0.9312 | — | — | — | — | — | — | — | — | — | — | — | — | 86.04 | |
| MetricX-ref*Source=whspr+mdld2025.11 | 0.9303 | — | — | — | — | — | — | — | — | — | — | — | — | — | |
| BTmdldEvaluation Metric=MetricX2025.11 | 0.9292 | 0.9523 | 0.9254 | 0.8843 | 0.9584 | 0.9366 | 0.918 | 89.8 | 86.66 | 72.11 | 90.48 | 87.14 | 85.86 | 85.6 | |
| BTMdld*Source=average2025.11 | 0.9292 | — | — | — | — | — | — | — | — | — | — | — | — | 85.6 | |
| owsm-mdldSource=owsm+mdld2025.11 | 0.9272 | — | — | — | — | — | — | — | — | — | — | — | — | 83.36 | |
| MetricX-ref*Source=whsprST2025.11 | 0.9271 | — | — | — | — | — | — | — | — | — | — | — | — | — | |
| MetricX-ref*Source=smlss+mdld2025.11 | 0.927 | — | — | — | — | — | — | — | — | — | — | — | — | — | |
| BTMdld*Source=whspr+mdld2025.11 | 0.9254 | — | — | — | — | — | — | — | — | — | — | — | — | 86.66 | |
| whspr-mdldSource=smlss+mdld2025.11 | 0.9248 | — | — | — | — | — | — | — | — | — | — | — | — | 87.04 | |
| owsm-nllbSource=owsm+mdld2025.11 | 0.9248 | — | — | — | — | — | — | — | — | — | — | — | — | 82.81 | |
| smlss-mdldSource=whspr+mdld2025.11 | 0.9234 | — | — | — | — | — | — | — | — | — | — | — | — | 86.31 | |
| BTnllbEvaluation Metric=MetricX2025.11 | 0.9227 | 0.9487 | 0.9153 | 0.8815 | 0.9524 | 0.9312 | 0.9074 | 89.02 | 84.86 | 71.43 | 89.12 | 86.04 | 84.03 | 84.3 | |
| BTNllb*Source=average2025.11 | 0.9227 | — | — | — | — | — | — | — | — | — | — | — | — | 84.3 | |
| ASRowsmEvaluation Metric=MetricX2025.11 | 0.9222 | 0.9311 | 0.9147 | 0.909 | 0.9326 | 0.9387 | 0.9069 | 85.26 | 84.75 | 78.07 | 84.59 | 87.56 | 83.95 | 84.18 | |
| owsm-mdldSource=whsprST2025.11 | 0.9216 | — | — | — | — | — | — | — | — | — | — | — | — | 83.24 | |
| whspr-nllbSource=smlss+mdld2025.11 | 0.9211 | — | — | — | — | — | — | — | — | — | — | — | — | 86.39 | |
| whspr-mdldSource=whspr+mdld2025.11 | 0.9207 | — | — | — | — | — | — | — | — | — | — | — | — | 85.83 | |
| smlss-nllbSource=whspr+mdld2025.11 | 0.9199 | — | — | — | — | — | — | — | — | — | — | — | — | 85.69 | |
| owsm-mdldSource=smlssST2025.11 | 0.9197 | — | — | — | — | — | — | — | — | — | — | — | — | 83.7 | |
| owsm-nllbSource=whsprST2025.11 | 0.9192 | — | — | — | — | — | — | — | — | — | — | — | — | 82.72 | |
| BTMdld*Source=smlss+mdld2025.11 | 0.918 | — | — | — | — | — | — | — | — | — | — | — | — | 85.86 | |
| whspr-nllbSource=whspr+mdld2025.11 | 0.9174 | — | — | — | — | — | — | — | — | — | — | — | — | 85.24 | |
| owsm-nllbSource=smlssST2025.11 | 0.9167 | — | — | — | — | — | — | — | — | — | — | — | — | 83.09 | |
| BTNllb*Source=whspr+mdld2025.11 | 0.9153 | — | — | — | — | — | — | — | — | — | — | — | — | 84.86 | |
| MetricX-refEvaluation Metric=MetricX2025.11 | 0.915 | 0.9271 | 0.9303 | 0.8187 | 0.9543 | 0.9327 | 0.927 | — | — | — | — | — | — | — | |
| MetricX-ref*Source=average2025.11 | 0.915 | — | — | — | — | — | — | — | — | — | — | — | — | — | |
| BTNllb*Source=smlss+mdld2025.11 | 0.9074 | — | — | — | — | — | — | — | — | — | — | — | — | 84.03 | |
| smlss-mdldSource=smlss+mdld2025.11 | 0.9067 | — | — | — | — | — | — | — | — | — | — | — | — | 83.9 | |
| owsm-mdldSource=average2025.11 | 0.9045 | — | — | — | — | — | — | — | — | — | — | — | — | 80.59 | |
| smlss-nllbSource=smlss+mdld2025.11 | 0.9025 | — | — | — | — | — | — | — | — | — | — | — | — | 83.18 | |
| owsm-nllbSource=average2025.11 | 0.9013 | — | — | — | — | — | — | — | — | — | — | — | — | 79.94 | |
| owsm-mdldSource=owsmST2025.11 | 0.8928 | — | — | — | — | — | — | — | — | — | — | — | — | 74.17 | |
| owsm-nllbSource=owsmST2025.11 | 0.8913 | — | — | — | — | — | — | — | — | — | — | — | — | 73.81 | |
| owsm-mdldSource=whspr+mdld2025.11 | 0.8894 | — | — | — | — | — | — | — | — | — | — | — | — | 80.23 | |
| owsm-nllbSource=whspr+mdld2025.11 | 0.8847 | — | — | — | — | — | — | — | — | — | — | — | — | 79.4 | |
| BTMdld*Source=owsmST2025.11 | 0.8843 | — | — | — | — | — | — | — | — | — | — | — | — | 72.11 | |
| BTNllb*Source=owsmST2025.11 | 0.8815 | — | — | — | — | — | — | — | — | — | — | — | — | 71.43 | |
| owsm-mdldSource=smlss+mdld2025.11 | 0.8762 | — | — | — | — | — | — | — | — | — | — | — | — | 78.65 | |
| owsm-nllbSource=smlss+mdld2025.11 | 0.8711 | — | — | — | — | — | — | — | — | — | — | — | — | 77.76 | |
| MetricX-ref*Source=owsmST2025.11 | 0.8187 | — | — | — | — | — | — | — | — | — | — | — | — | — | |
| shuff*Source=owsmST2025.11 | 0.5851 | — | — | — | — | — | — | — | — | — | — | — | — | — | |
| shuff*Source=owsm+mdld2025.11 | 0.5625 | — | — | — | — | — | — | — | — | — | — | — | — | — | |
| shuff*Source=whsprST2025.11 | 0.5325 | — | — | — | — | — | — | — | — | — | — | — | — | — | |
| shuffEvaluation Metric=MetricX2025.11 | 0.508 | 0.5325 | 0.4404 | 0.5851 | 0.5625 | 0.5075 | 0.4202 | — | — | — | — | — | — | — | |
| shuff*Source=average2025.11 | 0.508 | — | — | — | — | — | — | — | — | — | — | — | — | — | |
| shuff*Source=smlssST2025.11 | 0.5075 | — | — | — | — | — | — | — | — | — | — | — | — | — | |
| shuff*Source=whspr+mdld2025.11 | 0.4404 | — | — | — | — | — | — | — | — | — | — | — | — | — | |
| shuff*Source=smlss+mdld2025.11 | 0.4202 | — | — | — | — | — | — | — | — | — | — | — | — | — |