Factual Consistency Evaluation on TRUE 1.0 (test)
91.5Frank AUCEour
Evaluation results
| Method | Links | |||||||||||||||||
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| Eourmode=ensemble2023.05 | 91.5 | 76.4 | 83.9 | 83.8 | 86.7 | 84.7 | 94.5 | 88.5 | 92.3 | 86.8 | — | — | — | — | — | — | — | |
| Eorigmode=ensemble2023.05 | 91.2 | 76.6 | 82.9 | 84.8 | 87.7 | 86.2 | 90.4 | 88.8 | 91.2 | 86 | — | — | — | — | — | — | — | |
| T5 ANLIbackbone=T5, pretraining=ANLI2023.05 | 89.4 | 77.9 | 80.5 | 82.9 | 82.1 | 82.6 | 72.7 | 79.8 | 86.4 | 81.5 | — | — | — | — | — | — | — | |
| ANLIParameters=11B2022.12 | 89.4 | 77.9 | 80.5 | — | — | 82.6 | 77.7 | 72.7 | 86.4 | 81.5 | 82.1 | 83.8 | 82.5 | 77.7 | 24.9 | — | — | |
| Honovich et al., 20222022.12 | 89.4 | 77.9 | 80.5 | 83.8 | 82.1 | 82.6 | 77.7 | 72.7 | 86.4 | 83.1 | — | — | — | — | — | 93.2 | 88.3 | |
| SummacZSmode=zero-shot2023.05 | 89.1 | 71.3 | 81.7 | 78.1 | 80.9 | 82 | 84.1 | 77.4 | 88.2 | 81.4 | — | — | — | — | — | — | — | |
| SCZS2022.12 | 88.9 | 71.1 | 81.3 | — | — | 82 | 84.1 | 77.4 | 88.2 | 81.4 | 80.9 | 78.1 | 80.1 | 81.2 | 30.4 | — | — | |
| WeCheckParameters=435M, Training Mode=Weak supervision2022.12 | 88.1 | 83 | 79.8 | — | — | 84.6 | 90 | 84 | 89.6 | 84.8 | 82.6 | 81.4 | 83 | 86.2 | 13.2 | — | — | |
| Ours2022.12 | 88.1 | 77.2 | 75.6 | 84.5 | 82.8 | 80.5 | 80.1 | 72.8 | 82.6 | 82.4 | — | — | — | — | — | 93.7 | 88.2 | |
| Q22023.05 | 87.8 | 68.7 | 78.8 | 70.9 | 83.5 | 79.7 | 86.1 | 80.9 | 89.7 | 80.7 | — | — | — | — | — | — | — | |
| QAFact2022.12 | 87.8 | 68.7 | 77.4 | — | — | 76.3 | 84.5 | 80.4 | 85 | 80 | 83.3 | 76.9 | 78.8 | 80.4 | 34.4 | — | — | |
| Q^2Parameters=11B2022.12 | 87.8 | 68.7 | 78.8 | — | — | 79.7 | 86.1 | 80.9 | 89.7 | 80.7 | 83.5 | 70.9 | 77.9 | 82.2 | 51.6 | — | — | |
| AllMonte-Carlo Dropout=true, Data Augmentation=true2023.05 | 87.7 | 74.5 | 76.1 | 81.1 | 78 | 79.3 | 91.3 | 82.8 | 90 | 83.2 | — | — | — | — | — | — | — | |
| -MCMonte-Carlo Dropout=false, Data Augmentation=true2023.05 | 86.6 | 73.5 | 75.2 | 81.6 | 78.4 | 78.5 | 91.8 | 81.8 | 89 | 82.7 | — | — | — | — | — | — | — | |
| BARTS2022.12 | 86.1 | 60.9 | 73.5 | — | — | 86.3 | 65.6 | 64.9 | 77.5 | 72.2 | 80.9 | 53.8 | 71 | 72.3 | 13.2 | — | — | |
| NLI-warmupParameters=435M, Training Mode=NLI warmup2022.12 | 85.7 | 73.5 | 73.7 | — | — | 80.5 | 87.3 | 83.5 | 85.4 | 80.3 | 73.2 | 80.1 | 77.2 | 83.8 | 31.8 | — | — | |
| Basebackbone=T5-base, data_augmentation=false2023.05 | 85.6 | 74.6 | 72.8 | 83 | 74.1 | 80.4 | 77.7 | 79.8 | 87.8 | 79.8 | — | — | — | — | — | — | — | |
| BERTS2022.12 | 84.3 | 62.8 | 77.2 | — | — | 87.9 | 64.2 | 70 | 77.5 | 71.4 | 69.1 | 49.5 | 68.6 | 74 | 14 | — | — | |
| QuestEval2022.12 | 84 | 65.3 | 70.1 | — | — | 84.1 | 77.3 | 72.2 | 77.3 | 71.4 | 64.2 | 56.3 | 68 | 77.9 | 87.7 | — | — | |
| FactCC2022.12 | 76.4 | 59.4 | 75.9 | — | — | 64.4 | 55.3 | 63.7 | 64 | 66.7 | 76.4 | 64.9 | 70.6 | 61.1 | 60.1 | — | — |