duplicates.yaml

# Sentences that appear in the CCMatrix dataset that also occur in the respective test set. All the duplicated indexes were found
# by using the dedup.py script inside the scripts' directory.

# Duplicated wmt14 english sentences found in the en-de ccmatrix dataset
duplicates_en_de: [
  2801169, 3015352, 19775415, 20367662, 20785493, 23611708, 25969951,32771958, 33590564, 33799669, 38165983, 38349415,
  42732422, 45639868, 46533951, 48154585, 51122630, 52569871, 53253769, 53605070, 55897441, 56864670, 66495923,
  67445650, 72252425, 72300952, 72964017, 73662839, 76210766, 78842479, 78842480, 78842481, 82414911, 83162203,
  83162204, 83275757, 89471839, 92184954, 92184955, 105087545, 110218143, 115819739, 123521912, 130686046, 141563952,
  146476551, 147853628, 154125145, 155297361, 161224397, 175150743, 182672297, 185766813, 193096473, 202301926,
  202497244, 205450756, 208256156, 219628486, 227123665
]

# Duplicated wmt14 german sentences found in the en-de ccmatrix dataset
duplicates_de_en: [
  27922110, 31580101, 32771958, 40763567, 45639868, 63058682, 72964017, 73662839, 78842480, 79531236, 83162203,
  83238534, 92184955, 105831028, 111265110, 145839309, 164197978, 185766813, 188391705, 198131661, 198435949, 199964247,
  230948265, 240673484, 245361151, 246664703
]

# Duplicated wmt14 english sentences found in the en-fr ccmatrix dataset
duplicates_en_fr: [
  2372581, 6968567, 10821748, 11060884, 15767927, 25424386, 29725453, 45747545, 47137798, 50129051, 59177023, 59929203,
  61511560, 75542580, 100970169, 115986518, 127680776, 141459031, 156717917, 157018533, 162558439, 164150364, 175041176,
  184342700, 190148649, 190148650, 192658445, 220362372, 245452855, 256201123, 271393589, 272871204, 272877704,
  281597372, 294584774, 296244867, 321887045
]

# Duplicated wmt14 french sentences found in the en-fr ccmatrix dataset
duplicates_fr_en: [
  35870050, 48145532, 52684654, 58751416, 58882125, 65601877, 67930837, 77241694, 92977227, 110216804, 128101180,
  134271264, 141335940, 163685146, 170148774, 174846035, 175041176, 178472316, 187909576, 190148650, 190788599,
  199867191, 202841440, 203367259, 216538756, 216971114, 217029239, 217343772, 221404922, 228346708, 229954946,
  236475781, 238254814, 239313560, 240741477, 244758333, 244990634, 246716684, 246848239, 251633313, 252437626,
  258612355, 260023316, 261848203, 266413071, 269838607, 271039088, 280243425, 290825579, 303987750, 304028810,
  310067703, 310183397, 314725258, 323880921, 324665884
]

# Duplicated wmt13 english sentences found in the en-es ccmatrix dataset
duplicates_en_es: [104162920, 201875059, 220054035, 239139109, 242048881, 250889199, 299111881, 300507236, 357223245]

# Duplicated wmt13 spanish sentences found in the en-es ccmatrix dataset
duplicates_es_en: [
  6751585, 7725692, 35698791, 43267448, 48075653, 48925282, 48925283, 53971646, 55073237, 56061226, 59861190, 63114112,
  80366657, 80902620, 86295984, 87272571, 111795284, 113916617, 118570344, 119519063, 120600774, 129587251, 130850605,
  136272300, 136519043, 136733349, 137378072, 137491231, 137665670, 143721844, 145405838, 146284528, 146621153,
  153890220, 155856369, 159312479, 160583224, 163400236, 167062018, 167230158, 168709714, 171236351, 173926019,
  175953391, 180093066, 181892264, 188671101, 190817212, 191197133, 191440526, 193089123, 193231113, 202625957,
  218523240, 219695892, 225112141, 227156035, 231918342, 233796239, 233865889, 237259151, 274202045, 278337171,
  288838952, 289266873, 290450151, 291301513, 293807105, 294314852, 296624934, 297053788, 303558652, 317079342,
  318616807, 322993811, 326540772, 329582867, 341969381, 349023337, 352305431, 354125230, 356161656, 357435039,
  364171762, 378343556, 380637241, 393342927, 398415139, 399539622, 404908528, 407859230
]