deleted a TODO

Algo-Boys · Sep 6, 2023 · cbbe597 · cbbe597
1 parent 9bb4ffb
commit cbbe597
Showing 1 changed file with 2 additions and 2 deletions.
diff --git a/swr2_asr/tokenizer.py b/swr2_asr/tokenizer.py
@@ -156,7 +156,7 @@ def decode(self, labels: list[int], remove_special_tokens: bool = True):
         """
         string = []
         for i in labels:
-            if remove_special_tokens and self.index_map[f"{i}"] == "<UNK>":
+            if remove_special_tokens and self.index_map[f"{i}"] == "<UNK>": 
                 continue
             if remove_special_tokens and self.index_map[f"{i}"] == "<SPACE>":
                 string.append(" ")
@@ -329,7 +329,7 @@ def train_bpe_tokenizer(
         "ü",
     ]
 
-    # TODO: add padding token / whitespace token / special tokens
+
     trainer = BpeTrainer(
         special_tokens=["[UNK]"],
         vocab_size=vocab_size,