fix conversion of ## type bpe format to _ type (#23)

kensho-technologies · Aug 24, 2021 · a624a46 · a624a46
1 parent 50fd623
commit a624a46
Showing 1 changed file with 2 additions and 0 deletions.
diff --git a/pyctcdecode/alphabet.py b/pyctcdecode/alphabet.py
@@ -40,6 +40,8 @@ def _convert_bpe_format(token: str) -> str:
         return token
     elif token == "":  # nosec
         return token
+    elif token in ("<unk>", UNK_BPE_CHAR):
+        return token
     else:
         return BPE_CHAR + token