add ཞིང་ as clause boundary

OpenPecha · Jul 27, 2023 · 90bf27e · 90bf27e
1 parent fea72ac
commit 90bf27e
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/botok/tokenizers/sentencetokenizer.py b/botok/tokenizers/sentencetokenizer.py
@@ -33,7 +33,7 @@
     "ཏེ་",
     "དེ་",
 ]  # separated because these seem to cut long sentences
-clause_boundaries = te_particles + ["ནས་", "ན་", "ལ་"]
+clause_boundaries = te_particles + ["ནས་", "ན་", "ལ་", "ཞིང་"]
 dagdra = ["པ་", "བ་", "པོ་", "བོ་"]
 
 normalization_patterns = [(' <utt>', ''),