improve support for news articles

OpenPecha · Jul 20, 2023 · fea72ac · fea72ac
1 parent 5051708
commit fea72ac
Showing 1 changed file with 3 additions and 2 deletions.
diff --git a/botok/tokenizers/sentencetokenizer.py b/botok/tokenizers/sentencetokenizer.py
@@ -25,14 +25,15 @@
     "བགྱི་",
     "བྱ་",
     "བཞུགས་",
-    "འདུག",
+    "འདུག་",
+    "སོང་",
 ]
 te_particles = [
     "སྟེ་",
     "ཏེ་",
     "དེ་",
 ]  # separated because these seem to cut long sentences
-clause_boundaries = te_particles + ["ནས་", "ན་"]
+clause_boundaries = te_particles + ["ནས་", "ན་", "ལ་"]
 dagdra = ["པ་", "བ་", "པོ་", "བོ་"]
 
 normalization_patterns = [(' <utt>', ''),