Parse QWEN EOD_TOKEN_ID

(cherry picked from commit 029a341)
apaniukov · Apr 18, 2024 · 137964f · 137964f
1 parent 4891f99
commit 137964f
Show file tree

Hide file tree

Showing 2 changed files with 4 additions and 2 deletions.
diff --git a/python/openvino_tokenizers/hf_parser.py b/python/openvino_tokenizers/hf_parser.py
@@ -156,8 +156,8 @@ def parse(
             ),
         ]:
             add_steps()
+        self.pipeline.eos_token_id = pipeline.get_eos_token_id(self.original_tokenizer)
 
-        self.pipeline.eos_token_id = self.pipeline.get_eos_token_id(self.original_tokenizer)
         return self.pipeline
 
     normalizers_map: Dict[

diff --git a/tests/pass_rates.json b/tests/pass_rates.json
@@ -1,3 +1,5 @@
 {
-    "tests/tokenizers_test.py::test_": 0.8713874640310272
+    "tests/tokenizers_test.py::test_": 0.8713874640310272,
+    "tokenizers_test.py::test_": 0.8713874640310272,
+    "tokenizers_test.py::test_hf_bpe_tokenizers_outputs[MiniLM-L6-H384-uncased-sst2-": 0.8055555555555556
 }