Fix Test

apaniukov · Apr 8, 2024 · fd220db · fd220db
1 parent 0298fea
commit fd220db
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/tests/tokenizers_test.py b/tests/tokenizers_test.py
@@ -498,7 +498,7 @@ def test_eos_token_id_rt_info_bpe(hf_bpe_tokenizers):
 
 
 def test_eos_token_id_rt_info_tiktoken(hf_tiktoken_tokenizers):
-    eos_token_id = hf_tiktoken_tokenizers.eos_token_id
+    eos_token_id = hf_tiktoken_tokenizers.eos_token_id or hf_tiktoken_tokenizers.eod_id
     ov_tokenizer, ov_detokenizer = convert_tokenizer(
         hf_tiktoken_tokenizers,
         with_detokenizer=True,