From fd220db1d31be972d807d5bc6a86a32d6f762484 Mon Sep 17 00:00:00 2001 From: Artur Paniukov Date: Mon, 8 Apr 2024 19:46:48 +0100 Subject: [PATCH] Fix Test --- tests/tokenizers_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/tokenizers_test.py b/tests/tokenizers_test.py index d5090240d..df683619a 100644 --- a/tests/tokenizers_test.py +++ b/tests/tokenizers_test.py @@ -498,7 +498,7 @@ def test_eos_token_id_rt_info_bpe(hf_bpe_tokenizers): def test_eos_token_id_rt_info_tiktoken(hf_tiktoken_tokenizers): - eos_token_id = hf_tiktoken_tokenizers.eos_token_id + eos_token_id = hf_tiktoken_tokenizers.eos_token_id or hf_tiktoken_tokenizers.eod_id ov_tokenizer, ov_detokenizer = convert_tokenizer( hf_tiktoken_tokenizers, with_detokenizer=True,