Merge branch 'master' into releases/2024/4-master-update

CMakeLists.txt

@@ -38,7 +38,7 @@ else()
 endif()
 
 project(openvino_tokenizers
-        VERSION 2024.4.0.0
+        VERSION 2024.5.0.0
         DESCRIPTION "OpenVINO Tokenizers"
         HOMEPAGE_URL "https://github.com/openvinotoolkit/openvino_tokenizers"
         LANGUAGES CXX)
@@ -47,7 +47,11 @@ include(cmake/platforms.cmake)
 
 option(BUILD_CPP_EXTENSION "Builds C++ extension for OpenVINO Tokenizers" ON)
 
-if(DEFINED PY_BUILD_CMAKE_PACKAGE_VERSION AND NOT PY_BUILD_CMAKE_PACKAGE_VERSION EQUAL openvino_tokenizers_VERSION)
+if(NOT CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR)
+    set(is_subproject ON)
+endif()
+
+if(NOT is_subproject AND DEFINED PY_BUILD_CMAKE_PACKAGE_VERSION AND NOT PY_BUILD_CMAKE_PACKAGE_VERSION EQUAL openvino_tokenizers_VERSION)
     message(FATAL_ERROR "openvino_tokenizers_VERSION (${openvino_tokenizers_VERSION}) is not equal to PY_BUILD_CMAKE_PACKAGE_VERSION (${PY_BUILD_CMAKE_PACKAGE_VERSION})")
 endif()
 

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "openvino-tokenizers"
-version = "2024.4.0.0"
+version = "2024.5.0.0"
 description = "Convert tokenizers into OpenVINO models"
 requires-python = ">=3.8"
 readme = {file = "README.md", content-type="text/markdown"}
@@ -20,7 +20,7 @@ classifiers = [
 
 dependencies = [
     # support of nightly openvino packages with dev suffix
-    "openvino~=2024.4.0.0.dev"
+    "openvino~=2024.5.0.0.dev"
 ]
 
 [project.optional-dependencies]
@@ -106,6 +106,6 @@ python_abi = "none"
 requires = [
     "py-build-cmake@git+https://github.com/tttapa/py-build-cmake@7ab73da351c7140f06d727a8705bece4cf544cd9",
     "cmake~=3.15",
-    "openvino~=2024.4.0.0.dev"
+    "openvino~=2024.5.0.0.dev"
 ]
 build-backend = "py_build_cmake.build"

src/CMakeLists.txt

@@ -60,7 +60,7 @@ if(ANDROID)
     if(ANDROID_NATIVE_API_LEVEL LESS 33)
       message(FATAL_ERROR "FastTokenizers require ANDROID_NATIVE_API_LEVEL to be higher than 33. Please, either disable FastTokenizers or set ANDROID_NATIVE_API_LEVEL / ANDROID_PLATFORM")
     endif()
-  elseif(X86_64)
+  elseif(X86_64 OR RISCV64)
     message(WARNING "FastTokenizers are not available on ${CMAKE_SYSTEM_NAME} ${CMAKE_SYSTEM_PROCESSOR}. ENABLE_FAST_TOKENIZERS is set to OFF")
     set(FAST_TOKENIZERS_SUPPORTED OFF)
   else()

src/ragged_to_ragged.cpp

@@ -29,7 +29,7 @@ bool RaggedToRagged::evaluate(ov::TensorVector& outputs, const ov::TensorVector&
     auto rowids_size = static_cast<int32_t>(inputs[0].get_size());
     auto first_dim_size = inputs[1].data<const int32_t>();
 
-    const uint64_t batch_size = static_cast<uint64_t>(first_dim_size[0]);
+    const size_t batch_size = static_cast<size_t>(first_dim_size[0]);
     outputs[0].set_shape(ov::Shape{ batch_size });
     outputs[1].set_shape(ov::Shape{ batch_size });
 
@@ -41,7 +41,7 @@ bool RaggedToRagged::evaluate(ov::TensorVector& outputs, const ov::TensorVector&
     // prev_row_id stores row id for previous row
     int32_t prev_row_id = -1;
     for (int32_t rowids_idx = 0; rowids_idx < rowids_size; ++rowids_idx) {
-        int32_t curr_row_id = rowids[rowids_idx];
+        uint32_t curr_row_id = rowids[rowids_idx];
         OPENVINO_ASSERT(0 <= curr_row_id, "row id must be non-negative");
         if (curr_row_id >= batch_size) {
             break;

src/ragged_to_sparse.cpp

@@ -29,7 +29,7 @@ bool RaggedToSparse::evaluate(ov::TensorVector& outputs, const ov::TensorVector&
     auto ends   = inputs[1].data<const int32_t>();
 
     const auto last_element_index = inputs[1].get_size() - 1;
-    const uint64_t num_elements = static_cast<uint64_t>(ends[last_element_index] - begins[0]);
+    const size_t num_elements = static_cast<size_t>(ends[last_element_index] - begins[0]);
     outputs[0].set_shape(ov::Shape{num_elements, 2});
 
     auto batch_size = inputs[0].get_size();