From 9ad7cff22280d34f60973bf8f5fd7dadcb204758 Mon Sep 17 00:00:00 2001 From: p-wysocki Date: Fri, 23 Aug 2024 12:01:45 +0200 Subject: [PATCH 01/24] Initial commit --- .../python/src/pyopenvino/core/core.cpp | 114 +++++++++++++----- .../tests/test_runtime/test_compiled_model.py | 21 +++- 2 files changed, 101 insertions(+), 34 deletions(-) diff --git a/src/bindings/python/src/pyopenvino/core/core.cpp b/src/bindings/python/src/pyopenvino/core/core.cpp index 6cf405cd167423..a27ec3fef8a9c4 100644 --- a/src/bindings/python/src/pyopenvino/core/core.cpp +++ b/src/bindings/python/src/pyopenvino/core/core.cpp @@ -503,10 +503,46 @@ void regclass_Core(py::module m) { const std::string& device_name, const std::map& properties) { auto _properties = Common::utils::properties_to_any_map(properties); - py::gil_scoped_release release; - std::stringstream _stream; - _stream << model_stream; - return self.import_model(_stream, device_name, _properties); + py::object model_stream_buffer = model_stream.attr("getbuffer")(); + py::buffer buffer = py::buffer(model_stream_buffer); + py::buffer_info info = buffer.request(); + const double size_in_gb = (info.size * info.itemsize) / 1073741824.0; + model_stream.attr("seek")(0); // Always rewind stream! + ov::CompiledModel result; + // std::stringstream cannot handle big streams, in that case we use std::fstream + if (size_in_gb > 2) { + std::random_device rd; + std::mt19937 gen(rd()); + std::uniform_int_distribution<> distr(1000, 9999); + std::string filename = "model_stream_" + std::to_string(distr(gen)) + ".txt"; + std::fstream _stream(filename, std::ios::out | std::ios::binary); + if (_stream.is_open()) { + _stream.write(model_stream.data(), model_stream.size()); + _stream.close(); + } else { + OPENVINO_THROW("Failed to open temporary file for model stream"); + } + std::fstream _fstream(filename, std::ios::in | std::ios::binary); + if (_fstream.is_open()) { + py::gil_scoped_release release; + result = self.import_model(_fstream, device_name, _properties); + _fstream.close(); + if (std::remove(filename.c_str()) != 0) { + const std::string abs_path = + py::module_::import("os").attr("getcwd")().cast() + "/" + filename; + const std::string warning_message = "Temporary file " + abs_path + " failed to delete!"; + PyErr_WarnEx(PyExc_RuntimeWarning, warning_message.c_str(), 1); + } + } else { + OPENVINO_THROW("Failed to open temporary file for model stream"); + } + } else { + py::gil_scoped_release release; + std::stringstream _stream; + _stream << model_stream; + result = self.import_model(_stream, device_name, _properties); + } + return result; }, py::arg("model_stream"), py::arg("device_name"), @@ -552,40 +588,52 @@ void regclass_Core(py::module m) { "`model_stream` must be an io.BytesIO object but " + (std::string)(py::repr(model_stream)) + "` provided"); } - std::random_device rd; - std::mt19937 gen(rd()); - std::uniform_int_distribution<> distr(1000, 9999); - std::string filename = "model_stream_" + std::to_string(distr(gen)) + ".txt"; - std::fstream _stream(filename, std::ios::out | std::ios::binary); + py::object model_stream_buffer = model_stream.attr("getbuffer")(); + py::buffer buffer = py::buffer(model_stream_buffer); + py::buffer_info info = buffer.request(); + const double size_in_gb = (info.size * info.itemsize) / 1073741824.0; model_stream.attr("seek")(0); // Always rewind stream! - if (_stream.is_open()) { - const py::bytes data = model_stream.attr("read")(); - // convert the Python bytes object to C++ string - char* buffer; - Py_ssize_t length; - PYBIND11_BYTES_AS_STRING_AND_SIZE(data.ptr(), &buffer, &length); - _stream.write(buffer, length); - _stream.close(); - } else { - OPENVINO_THROW("Failed to open temporary file for model stream"); - } - ov::CompiledModel result; - std::fstream _fstream(filename, std::ios::in | std::ios::binary); - if (_fstream.is_open()) { - py::gil_scoped_release release; - result = self.import_model(_fstream, device_name, _properties); - _fstream.close(); - if (std::remove(filename.c_str()) != 0) { - const std::string abs_path = - py::module_::import("os").attr("getcwd")().cast() + "/" + filename; - const std::string warning_message = "Temporary file " + abs_path + " failed to delete!"; - PyErr_WarnEx(PyExc_RuntimeWarning, warning_message.c_str(), 1); + // std::stringstream cannot handle big streams, in that case we use std::fstream + if (size_in_gb > 2) { + std::random_device rd; + std::mt19937 gen(rd()); + std::uniform_int_distribution<> distr(1000, 9999); + std::string filename = "model_stream_" + std::to_string(distr(gen)) + ".txt"; + std::fstream _stream(filename, std::ios::out | std::ios::binary); + if (_stream.is_open()) { + const py::bytes data = model_stream.attr("read")(); + // convert the Python bytes object to C++ string + char* buffer; + Py_ssize_t length; + PYBIND11_BYTES_AS_STRING_AND_SIZE(data.ptr(), &buffer, &length); + _stream.write(buffer, length); + _stream.close(); + } else { + OPENVINO_THROW("Failed to open temporary file for model stream"); + } + std::fstream _fstream(filename, std::ios::in | std::ios::binary); + if (_fstream.is_open()) { + py::gil_scoped_release release; + result = self.import_model(_fstream, device_name, _properties); + _fstream.close(); + if (std::remove(filename.c_str()) != 0) { + const std::string abs_path = + py::module_::import("os").attr("getcwd")().cast() + "/" + filename; + const std::string warning_message = "Temporary file " + abs_path + " failed to delete!"; + PyErr_WarnEx(PyExc_RuntimeWarning, warning_message.c_str(), 1); + } + } else { + OPENVINO_THROW("Failed to open temporary file for model stream"); } } else { - OPENVINO_THROW("Failed to open temporary file for model stream"); + std::stringstream _stream; + _stream << model_stream + .attr("read")() + .cast(); + py::gil_scoped_release release; + result = self.import_model(_stream, device_name, _properties); } - return result; }, py::arg("model_stream"), diff --git a/src/bindings/python/tests/test_runtime/test_compiled_model.py b/src/bindings/python/tests/test_runtime/test_compiled_model.py index 3c71b09323f0f5..55789ba90cba5d 100644 --- a/src/bindings/python/tests/test_runtime/test_compiled_model.py +++ b/src/bindings/python/tests/test_runtime/test_compiled_model.py @@ -11,8 +11,9 @@ generate_image, generate_model_and_image, generate_relu_compiled_model, + generate_model_with_memory, create_filename_for_test) -from openvino import Model, Shape, Core, Tensor, serialize +from openvino import Model, Shape, Core, Tensor, serialize, Type from openvino.runtime import ConstOutput import openvino.properties as props @@ -74,6 +75,24 @@ def test_export_import_advanced(device): res = new_compiled.infer_new_request({"data": img}) assert np.argmax(res[new_compiled.outputs[0]]) == 531 + + +def test_export_import_large_model(device): + import io + core = Core() + if props.device.Capability.EXPORT_IMPORT not in core.get_property(device, props.device.capabilities): + pytest.skip(f"{core.get_property(device, props.device.full_name)} plugin due-to export, import model API isn't implemented.") + + # model of size of roughly 2.2GB + model = generate_model_with_memory([6, 10000, 10000], Type.f32) + core = Core() + compiled_model = core.compile_model(model, device, {}) + user_stream = io.BytesIO() + compiled_model.export_model(user_stream) + new_compiled = core.import_model(user_stream, device) + img = generate_image([6, 10000, 10000]) + res = new_compiled.infer_new_request({"input_data": img}) + assert np.argmax(res[new_compiled.outputs[0]]) == 44372822 @pytest.mark.parametrize("input_arguments", [[0], ["data"], []]) From 4ee2e307fdc7d565bdd96b390078677107c12210 Mon Sep 17 00:00:00 2001 From: p-wysocki Date: Mon, 26 Aug 2024 11:14:47 +0200 Subject: [PATCH 02/24] Checkpoint --- .../src/pyopenvino/core/compiled_model.cpp | 50 ++++++++++++++++- .../python/src/pyopenvino/core/core.cpp | 53 +++---------------- .../tests/test_runtime/test_compiled_model.py | 21 ++++++++ 3 files changed, 78 insertions(+), 46 deletions(-) diff --git a/src/bindings/python/src/pyopenvino/core/compiled_model.cpp b/src/bindings/python/src/pyopenvino/core/compiled_model.cpp index 1cbf33e11a9c50..0cfca550c85d4b 100644 --- a/src/bindings/python/src/pyopenvino/core/compiled_model.cpp +++ b/src/bindings/python/src/pyopenvino/core/compiled_model.cpp @@ -4,6 +4,9 @@ #include "openvino/runtime/compiled_model.hpp" +#include +#include + #include #include @@ -79,13 +82,58 @@ void regclass_CompiledModel(py::module m) { "`model_stream` must be an io.BytesIO object but " + (std::string)(py::repr(model_stream)) + "` provided"); } - std::stringstream _stream; + py::object model_stream_buffer = model_stream.attr("getbuffer")(); + py::buffer buffer = py::buffer(model_stream_buffer); + py::buffer_info info = buffer.request(); + constexpr auto one_gigabyte = static_cast(1024) * 1024 * 1024; + const double size_in_gb = (info.size * info.itemsize) / one_gigabyte; + model_stream.attr("seek")(0); // Always rewind stream! + // std::stringstream cannot handle streams > 2GB, in that case we use std::fstream + if (size_in_gb > 2) { + std::random_device rd; + std::mt19937 gen(rd()); + std::uniform_int_distribution<> distr(1000, 9999); + std::string filename = "model_stream_" + std::to_string(distr(gen)) + ".txt"; + std::fstream _stream(filename, std::ios::out | std::ios::binary); + if (_stream.is_open()) { + const py::bytes data = model_stream.attr("read")(); + // convert the Python bytes object to C++ string + char* buffer; + Py_ssize_t length; + PYBIND11_BYTES_AS_STRING_AND_SIZE(data.ptr(), &buffer, &length); + _stream.write(buffer, length); + _stream.close(); + } else { + OPENVINO_THROW("Failed to open temporary file for model stream"); + } + std::fstream _fstream(filename, std::ios::in | std::ios::binary); + if (_fstream.is_open()) { + py::gil_scoped_release release; + self.export_model(_fstream); + _fstream.seekg(0, std::ios::beg); + std::string str((std::istreambuf_iterator(_fstream)), std::istreambuf_iterator()); + _fstream.close(); + model_stream.attr("flush")(); + model_stream.attr("write")(py::bytes(str)); + _fstream.close(); + if (std::remove(filename.c_str()) != 0) { + const std::string abs_path = + py::module_::import("os").attr("getcwd")().cast() + "/" + filename; + const std::string warning_message = "Temporary file " + abs_path + " failed to delete!"; + PyErr_WarnEx(PyExc_RuntimeWarning, warning_message.c_str(), 1); + } + } else { + OPENVINO_THROW("Failed to open temporary file for model stream"); + } + } else { + std::stringstream _stream; { py::gil_scoped_release release; self.export_model(_stream); } model_stream.attr("flush")(); model_stream.attr("write")(py::bytes(_stream.str())); + } model_stream.attr("seek")(0); // Always rewind stream! }, py::arg("model_stream"), diff --git a/src/bindings/python/src/pyopenvino/core/core.cpp b/src/bindings/python/src/pyopenvino/core/core.cpp index a27ec3fef8a9c4..f8e89ddad67da6 100644 --- a/src/bindings/python/src/pyopenvino/core/core.cpp +++ b/src/bindings/python/src/pyopenvino/core/core.cpp @@ -503,46 +503,10 @@ void regclass_Core(py::module m) { const std::string& device_name, const std::map& properties) { auto _properties = Common::utils::properties_to_any_map(properties); - py::object model_stream_buffer = model_stream.attr("getbuffer")(); - py::buffer buffer = py::buffer(model_stream_buffer); - py::buffer_info info = buffer.request(); - const double size_in_gb = (info.size * info.itemsize) / 1073741824.0; - model_stream.attr("seek")(0); // Always rewind stream! - ov::CompiledModel result; - // std::stringstream cannot handle big streams, in that case we use std::fstream - if (size_in_gb > 2) { - std::random_device rd; - std::mt19937 gen(rd()); - std::uniform_int_distribution<> distr(1000, 9999); - std::string filename = "model_stream_" + std::to_string(distr(gen)) + ".txt"; - std::fstream _stream(filename, std::ios::out | std::ios::binary); - if (_stream.is_open()) { - _stream.write(model_stream.data(), model_stream.size()); - _stream.close(); - } else { - OPENVINO_THROW("Failed to open temporary file for model stream"); - } - std::fstream _fstream(filename, std::ios::in | std::ios::binary); - if (_fstream.is_open()) { - py::gil_scoped_release release; - result = self.import_model(_fstream, device_name, _properties); - _fstream.close(); - if (std::remove(filename.c_str()) != 0) { - const std::string abs_path = - py::module_::import("os").attr("getcwd")().cast() + "/" + filename; - const std::string warning_message = "Temporary file " + abs_path + " failed to delete!"; - PyErr_WarnEx(PyExc_RuntimeWarning, warning_message.c_str(), 1); - } - } else { - OPENVINO_THROW("Failed to open temporary file for model stream"); - } - } else { - py::gil_scoped_release release; - std::stringstream _stream; - _stream << model_stream; - result = self.import_model(_stream, device_name, _properties); - } - return result; + py::gil_scoped_release release; + std::stringstream _stream; + _stream << model_stream; + return self.import_model(_stream, device_name, _properties); }, py::arg("model_stream"), py::arg("device_name"), @@ -588,13 +552,12 @@ void regclass_Core(py::module m) { "`model_stream` must be an io.BytesIO object but " + (std::string)(py::repr(model_stream)) + "` provided"); } - py::object model_stream_buffer = model_stream.attr("getbuffer")(); - py::buffer buffer = py::buffer(model_stream_buffer); - py::buffer_info info = buffer.request(); - const double size_in_gb = (info.size * info.itemsize) / 1073741824.0; + py::buffer_info info = py::buffer(model_stream.attr("getbuffer")()).request(); + constexpr auto one_gigabyte = static_cast(1024) * 1024 * 1024; + const double size_in_gb = (info.size * info.itemsize) / one_gigabyte; model_stream.attr("seek")(0); // Always rewind stream! ov::CompiledModel result; - // std::stringstream cannot handle big streams, in that case we use std::fstream + // std::stringstream cannot handle streams > 2GB, in that case use std::fstream if (size_in_gb > 2) { std::random_device rd; std::mt19937 gen(rd()); diff --git a/src/bindings/python/tests/test_runtime/test_compiled_model.py b/src/bindings/python/tests/test_runtime/test_compiled_model.py index 55789ba90cba5d..708c18b73698e3 100644 --- a/src/bindings/python/tests/test_runtime/test_compiled_model.py +++ b/src/bindings/python/tests/test_runtime/test_compiled_model.py @@ -4,6 +4,7 @@ import sys import pytest +import random import numpy as np from tests.utils.helpers import ( @@ -95,6 +96,26 @@ def test_export_import_large_model(device): assert np.argmax(res[new_compiled.outputs[0]]) == 44372822 +def test_import_large_model_with_string_input(device): + import io + core = Core() + if props.device.Capability.EXPORT_IMPORT not in core.get_property(device, props.device.capabilities): + pytest.skip(f"{core.get_property(device, props.device.full_name)} plugin due-to export, import model API isn't implemented.") + + # model of size of roughly 2.2GB + model = generate_model_with_memory([6, 10000, 10000], Type.f32) + core = Core() + compiled_model = core.compile_model(model, device, {}) + user_stream = io.BytesIO() + compiled_model.export_model(user_stream) + user_stream.seek(0) + + new_compiled = core.import_model(user_stream.read().decode('utf-8'), "CPU") + img = generate_image([6, 10000, 10000]) + res = new_compiled.infer_new_request({"input_data": img}) + assert np.argmax(res[new_compiled.outputs[0]]) == 44372822 + + @pytest.mark.parametrize("input_arguments", [[0], ["data"], []]) def test_get_input(device, input_arguments): compiled_model = generate_relu_compiled_model(device) From 19fc4c9685e8f5397d9ae57fce1e5619e1f72d59 Mon Sep 17 00:00:00 2001 From: p-wysocki Date: Mon, 26 Aug 2024 15:26:05 +0200 Subject: [PATCH 03/24] Minor change --- .../src/pyopenvino/core/compiled_model.cpp | 50 +------------------ 1 file changed, 1 insertion(+), 49 deletions(-) diff --git a/src/bindings/python/src/pyopenvino/core/compiled_model.cpp b/src/bindings/python/src/pyopenvino/core/compiled_model.cpp index 0cfca550c85d4b..1cbf33e11a9c50 100644 --- a/src/bindings/python/src/pyopenvino/core/compiled_model.cpp +++ b/src/bindings/python/src/pyopenvino/core/compiled_model.cpp @@ -4,9 +4,6 @@ #include "openvino/runtime/compiled_model.hpp" -#include -#include - #include #include @@ -82,58 +79,13 @@ void regclass_CompiledModel(py::module m) { "`model_stream` must be an io.BytesIO object but " + (std::string)(py::repr(model_stream)) + "` provided"); } - py::object model_stream_buffer = model_stream.attr("getbuffer")(); - py::buffer buffer = py::buffer(model_stream_buffer); - py::buffer_info info = buffer.request(); - constexpr auto one_gigabyte = static_cast(1024) * 1024 * 1024; - const double size_in_gb = (info.size * info.itemsize) / one_gigabyte; - model_stream.attr("seek")(0); // Always rewind stream! - // std::stringstream cannot handle streams > 2GB, in that case we use std::fstream - if (size_in_gb > 2) { - std::random_device rd; - std::mt19937 gen(rd()); - std::uniform_int_distribution<> distr(1000, 9999); - std::string filename = "model_stream_" + std::to_string(distr(gen)) + ".txt"; - std::fstream _stream(filename, std::ios::out | std::ios::binary); - if (_stream.is_open()) { - const py::bytes data = model_stream.attr("read")(); - // convert the Python bytes object to C++ string - char* buffer; - Py_ssize_t length; - PYBIND11_BYTES_AS_STRING_AND_SIZE(data.ptr(), &buffer, &length); - _stream.write(buffer, length); - _stream.close(); - } else { - OPENVINO_THROW("Failed to open temporary file for model stream"); - } - std::fstream _fstream(filename, std::ios::in | std::ios::binary); - if (_fstream.is_open()) { - py::gil_scoped_release release; - self.export_model(_fstream); - _fstream.seekg(0, std::ios::beg); - std::string str((std::istreambuf_iterator(_fstream)), std::istreambuf_iterator()); - _fstream.close(); - model_stream.attr("flush")(); - model_stream.attr("write")(py::bytes(str)); - _fstream.close(); - if (std::remove(filename.c_str()) != 0) { - const std::string abs_path = - py::module_::import("os").attr("getcwd")().cast() + "/" + filename; - const std::string warning_message = "Temporary file " + abs_path + " failed to delete!"; - PyErr_WarnEx(PyExc_RuntimeWarning, warning_message.c_str(), 1); - } - } else { - OPENVINO_THROW("Failed to open temporary file for model stream"); - } - } else { - std::stringstream _stream; + std::stringstream _stream; { py::gil_scoped_release release; self.export_model(_stream); } model_stream.attr("flush")(); model_stream.attr("write")(py::bytes(_stream.str())); - } model_stream.attr("seek")(0); // Always rewind stream! }, py::arg("model_stream"), From d84cea65cc2ef0c99ed5e1a7cab84cd075bff9f7 Mon Sep 17 00:00:00 2001 From: p-wysocki Date: Mon, 26 Aug 2024 15:27:12 +0200 Subject: [PATCH 04/24] Minor change --- .../tests/test_runtime/test_compiled_model.py | 20 ------------------- 1 file changed, 20 deletions(-) diff --git a/src/bindings/python/tests/test_runtime/test_compiled_model.py b/src/bindings/python/tests/test_runtime/test_compiled_model.py index 708c18b73698e3..9ec255048ba89e 100644 --- a/src/bindings/python/tests/test_runtime/test_compiled_model.py +++ b/src/bindings/python/tests/test_runtime/test_compiled_model.py @@ -96,26 +96,6 @@ def test_export_import_large_model(device): assert np.argmax(res[new_compiled.outputs[0]]) == 44372822 -def test_import_large_model_with_string_input(device): - import io - core = Core() - if props.device.Capability.EXPORT_IMPORT not in core.get_property(device, props.device.capabilities): - pytest.skip(f"{core.get_property(device, props.device.full_name)} plugin due-to export, import model API isn't implemented.") - - # model of size of roughly 2.2GB - model = generate_model_with_memory([6, 10000, 10000], Type.f32) - core = Core() - compiled_model = core.compile_model(model, device, {}) - user_stream = io.BytesIO() - compiled_model.export_model(user_stream) - user_stream.seek(0) - - new_compiled = core.import_model(user_stream.read().decode('utf-8'), "CPU") - img = generate_image([6, 10000, 10000]) - res = new_compiled.infer_new_request({"input_data": img}) - assert np.argmax(res[new_compiled.outputs[0]]) == 44372822 - - @pytest.mark.parametrize("input_arguments", [[0], ["data"], []]) def test_get_input(device, input_arguments): compiled_model = generate_relu_compiled_model(device) From 9483d3b5f4640c11c18ae9c3040cfb45fb0bdf9f Mon Sep 17 00:00:00 2001 From: p-wysocki Date: Mon, 26 Aug 2024 15:28:23 +0200 Subject: [PATCH 05/24] Clang --- src/bindings/python/src/pyopenvino/core/core.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/bindings/python/src/pyopenvino/core/core.cpp b/src/bindings/python/src/pyopenvino/core/core.cpp index f8e89ddad67da6..2cd9736c0b6c47 100644 --- a/src/bindings/python/src/pyopenvino/core/core.cpp +++ b/src/bindings/python/src/pyopenvino/core/core.cpp @@ -591,9 +591,7 @@ void regclass_Core(py::module m) { } } else { std::stringstream _stream; - _stream << model_stream - .attr("read")() - .cast(); + _stream << model_stream.attr("read")().cast(); py::gil_scoped_release release; result = self.import_model(_stream, device_name, _properties); } From 212d46b963cd6ef831b079c7cf4bc4811f7d3d81 Mon Sep 17 00:00:00 2001 From: p-wysocki Date: Mon, 26 Aug 2024 15:29:33 +0200 Subject: [PATCH 06/24] Cleanup --- src/bindings/python/tests/test_runtime/test_compiled_model.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/bindings/python/tests/test_runtime/test_compiled_model.py b/src/bindings/python/tests/test_runtime/test_compiled_model.py index 9ec255048ba89e..55789ba90cba5d 100644 --- a/src/bindings/python/tests/test_runtime/test_compiled_model.py +++ b/src/bindings/python/tests/test_runtime/test_compiled_model.py @@ -4,7 +4,6 @@ import sys import pytest -import random import numpy as np from tests.utils.helpers import ( From 0c511502295c75a8e07bec3e0bffb8fc237795fa Mon Sep 17 00:00:00 2001 From: p-wysocki Date: Mon, 26 Aug 2024 15:35:05 +0200 Subject: [PATCH 07/24] Linter --- src/bindings/python/tests/test_runtime/test_compiled_model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bindings/python/tests/test_runtime/test_compiled_model.py b/src/bindings/python/tests/test_runtime/test_compiled_model.py index 55789ba90cba5d..e1d0ed3c8c64a8 100644 --- a/src/bindings/python/tests/test_runtime/test_compiled_model.py +++ b/src/bindings/python/tests/test_runtime/test_compiled_model.py @@ -75,7 +75,7 @@ def test_export_import_advanced(device): res = new_compiled.infer_new_request({"data": img}) assert np.argmax(res[new_compiled.outputs[0]]) == 531 - + def test_export_import_large_model(device): import io From a989a42d118d9345b2bfcca13dc996c3419f594e Mon Sep 17 00:00:00 2001 From: p-wysocki Date: Mon, 26 Aug 2024 16:07:32 +0200 Subject: [PATCH 08/24] CR --- .../python/src/pyopenvino/core/core.cpp | 44 +++++++++---------- 1 file changed, 20 insertions(+), 24 deletions(-) diff --git a/src/bindings/python/src/pyopenvino/core/core.cpp b/src/bindings/python/src/pyopenvino/core/core.cpp index 2cd9736c0b6c47..f0a32421fbdfac 100644 --- a/src/bindings/python/src/pyopenvino/core/core.cpp +++ b/src/bindings/python/src/pyopenvino/core/core.cpp @@ -553,7 +553,7 @@ void regclass_Core(py::module m) { (std::string)(py::repr(model_stream)) + "` provided"); } py::buffer_info info = py::buffer(model_stream.attr("getbuffer")()).request(); - constexpr auto one_gigabyte = static_cast(1024) * 1024 * 1024; + constexpr auto one_gigabyte = static_cast(1024) * 1024 * 1024; const double size_in_gb = (info.size * info.itemsize) / one_gigabyte; model_stream.attr("seek")(0); // Always rewind stream! ov::CompiledModel result; @@ -564,30 +564,26 @@ void regclass_Core(py::module m) { std::uniform_int_distribution<> distr(1000, 9999); std::string filename = "model_stream_" + std::to_string(distr(gen)) + ".txt"; std::fstream _stream(filename, std::ios::out | std::ios::binary); - if (_stream.is_open()) { - const py::bytes data = model_stream.attr("read")(); - // convert the Python bytes object to C++ string - char* buffer; - Py_ssize_t length; - PYBIND11_BYTES_AS_STRING_AND_SIZE(data.ptr(), &buffer, &length); - _stream.write(buffer, length); - _stream.close(); - } else { - OPENVINO_THROW("Failed to open temporary file for model stream"); - } + OPENVINO_ASSERT(_stream.is_open(), "Failed to open temporary file for model stream"); + const py::bytes data = model_stream.attr("read")(); + + // convert the Python bytes object to C++ string + char* buffer; + Py_ssize_t length; + PYBIND11_BYTES_AS_STRING_AND_SIZE(data.ptr(), &buffer, &length); + _stream.write(buffer, length); + _stream.close(); + std::fstream _fstream(filename, std::ios::in | std::ios::binary); - if (_fstream.is_open()) { - py::gil_scoped_release release; - result = self.import_model(_fstream, device_name, _properties); - _fstream.close(); - if (std::remove(filename.c_str()) != 0) { - const std::string abs_path = - py::module_::import("os").attr("getcwd")().cast() + "/" + filename; - const std::string warning_message = "Temporary file " + abs_path + " failed to delete!"; - PyErr_WarnEx(PyExc_RuntimeWarning, warning_message.c_str(), 1); - } - } else { - OPENVINO_THROW("Failed to open temporary file for model stream"); + OPENVINO_ASSERT(_fstream.is_open(), "Failed to open temporary file for model stream"); + py::gil_scoped_release release; + result = self.import_model(_fstream, device_name, _properties); + _fstream.close(); + if (std::remove(filename.c_str()) != 0) { + const std::string abs_path = + py::module_::import("os").attr("getcwd")().cast() + "/" + filename; + const std::string warning_message = "Temporary file " + abs_path + " failed to delete!"; + PyErr_WarnEx(PyExc_RuntimeWarning, warning_message.c_str(), 1); } } else { std::stringstream _stream; From 1238c2a575559ecdf9aec3aa544c085b6dc62e17 Mon Sep 17 00:00:00 2001 From: p-wysocki Date: Mon, 26 Aug 2024 16:36:52 +0200 Subject: [PATCH 09/24] Change expected val --- src/bindings/python/tests/test_runtime/test_compiled_model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bindings/python/tests/test_runtime/test_compiled_model.py b/src/bindings/python/tests/test_runtime/test_compiled_model.py index e1d0ed3c8c64a8..440a775629a4b0 100644 --- a/src/bindings/python/tests/test_runtime/test_compiled_model.py +++ b/src/bindings/python/tests/test_runtime/test_compiled_model.py @@ -92,7 +92,7 @@ def test_export_import_large_model(device): new_compiled = core.import_model(user_stream, device) img = generate_image([6, 10000, 10000]) res = new_compiled.infer_new_request({"input_data": img}) - assert np.argmax(res[new_compiled.outputs[0]]) == 44372822 + assert np.argmax(res[new_compiled.outputs[0]]) == 14970 @pytest.mark.parametrize("input_arguments", [[0], ["data"], []]) From 2ad1884f7779bc6873f71720f3cde699ce54509f Mon Sep 17 00:00:00 2001 From: p-wysocki Date: Mon, 26 Aug 2024 17:25:10 +0200 Subject: [PATCH 10/24] Lower the test model size --- .../python/tests/test_runtime/test_compiled_model.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/bindings/python/tests/test_runtime/test_compiled_model.py b/src/bindings/python/tests/test_runtime/test_compiled_model.py index 440a775629a4b0..6e26d9b0c04272 100644 --- a/src/bindings/python/tests/test_runtime/test_compiled_model.py +++ b/src/bindings/python/tests/test_runtime/test_compiled_model.py @@ -83,14 +83,14 @@ def test_export_import_large_model(device): if props.device.Capability.EXPORT_IMPORT not in core.get_property(device, props.device.capabilities): pytest.skip(f"{core.get_property(device, props.device.full_name)} plugin due-to export, import model API isn't implemented.") - # model of size of roughly 2.2GB - model = generate_model_with_memory([6, 10000, 10000], Type.f32) + # model of size of roughly 2.01GB + model = generate_model_with_memory([6, 10000, 9000], Type.f32) core = Core() compiled_model = core.compile_model(model, device, {}) user_stream = io.BytesIO() compiled_model.export_model(user_stream) new_compiled = core.import_model(user_stream, device) - img = generate_image([6, 10000, 10000]) + img = generate_image([6, 10000, 9000]) res = new_compiled.infer_new_request({"input_data": img}) assert np.argmax(res[new_compiled.outputs[0]]) == 14970 From 21c96b4f1acb205d0875ecdbda1763f9deb6548f Mon Sep 17 00:00:00 2001 From: p-wysocki Date: Tue, 27 Aug 2024 09:44:33 +0200 Subject: [PATCH 11/24] Modify test --- src/bindings/python/tests/test_runtime/test_compiled_model.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/bindings/python/tests/test_runtime/test_compiled_model.py b/src/bindings/python/tests/test_runtime/test_compiled_model.py index 6e26d9b0c04272..f7aec87bf8f419 100644 --- a/src/bindings/python/tests/test_runtime/test_compiled_model.py +++ b/src/bindings/python/tests/test_runtime/test_compiled_model.py @@ -84,6 +84,7 @@ def test_export_import_large_model(device): pytest.skip(f"{core.get_property(device, props.device.full_name)} plugin due-to export, import model API isn't implemented.") # model of size of roughly 2.01GB + print(device) model = generate_model_with_memory([6, 10000, 9000], Type.f32) core = Core() compiled_model = core.compile_model(model, device, {}) From aef175001fe8b343d48434de19f654c4e61d2cff Mon Sep 17 00:00:00 2001 From: p-wysocki Date: Tue, 27 Aug 2024 10:24:51 +0200 Subject: [PATCH 12/24] Remove debug print --- src/bindings/python/tests/test_runtime/test_compiled_model.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/bindings/python/tests/test_runtime/test_compiled_model.py b/src/bindings/python/tests/test_runtime/test_compiled_model.py index f7aec87bf8f419..6e26d9b0c04272 100644 --- a/src/bindings/python/tests/test_runtime/test_compiled_model.py +++ b/src/bindings/python/tests/test_runtime/test_compiled_model.py @@ -84,7 +84,6 @@ def test_export_import_large_model(device): pytest.skip(f"{core.get_property(device, props.device.full_name)} plugin due-to export, import model API isn't implemented.") # model of size of roughly 2.01GB - print(device) model = generate_model_with_memory([6, 10000, 9000], Type.f32) core = Core() compiled_model = core.compile_model(model, device, {}) From d0eb77114408acc1ea6016828812182aaefd5815 Mon Sep 17 00:00:00 2001 From: p-wysocki Date: Tue, 27 Aug 2024 11:58:46 +0200 Subject: [PATCH 13/24] Experiment with GIL --- src/bindings/python/src/pyopenvino/core/core.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/bindings/python/src/pyopenvino/core/core.cpp b/src/bindings/python/src/pyopenvino/core/core.cpp index f0a32421fbdfac..58e020a2f76e0d 100644 --- a/src/bindings/python/src/pyopenvino/core/core.cpp +++ b/src/bindings/python/src/pyopenvino/core/core.cpp @@ -576,7 +576,6 @@ void regclass_Core(py::module m) { std::fstream _fstream(filename, std::ios::in | std::ios::binary); OPENVINO_ASSERT(_fstream.is_open(), "Failed to open temporary file for model stream"); - py::gil_scoped_release release; result = self.import_model(_fstream, device_name, _properties); _fstream.close(); if (std::remove(filename.c_str()) != 0) { From 3b9ed888141218b2278f4acf67769ff773314bae Mon Sep 17 00:00:00 2001 From: p-wysocki Date: Tue, 27 Aug 2024 12:34:15 +0200 Subject: [PATCH 14/24] Experiment with GIL --- src/bindings/python/src/pyopenvino/core/core.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/bindings/python/src/pyopenvino/core/core.cpp b/src/bindings/python/src/pyopenvino/core/core.cpp index 58e020a2f76e0d..766872fa4f0e7f 100644 --- a/src/bindings/python/src/pyopenvino/core/core.cpp +++ b/src/bindings/python/src/pyopenvino/core/core.cpp @@ -578,6 +578,7 @@ void regclass_Core(py::module m) { OPENVINO_ASSERT(_fstream.is_open(), "Failed to open temporary file for model stream"); result = self.import_model(_fstream, device_name, _properties); _fstream.close(); + py::gil_scoped_release release; if (std::remove(filename.c_str()) != 0) { const std::string abs_path = py::module_::import("os").attr("getcwd")().cast() + "/" + filename; From 1fbf729a531bcd577c7ecbfec286903835f1cf2a Mon Sep 17 00:00:00 2001 From: p-wysocki Date: Tue, 27 Aug 2024 13:09:51 +0200 Subject: [PATCH 15/24] Experiment with GIL --- src/bindings/python/src/pyopenvino/core/core.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/bindings/python/src/pyopenvino/core/core.cpp b/src/bindings/python/src/pyopenvino/core/core.cpp index 766872fa4f0e7f..58e020a2f76e0d 100644 --- a/src/bindings/python/src/pyopenvino/core/core.cpp +++ b/src/bindings/python/src/pyopenvino/core/core.cpp @@ -578,7 +578,6 @@ void regclass_Core(py::module m) { OPENVINO_ASSERT(_fstream.is_open(), "Failed to open temporary file for model stream"); result = self.import_model(_fstream, device_name, _properties); _fstream.close(); - py::gil_scoped_release release; if (std::remove(filename.c_str()) != 0) { const std::string abs_path = py::module_::import("os").attr("getcwd")().cast() + "/" + filename; From 50fe743f32b38d74db72964ad5f37c38b4101f05 Mon Sep 17 00:00:00 2001 From: Anastasia Kuporosova Date: Tue, 27 Aug 2024 22:19:48 +0200 Subject: [PATCH 16/24] Update src/bindings/python/tests/test_runtime/test_compiled_model.py --- src/bindings/python/tests/test_runtime/test_compiled_model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bindings/python/tests/test_runtime/test_compiled_model.py b/src/bindings/python/tests/test_runtime/test_compiled_model.py index 6e26d9b0c04272..f7449f1ca160b2 100644 --- a/src/bindings/python/tests/test_runtime/test_compiled_model.py +++ b/src/bindings/python/tests/test_runtime/test_compiled_model.py @@ -92,7 +92,7 @@ def test_export_import_large_model(device): new_compiled = core.import_model(user_stream, device) img = generate_image([6, 10000, 9000]) res = new_compiled.infer_new_request({"input_data": img}) - assert np.argmax(res[new_compiled.outputs[0]]) == 14970 + assert np.argmax(res[new_compiled.outputs[0]]) == 44372822 @pytest.mark.parametrize("input_arguments", [[0], ["data"], []]) From 2d981d55da5827939c2121dd1c1046c8374cd6c3 Mon Sep 17 00:00:00 2001 From: p-wysocki Date: Wed, 28 Aug 2024 01:10:18 +0200 Subject: [PATCH 17/24] Reduce np array size --- .../python/tests/test_runtime/test_compiled_model.py | 8 ++++---- src/bindings/python/tests/utils/helpers.py | 8 ++++++++ 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/src/bindings/python/tests/test_runtime/test_compiled_model.py b/src/bindings/python/tests/test_runtime/test_compiled_model.py index 6e26d9b0c04272..3c59e1ec647b00 100644 --- a/src/bindings/python/tests/test_runtime/test_compiled_model.py +++ b/src/bindings/python/tests/test_runtime/test_compiled_model.py @@ -11,7 +11,7 @@ generate_image, generate_model_and_image, generate_relu_compiled_model, - generate_model_with_memory, + generate_big_model_with_tile, create_filename_for_test) from openvino import Model, Shape, Core, Tensor, serialize, Type from openvino.runtime import ConstOutput @@ -84,15 +84,15 @@ def test_export_import_large_model(device): pytest.skip(f"{core.get_property(device, props.device.full_name)} plugin due-to export, import model API isn't implemented.") # model of size of roughly 2.01GB - model = generate_model_with_memory([6, 10000, 9000], Type.f32) + model = generate_big_model_with_tile([1, 10, 9], [6, 10000, 9000], [6, 1000, 1000], Type.f32) core = Core() compiled_model = core.compile_model(model, device, {}) user_stream = io.BytesIO() compiled_model.export_model(user_stream) new_compiled = core.import_model(user_stream, device) - img = generate_image([6, 10000, 9000]) + img = generate_image([1, 10, 9]) res = new_compiled.infer_new_request({"input_data": img}) - assert np.argmax(res[new_compiled.outputs[0]]) == 14970 + assert np.argmax(res[new_compiled.outputs[0]]) == 63006 @pytest.mark.parametrize("input_arguments", [[0], ["data"], []]) diff --git a/src/bindings/python/tests/utils/helpers.py b/src/bindings/python/tests/utils/helpers.py index ce21dbe8db08cf..76c7e329e04345 100644 --- a/src/bindings/python/tests/utils/helpers.py +++ b/src/bindings/python/tests/utils/helpers.py @@ -234,6 +234,14 @@ def generate_model_with_memory(input_shape, data_type) -> openvino._pyopenvino.M return model +def generate_big_model_with_tile(input_shape, constant_shape, tile_shape, data_type) -> openvino._pyopenvino.Model: + input_data = ops.parameter(input_shape, name="input_data", dtype=data_type) + init_val = ops.constant(np.ones(constant_shape), data_type) + tile = ops.tile(input_data, tile_shape) + add = ops.add(init_val, tile, name="MemoryAdd") + return Model(add, [input_data], "TestModel") + + def create_filename_for_test(test_name, tmp_path, is_xml_path=False, is_bin_path=False): """Return a tuple with automatically generated paths for xml and bin files. From 8c009b269268be4ec9d86ec57ea1301970fc55b0 Mon Sep 17 00:00:00 2001 From: p-wysocki Date: Wed, 28 Aug 2024 10:46:57 +0200 Subject: [PATCH 18/24] Add support for export_model --- .../src/pyopenvino/core/compiled_model.cpp | 52 ++++++++++++++++--- .../python/src/pyopenvino/core/core.cpp | 6 +-- .../python/src/pyopenvino/utils/utils.cpp | 8 +++ .../python/src/pyopenvino/utils/utils.hpp | 2 + 4 files changed, 58 insertions(+), 10 deletions(-) diff --git a/src/bindings/python/src/pyopenvino/core/compiled_model.cpp b/src/bindings/python/src/pyopenvino/core/compiled_model.cpp index 1cbf33e11a9c50..7f50a4e211b1f7 100644 --- a/src/bindings/python/src/pyopenvino/core/compiled_model.cpp +++ b/src/bindings/python/src/pyopenvino/core/compiled_model.cpp @@ -7,6 +7,9 @@ #include #include +#include +#include + #include "common.hpp" #include "pyopenvino/core/compiled_model.hpp" #include "pyopenvino/core/infer_request.hpp" @@ -79,13 +82,50 @@ void regclass_CompiledModel(py::module m) { "`model_stream` must be an io.BytesIO object but " + (std::string)(py::repr(model_stream)) + "` provided"); } - std::stringstream _stream; - { - py::gil_scoped_release release; - self.export_model(_stream); + const auto model_stream_size = Common::utils::get_stream_size(model_stream); + model_stream.attr("seek")(0); // Always rewind stream! + // std::stringstream cannot handle streams > 2GB, in that case we use std::fstream + if (model_stream_size > 2) { + std::random_device rd; + std::mt19937 gen(rd()); + std::uniform_int_distribution<> distr(1000, 9999); + std::string filename = "model_stream_" + std::to_string(distr(gen)) + ".txt"; + std::fstream _stream(filename, std::ios::out | std::ios::binary); + + OPENVINO_ASSERT(_stream.is_open(), "Failed to open temporary file for model stream"); + const py::bytes data = model_stream.attr("read")(); + // convert the Python bytes object to C++ string + char* buffer; + Py_ssize_t length; + PYBIND11_BYTES_AS_STRING_AND_SIZE(data.ptr(), &buffer, &length); + _stream.write(buffer, length); + _stream.close(); + + std::fstream _fstream(filename, std::ios::in | std::ios::binary); + OPENVINO_ASSERT(_fstream.is_open(), "Failed to open temporary file for model stream"); + self.export_model(_fstream); + _fstream.seekg(0, std::ios::beg); + std::string str((std::istreambuf_iterator(_fstream)), std::istreambuf_iterator()); + _fstream.close(); + model_stream.attr("flush")(); + model_stream.attr("write")(py::bytes(str)); + _fstream.close(); + + if (std::remove(filename.c_str()) != 0) { + const std::string abs_path = + py::module_::import("os").attr("getcwd")().cast() + "/" + filename; + const std::string warning_message = "Temporary file " + abs_path + " failed to delete!"; + PyErr_WarnEx(PyExc_RuntimeWarning, warning_message.c_str(), 1); + } + } else { + std::stringstream _stream; + { + py::gil_scoped_release release; + self.export_model(_stream); + } + model_stream.attr("flush")(); + model_stream.attr("write")(py::bytes(_stream.str())); } - model_stream.attr("flush")(); - model_stream.attr("write")(py::bytes(_stream.str())); model_stream.attr("seek")(0); // Always rewind stream! }, py::arg("model_stream"), diff --git a/src/bindings/python/src/pyopenvino/core/core.cpp b/src/bindings/python/src/pyopenvino/core/core.cpp index 58e020a2f76e0d..6e200570f137cb 100644 --- a/src/bindings/python/src/pyopenvino/core/core.cpp +++ b/src/bindings/python/src/pyopenvino/core/core.cpp @@ -552,13 +552,11 @@ void regclass_Core(py::module m) { "`model_stream` must be an io.BytesIO object but " + (std::string)(py::repr(model_stream)) + "` provided"); } - py::buffer_info info = py::buffer(model_stream.attr("getbuffer")()).request(); - constexpr auto one_gigabyte = static_cast(1024) * 1024 * 1024; - const double size_in_gb = (info.size * info.itemsize) / one_gigabyte; + const auto model_stream_size = Common::utils::get_stream_size(model_stream); model_stream.attr("seek")(0); // Always rewind stream! ov::CompiledModel result; // std::stringstream cannot handle streams > 2GB, in that case use std::fstream - if (size_in_gb > 2) { + if (model_stream_size > 2) { std::random_device rd; std::mt19937 gen(rd()); std::uniform_int_distribution<> distr(1000, 9999); diff --git a/src/bindings/python/src/pyopenvino/utils/utils.cpp b/src/bindings/python/src/pyopenvino/utils/utils.cpp index 845691749b64f4..1e43bb755dbdf0 100644 --- a/src/bindings/python/src/pyopenvino/utils/utils.cpp +++ b/src/bindings/python/src/pyopenvino/utils/utils.cpp @@ -440,5 +440,13 @@ std::shared_ptr wrap_pyfunction(py::function f_callback) { }); return callback_sp; } + +double get_stream_size(const py::object& model_stream) { + // Returns io.BytesIO stream size in GB + py::object model_stream_buffer = model_stream.attr("getbuffer")(); + py::buffer_info info = py::buffer(model_stream_buffer).request(); + constexpr auto one_gigabyte = static_cast(1024) * 1024 * 1024; + return (info.size * info.itemsize) / one_gigabyte; +} }; // namespace utils }; // namespace Common diff --git a/src/bindings/python/src/pyopenvino/utils/utils.hpp b/src/bindings/python/src/pyopenvino/utils/utils.hpp index 2a376cc27b456a..67091cb2b5bba1 100644 --- a/src/bindings/python/src/pyopenvino/utils/utils.hpp +++ b/src/bindings/python/src/pyopenvino/utils/utils.hpp @@ -64,5 +64,7 @@ namespace utils { std::shared_ptr wrap_pyfunction(py::function f_callback); + double get_stream_size(const py::object& model_stream); + }; // namespace utils }; // namespace Common From d997310bb884f4498feb63acb3bd4b51c5bb836b Mon Sep 17 00:00:00 2001 From: p-wysocki Date: Wed, 28 Aug 2024 11:03:46 +0200 Subject: [PATCH 19/24] Fix data type --- src/bindings/python/src/pyopenvino/utils/utils.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bindings/python/src/pyopenvino/utils/utils.cpp b/src/bindings/python/src/pyopenvino/utils/utils.cpp index 8dfad6fa58acef..09a930ee03ce7b 100644 --- a/src/bindings/python/src/pyopenvino/utils/utils.cpp +++ b/src/bindings/python/src/pyopenvino/utils/utils.cpp @@ -465,7 +465,7 @@ std::shared_ptr wrap_pyfunction(py::function f_callback) { return callback_sp; } -double get_stream_size(const py::object& model_stream) { +int64_t get_stream_size(const py::object& model_stream) { // Returns io.BytesIO stream size in GB py::object model_stream_buffer = model_stream.attr("getbuffer")(); py::buffer_info info = py::buffer(model_stream_buffer).request(); From c5a9c4793374eeff6218e49c0ad691566e97d59a Mon Sep 17 00:00:00 2001 From: p-wysocki Date: Wed, 28 Aug 2024 11:15:51 +0200 Subject: [PATCH 20/24] Minor change --- src/bindings/python/src/pyopenvino/utils/utils.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bindings/python/src/pyopenvino/utils/utils.hpp b/src/bindings/python/src/pyopenvino/utils/utils.hpp index 1cc1ba277138bb..6d0284b25371c7 100644 --- a/src/bindings/python/src/pyopenvino/utils/utils.hpp +++ b/src/bindings/python/src/pyopenvino/utils/utils.hpp @@ -64,7 +64,7 @@ namespace utils { std::shared_ptr wrap_pyfunction(py::function f_callback); - double get_stream_size(const py::object& model_stream); + int64_t get_stream_size(const py::object& model_stream); }; // namespace utils }; // namespace Common From 2c3486dcd6edc528b26a7b23b9e5da3f682201fa Mon Sep 17 00:00:00 2001 From: p-wysocki Date: Wed, 28 Aug 2024 12:45:37 +0200 Subject: [PATCH 21/24] Change py::bytes to std::string conversion --- .../python/src/pyopenvino/core/compiled_model.cpp | 7 ++----- src/bindings/python/src/pyopenvino/core/core.cpp | 8 ++------ 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/src/bindings/python/src/pyopenvino/core/compiled_model.cpp b/src/bindings/python/src/pyopenvino/core/compiled_model.cpp index 7f50a4e211b1f7..184057e83bc612 100644 --- a/src/bindings/python/src/pyopenvino/core/compiled_model.cpp +++ b/src/bindings/python/src/pyopenvino/core/compiled_model.cpp @@ -94,11 +94,8 @@ void regclass_CompiledModel(py::module m) { OPENVINO_ASSERT(_stream.is_open(), "Failed to open temporary file for model stream"); const py::bytes data = model_stream.attr("read")(); - // convert the Python bytes object to C++ string - char* buffer; - Py_ssize_t length; - PYBIND11_BYTES_AS_STRING_AND_SIZE(data.ptr(), &buffer, &length); - _stream.write(buffer, length); + std::string buffer = data.cast(); + _stream.write(buffer.c_str(), buffer.size()); _stream.close(); std::fstream _fstream(filename, std::ios::in | std::ios::binary); diff --git a/src/bindings/python/src/pyopenvino/core/core.cpp b/src/bindings/python/src/pyopenvino/core/core.cpp index 6e200570f137cb..565e247dc6182c 100644 --- a/src/bindings/python/src/pyopenvino/core/core.cpp +++ b/src/bindings/python/src/pyopenvino/core/core.cpp @@ -564,12 +564,8 @@ void regclass_Core(py::module m) { std::fstream _stream(filename, std::ios::out | std::ios::binary); OPENVINO_ASSERT(_stream.is_open(), "Failed to open temporary file for model stream"); const py::bytes data = model_stream.attr("read")(); - - // convert the Python bytes object to C++ string - char* buffer; - Py_ssize_t length; - PYBIND11_BYTES_AS_STRING_AND_SIZE(data.ptr(), &buffer, &length); - _stream.write(buffer, length); + std::string buffer = data.cast(); + _stream.write(buffer.c_str(), buffer.size()); _stream.close(); std::fstream _fstream(filename, std::ios::in | std::ios::binary); From ffed371c30bbbf6c5bf103eec5572a66cfa51147 Mon Sep 17 00:00:00 2001 From: p-wysocki Date: Wed, 28 Aug 2024 14:12:25 +0200 Subject: [PATCH 22/24] Reduce np array size --- src/bindings/python/tests/utils/helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bindings/python/tests/utils/helpers.py b/src/bindings/python/tests/utils/helpers.py index 35ea802e9294f4..bb061dd8cf25c8 100644 --- a/src/bindings/python/tests/utils/helpers.py +++ b/src/bindings/python/tests/utils/helpers.py @@ -258,7 +258,7 @@ def generate_model_with_memory(input_shape, data_type) -> openvino._pyopenvino.M def generate_big_model_with_tile(input_shape, constant_shape, tile_shape, data_type) -> openvino._pyopenvino.Model: input_data = ops.parameter(input_shape, name="input_data", dtype=data_type) - init_val = ops.constant(np.ones(constant_shape), data_type) + init_val = ops.constant(np.ones(constant_shape, np.float32), data_type) tile = ops.tile(input_data, tile_shape) add = ops.add(init_val, tile, name="MemoryAdd") return Model(add, [input_data], "TestModel") From 36fb6fdd75e34c34a81e8c2c375f72e5eab9f2ac Mon Sep 17 00:00:00 2001 From: p-wysocki Date: Fri, 30 Aug 2024 12:17:03 +0200 Subject: [PATCH 23/24] Change rtype of util --- src/bindings/python/src/pyopenvino/core/compiled_model.cpp | 2 +- src/bindings/python/src/pyopenvino/core/core.cpp | 2 +- src/bindings/python/src/pyopenvino/utils/utils.cpp | 4 ++-- src/bindings/python/src/pyopenvino/utils/utils.hpp | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/bindings/python/src/pyopenvino/core/compiled_model.cpp b/src/bindings/python/src/pyopenvino/core/compiled_model.cpp index 184057e83bc612..2fe5b416ab910e 100644 --- a/src/bindings/python/src/pyopenvino/core/compiled_model.cpp +++ b/src/bindings/python/src/pyopenvino/core/compiled_model.cpp @@ -85,7 +85,7 @@ void regclass_CompiledModel(py::module m) { const auto model_stream_size = Common::utils::get_stream_size(model_stream); model_stream.attr("seek")(0); // Always rewind stream! // std::stringstream cannot handle streams > 2GB, in that case we use std::fstream - if (model_stream_size > 2) { + if (model_stream_size > 2.0) { std::random_device rd; std::mt19937 gen(rd()); std::uniform_int_distribution<> distr(1000, 9999); diff --git a/src/bindings/python/src/pyopenvino/core/core.cpp b/src/bindings/python/src/pyopenvino/core/core.cpp index 565e247dc6182c..df08da3bb2bc3c 100644 --- a/src/bindings/python/src/pyopenvino/core/core.cpp +++ b/src/bindings/python/src/pyopenvino/core/core.cpp @@ -556,7 +556,7 @@ void regclass_Core(py::module m) { model_stream.attr("seek")(0); // Always rewind stream! ov::CompiledModel result; // std::stringstream cannot handle streams > 2GB, in that case use std::fstream - if (model_stream_size > 2) { + if (model_stream_size > 2.0) { std::random_device rd; std::mt19937 gen(rd()); std::uniform_int_distribution<> distr(1000, 9999); diff --git a/src/bindings/python/src/pyopenvino/utils/utils.cpp b/src/bindings/python/src/pyopenvino/utils/utils.cpp index 09a930ee03ce7b..81cab580b898ab 100644 --- a/src/bindings/python/src/pyopenvino/utils/utils.cpp +++ b/src/bindings/python/src/pyopenvino/utils/utils.cpp @@ -465,12 +465,12 @@ std::shared_ptr wrap_pyfunction(py::function f_callback) { return callback_sp; } -int64_t get_stream_size(const py::object& model_stream) { +double get_stream_size(const py::object& model_stream) { // Returns io.BytesIO stream size in GB py::object model_stream_buffer = model_stream.attr("getbuffer")(); py::buffer_info info = py::buffer(model_stream_buffer).request(); constexpr auto one_gigabyte = static_cast(1024) * 1024 * 1024; - return (info.size * info.itemsize) / one_gigabyte; + return static_cast(info.size * info.itemsize) / one_gigabyte; } }; // namespace utils }; // namespace Common diff --git a/src/bindings/python/src/pyopenvino/utils/utils.hpp b/src/bindings/python/src/pyopenvino/utils/utils.hpp index 6d0284b25371c7..1cc1ba277138bb 100644 --- a/src/bindings/python/src/pyopenvino/utils/utils.hpp +++ b/src/bindings/python/src/pyopenvino/utils/utils.hpp @@ -64,7 +64,7 @@ namespace utils { std::shared_ptr wrap_pyfunction(py::function f_callback); - int64_t get_stream_size(const py::object& model_stream); + double get_stream_size(const py::object& model_stream); }; // namespace utils }; // namespace Common From fd10e607a6ceafc49341ef7b10bdca5fa871d848 Mon Sep 17 00:00:00 2001 From: p-wysocki Date: Fri, 6 Sep 2024 12:28:10 +0200 Subject: [PATCH 24/24] Replace unknowns with TODOs --- .../src/pyopenvino/core/compiled_model.cpp | 27 ++++++++----------- 1 file changed, 11 insertions(+), 16 deletions(-) diff --git a/src/bindings/python/src/pyopenvino/core/compiled_model.cpp b/src/bindings/python/src/pyopenvino/core/compiled_model.cpp index 2fe5b416ab910e..718bad98202fb1 100644 --- a/src/bindings/python/src/pyopenvino/core/compiled_model.cpp +++ b/src/bindings/python/src/pyopenvino/core/compiled_model.cpp @@ -82,32 +82,27 @@ void regclass_CompiledModel(py::module m) { "`model_stream` must be an io.BytesIO object but " + (std::string)(py::repr(model_stream)) + "` provided"); } - const auto model_stream_size = Common::utils::get_stream_size(model_stream); model_stream.attr("seek")(0); // Always rewind stream! // std::stringstream cannot handle streams > 2GB, in that case we use std::fstream - if (model_stream_size > 2.0) { + // TODO: Determine model size after serialization + if (true) { std::random_device rd; std::mt19937 gen(rd()); std::uniform_int_distribution<> distr(1000, 9999); std::string filename = "model_stream_" + std::to_string(distr(gen)) + ".txt"; - std::fstream _stream(filename, std::ios::out | std::ios::binary); - OPENVINO_ASSERT(_stream.is_open(), "Failed to open temporary file for model stream"); - const py::bytes data = model_stream.attr("read")(); - std::string buffer = data.cast(); - _stream.write(buffer.c_str(), buffer.size()); - _stream.close(); - - std::fstream _fstream(filename, std::ios::in | std::ios::binary); + std::fstream _fstream(filename, std::ios::out | std::ios::binary); OPENVINO_ASSERT(_fstream.is_open(), "Failed to open temporary file for model stream"); + + // TODO: Assert all plugins' export_model functions work correctly for std::fstream and models > 2GB self.export_model(_fstream); _fstream.seekg(0, std::ios::beg); - std::string str((std::istreambuf_iterator(_fstream)), std::istreambuf_iterator()); - _fstream.close(); - model_stream.attr("flush")(); - model_stream.attr("write")(py::bytes(str)); - _fstream.close(); - + { + std::string str((std::istreambuf_iterator(_fstream)), std::istreambuf_iterator()); + _fstream.close(); + model_stream.attr("flush")(); + model_stream.attr("write")(py::bytes(str)); + } if (std::remove(filename.c_str()) != 0) { const std::string abs_path = py::module_::import("os").attr("getcwd")().cast() + "/" + filename;