From 98adad1c803a638696100820239e93170c479862 Mon Sep 17 00:00:00 2001 From: Pawel Raasz Date: Wed, 2 Oct 2024 11:22:40 +0200 Subject: [PATCH 1/3] [coverity, core] Fix unchecked return value in bound evaluate (#26768) ### Details: - Add check return value of node's evaluate in `ov::interval_bound_evaluator` function. This check allows to replace check if tensors are not empty (initialized), as when evaluate returns true the result is valid. ### Tickets: - CVS-153113 --- src/core/src/bound_evaluate.cpp | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/core/src/bound_evaluate.cpp b/src/core/src/bound_evaluate.cpp index 22b91a15e3dcee..f1c6a0601eea90 100644 --- a/src/core/src/bound_evaluate.cpp +++ b/src/core/src/bound_evaluate.cpp @@ -494,14 +494,12 @@ bool ov::interval_bound_evaluator(const Node* node, vector_of_output_variants.emplace_back(output.get_element_type(), output.get_shape()); } - node->evaluate(vector_of_output_variants, input_variant); + if (!node->evaluate(vector_of_output_variants, input_variant)) { + return false; + }; TensorVector vector_of_unsqueezed_output_variants; for (const auto& output : vector_of_output_variants) { - if (!output) { - return false; - } - auto unsqueezed_shape = output.get_shape(); unsqueezed_shape.insert(unsqueezed_shape.begin(), 1); From 32aaa2fbd969fb49d770e5abedf16198ea4ee0e4 Mon Sep 17 00:00:00 2001 From: Alicja Miloszewska Date: Wed, 2 Oct 2024 11:27:03 +0200 Subject: [PATCH 2/3] [OV JS] Add TS definition for AnyMap (#26711) ### Details: - Add definition of `OVAny` to `addon.ts` to encapsulate the concept of [ov::Any](https://docs.openvino.ai/2024/api/c_cpp_api/classov_1_1_any.html) from C++ API in a way that's clear and can be easily updated if the underlying C++ type changes. Improved readability and maintainability. -- 'OVAny' corresponds to Python API [OVAny](https://docs.openvino.ai/2024/api/ie_python_api/_autosummary/openvino.OVAny.html) - In future updates to Node.js API, the OVAny type is expected to be enhanced with additional functionality. - Missed linter changes to `core.test.js` due to upstream merges ### Tickets: - [CVS-149319](https://jira.devtools.intel.com/browse/CVS-149319) --- src/bindings/js/node/lib/addon.ts | 35 +++++++++----------- src/bindings/js/node/tests/unit/core.test.js | 24 +++++++------- 2 files changed, 28 insertions(+), 31 deletions(-) diff --git a/src/bindings/js/node/lib/addon.ts b/src/bindings/js/node/lib/addon.ts index 060af2cfec92e8..24c9d780aa9f7e 100644 --- a/src/bindings/js/node/lib/addon.ts +++ b/src/bindings/js/node/lib/addon.ts @@ -21,6 +21,8 @@ type elementTypeString = | 'f32' | 'string'; +type OVAny = string | number | boolean; + /** * Core represents an OpenVINO runtime Core entity. * @@ -48,7 +50,7 @@ interface Core { compileModel( model: Model, deviceName: string, - config?: { [propertyName: string]: string }, + config?: Record, ): Promise; /** * Asynchronously reads a model and creates a compiled model @@ -67,7 +69,7 @@ interface Core { compileModel( modelPath: string, deviceName: string, - config?: { [propertyName: string]: string }, + config?: Record, ): Promise; /** * A synchronous version of {@link Core.compileModel}. @@ -76,7 +78,7 @@ interface Core { compileModelSync( model: Model, deviceName: string, - config?: { [propertyName: string]: string }, + config?: Record, ): CompiledModel; /** * A synchronous version of {@link Core.compileModel}. @@ -85,7 +87,7 @@ interface Core { compileModelSync( modelPath: string, deviceName: string, - config?: { [propertyName: string]: string }, + config?: Record, ): CompiledModel; /** * It returns a list of available inference devices. @@ -101,7 +103,7 @@ interface Core { * It gets the properties dedicated to device behaviour. * @param propertyName A property name. */ - getProperty(propertyName: string): string | number | boolean; + getProperty(propertyName: string): OVAny; /** * It gets the properties dedicated to device behaviour. @@ -111,7 +113,7 @@ interface Core { getProperty( deviceName: string, propertyName: string, - ): string | number | boolean; + ): OVAny; /** * It returns information on the version of device plugins. * @param deviceName A device name to identify a plugin. @@ -135,7 +137,7 @@ interface Core { importModel( modelStream: Buffer, device: string, - config?: { [key: string]: string | number | boolean }, + config?: Record, ): Promise; /** * A synchronous version of {@link Core.importModel}. @@ -144,7 +146,7 @@ interface Core { importModelSync( modelStream: Buffer, device: string, - config?: { [key: string]: string | number | boolean }, + config?: Record, ): CompiledModel; /** * It reads models from the IR / ONNX / PDPD / TF and TFLite formats. @@ -197,16 +199,13 @@ interface Core { * It sets the properties. * @param properties An object with the property name - property value pairs. */ - setProperty(properties: { [key: string]: string | number | boolean }): void; + setProperty(properties: Record): void; /** * It sets the properties for a device. * @param deviceName The name of a device. * @param properties An object with the property name - property value pairs. */ - setProperty( - deviceName: string, - properties: { [key: string]: string | number | boolean }, - ): void; + setProperty(deviceName: string, properties: Record): void; /** * It queries the device if it supports specified model with the specified * properties. @@ -218,8 +217,8 @@ interface Core { queryModel( model: Model, deviceName: string, - properties?: {[key: string]: string | number | boolean}, - ): {[key: string]: string | number | boolean}; + properties?: Record, + ): { [key: string]: string }; } interface CoreConstructor { new (): Core; @@ -325,7 +324,7 @@ interface CompiledModel { * @param propertyName A string to get the property value. * @returns The property value. */ - getProperty(propertyName: string): string | number | boolean; + getProperty(propertyName: string): OVAny; /** * It creates an inference request object used to infer the compiled model. * @return {InferRequest} @@ -380,9 +379,7 @@ interface CompiledModel { * @param property An object with the key-value pairs. * (property name, property value) */ - setProperty(properties: { - [propertyName: string]: string | number | boolean; - }): void; + setProperty(properties: Record): void; } /** diff --git a/src/bindings/js/node/tests/unit/core.test.js b/src/bindings/js/node/tests/unit/core.test.js index 6cf431a38b5030..f62adda9f90f9c 100644 --- a/src/bindings/js/node/tests/unit/core.test.js +++ b/src/bindings/js/node/tests/unit/core.test.js @@ -12,11 +12,11 @@ describe('ov.Core tests', () => { before(async () => { await isModelAvailable(testModels.testModelFP32); }); - + beforeEach(() => { core = new ov.Core(); }); - + it('Core.setProperty()', () => { const tmpDir = '/tmp'; @@ -83,29 +83,29 @@ describe('ov.Core tests', () => { it('Core.queryModel() with empty parameters should throw an error', () => { assert.throws( () => core.queryModel().then(), - /'queryModel' method called with incorrect parameters./ - ) + /'queryModel' method called with incorrect parameters./, + ); }); it('Core.queryModel() with less arguments should throw an error', () => { assert.throws( - () => core.queryModel("Unexpected Argument").then(), - /'queryModel' method called with incorrect parameters./ - ) + () => core.queryModel('Unexpected Argument').then(), + /'queryModel' method called with incorrect parameters./, + ); }); it('Core.queryModel() with incorrect arguments should throw an error', () => { const model = core.readModelSync(getModelPath().xml); assert.throws( - () => core.queryModel(model, "arg1", "arg2").then(), - /'queryModel' method called with incorrect parameters./ - ) + () => core.queryModel(model, 'arg1', 'arg2').then(), + /'queryModel' method called with incorrect parameters./, + ); }); it('Core.queryModel() should have device in the result values', () => { const model = core.readModelSync(getModelPath().xml); const device = 'CPU'; - const query_model = core.queryModel(model, device); - assert(Object.values(query_model).includes(device)); + const queryModel = core.queryModel(model, device); + assert(Object.values(queryModel).includes(device)); }); }); From 205c11e0a8aa8d90f911f26689611c11db0751c7 Mon Sep 17 00:00:00 2001 From: Bogdan Pereanu Date: Wed, 2 Oct 2024 17:28:33 +0300 Subject: [PATCH 3/3] [NPU] Add documentation for batching on NPU plugin (#26865) ### Details: - *Add documentation for batching on NPU plugin* - *...* ### Tickets: - *EISW-118045* --------- Co-authored-by: Karol Blaszczak Co-authored-by: Tatiana Savina --- .../npu-device.rst | 1 + .../npu-device/batching-on-npu-plugin.rst | 37 +++++++++++++++++++ 2 files changed, 38 insertions(+) create mode 100644 docs/articles_en/openvino-workflow/running-inference/inference-devices-and-modes/npu-device/batching-on-npu-plugin.rst diff --git a/docs/articles_en/openvino-workflow/running-inference/inference-devices-and-modes/npu-device.rst b/docs/articles_en/openvino-workflow/running-inference/inference-devices-and-modes/npu-device.rst index d9f5e25c332984..7b135fa7ff0b14 100644 --- a/docs/articles_en/openvino-workflow/running-inference/inference-devices-and-modes/npu-device.rst +++ b/docs/articles_en/openvino-workflow/running-inference/inference-devices-and-modes/npu-device.rst @@ -11,6 +11,7 @@ NPU Device :hidden: npu-device/remote-tensor-api-npu-plugin + npu-device/batching-on-npu-plugin The Neural Processing Unit is a low-power hardware solution, introduced with the diff --git a/docs/articles_en/openvino-workflow/running-inference/inference-devices-and-modes/npu-device/batching-on-npu-plugin.rst b/docs/articles_en/openvino-workflow/running-inference/inference-devices-and-modes/npu-device/batching-on-npu-plugin.rst new file mode 100644 index 00000000000000..379822e327c8cd --- /dev/null +++ b/docs/articles_en/openvino-workflow/running-inference/inference-devices-and-modes/npu-device/batching-on-npu-plugin.rst @@ -0,0 +1,37 @@ +NPU Plugin Batching +=============================== + + +.. meta:: + :description: OpenVINO™ NPU plugin supports batching + either by executing concurrent inferences or by + relying on native compiler support for batching. + +OpenVINO™ NPU plugin supports batching either by executing concurrent inferences or by relying on native compiler support for batching. + +First, the NPU plugin checks if the following conditions are met: + +* The batch size is on the first axis. +* All inputs and outputs have the same batch size. +* The model does not contain states. + +**If the conditions are met**, the NPU plugin attempts to compile and execute the original model with batch_size forced to 1. This approach is due to current compiler limitations and ongoing work to improve performance for batch_size greater than one. +If the compilation is successful, the plugin detects a difference in batch size between the original model layout (with a batch size set to N) +and the transformed/compiled layout (with a batch size set to 1). Then it executes the following steps: + +1. Internally constructs multiple command lists, one for each input. +2. Executes each command list for the proper offsets of input/output buffers. +3. Notifies the user of the completion of the inference request after all command lists have been executed. + +This concurrency-based batching mode is transparent to the application. A single inference request handles all inputs from the batch. +While performance may be lower compared to regular batching (based on native compiler support), this mode provides basic batching functionality for use either with older drivers +or when the model cannot yet be compiled with a batch size larger than one. + +**If the conditions are not met**, the NPU plugin tries to compile and execute the original model with the given +batch_size to N as any other regular model. + +.. note:: + + With future performance improvements and support for compiling multiple models with a batch size larger + than one, the default order will change. NPU will try first to compile and execute the original model with the + given batch size and fall back to concurrent batching if compilation fails.