From 71608740f9f4578d5bf9c4b37ab1ed34e97790b5 Mon Sep 17 00:00:00 2001 From: Masamitsu MURASE <29245944+murase-masamitsu@users.noreply.github.com> Date: Mon, 13 May 2024 18:27:34 +0900 Subject: [PATCH] Support "mask" parameter passed as 4th input of `DeformableConv2D` node. (#24347) ### Details: - This pull request improves [`openvino.convert_model`](https://docs.openvino.ai/2024/api/ie_python_api/_autosummary/openvino.convert_model.html) in Python, so that `DeformableConv2D` can handle the 4th input as `mask` parameter, which is described in [DeformableConvolution-**8**](https://docs.openvino.ai/2023.3/openvino_docs_ops_convolution_DeformableConvolution_8.html). ### Tickets: - Closes #24346 --- .../deformable_conv_2d.cpp | 37 +++-- .../deformable_conv_2d_with_mask.prototxt | 138 ++++++++++++++++++ .../tests/onnx_import_org_openvino.in.cpp | 35 +++++ .../runtime/interpreter/unit_test.manifest | 1 + 4 files changed, 201 insertions(+), 10 deletions(-) create mode 100644 src/frontends/onnx/tests/models/org.openvinotoolkit/deformable_conv_2d_with_mask.prototxt diff --git a/src/frontends/onnx/frontend/src/op/org.openvinotoolkit/deformable_conv_2d.cpp b/src/frontends/onnx/frontend/src/op/org.openvinotoolkit/deformable_conv_2d.cpp index 7abb16859b9dcc..ff02d0f6e0a0d5 100644 --- a/src/frontends/onnx/frontend/src/op/org.openvinotoolkit/deformable_conv_2d.cpp +++ b/src/frontends/onnx/frontend/src/op/org.openvinotoolkit/deformable_conv_2d.cpp @@ -16,6 +16,7 @@ #include "op/org.openvinotoolkit/deformable_conv_2d.hpp" +#include "openvino/frontend/exception.hpp" #include "openvino/op/deformable_convolution.hpp" #include "utils/convpool.hpp" @@ -36,16 +37,32 @@ ov::OutputVector deformable_conv_2d(const ov::frontend::onnx::Node& node) { const auto deformable_groups = node.get_attribute_value("deformable_groups", 1); const auto auto_pad_type = convpool::get_auto_pad(node); - return {std::make_shared(inputs.at(0), - inputs.at(1), - inputs.at(2), - strides, - paddings.first, - paddings.second, - dilations, - auto_pad_type, - group, - deformable_groups)}; + if (inputs.size() == 3) { + return {std::make_shared(inputs.at(0), + inputs.at(1), + inputs.at(2), + strides, + paddings.first, + paddings.second, + dilations, + auto_pad_type, + group, + deformable_groups)}; + } else if (inputs.size() == 4) { + return {std::make_shared(inputs.at(0), + inputs.at(1), + inputs.at(2), + inputs.at(3), + strides, + paddings.first, + paddings.second, + dilations, + auto_pad_type, + group, + deformable_groups)}; + } else { + FRONT_END_GENERAL_CHECK(false, "Invalid number of inputs"); + } } } // namespace set_1 } // namespace op diff --git a/src/frontends/onnx/tests/models/org.openvinotoolkit/deformable_conv_2d_with_mask.prototxt b/src/frontends/onnx/tests/models/org.openvinotoolkit/deformable_conv_2d_with_mask.prototxt new file mode 100644 index 00000000000000..c317dca2669536 --- /dev/null +++ b/src/frontends/onnx/tests/models/org.openvinotoolkit/deformable_conv_2d_with_mask.prototxt @@ -0,0 +1,138 @@ +ir_version: 7 +producer_name: "OpenVINO ONNX Frontend" +graph { + node { + input: "data" + input: "deformation" + input: "filters" + input: "mask" + output: "out" + op_type: "DeformableConv2D" + } + name: "test_graph" + input { + name: "data" + type { + tensor_type { + elem_type: 1 + shape { + dim { + dim_value: 1 + } + dim { + dim_value: 1 + } + dim { + dim_value: 4 + } + dim { + dim_value: 4 + } + } + } + } + } + input { + name: "deformation" + type { + tensor_type { + elem_type: 1 + shape { + dim { + dim_value: 1 + } + dim { + dim_value: 8 + } + dim { + dim_value: 3 + } + dim { + dim_value: 3 + } + } + } + } + } + input { + name: "filters" + type { + tensor_type { + elem_type: 1 + shape { + dim { + dim_value: 1 + } + dim { + dim_value: 1 + } + dim { + dim_value: 2 + } + dim { + dim_value: 2 + } + } + } + } + } + input { + name: "mask" + type { + tensor_type { + elem_type: 1 + shape { + dim { + dim_value: 1 + } + dim { + dim_value: 4 + } + dim { + dim_value: 3 + } + dim { + dim_value: 3 + } + } + } + } + } + initializer { + name: "filters" + dims: 1 + dims: 1 + dims: 2 + dims: 2 + data_type: 1 + float_data: 0.1 + float_data: 0.2 + float_data: 0.3 + float_data: 0.4 + } + output { + name: "out" + type { + tensor_type { + elem_type: 1 + shape { + dim { + dim_value: 1 + } + dim { + dim_value: 1 + } + dim { + dim_value: 3 + } + dim { + dim_value: 3 + } + } + } + } + } +} +opset_import { + version: 7 +} diff --git a/src/frontends/onnx/tests/onnx_import_org_openvino.in.cpp b/src/frontends/onnx/tests/onnx_import_org_openvino.in.cpp index 332467a0b92c2f..9116771f352b9e 100644 --- a/src/frontends/onnx/tests/onnx_import_org_openvino.in.cpp +++ b/src/frontends/onnx/tests/onnx_import_org_openvino.in.cpp @@ -565,6 +565,41 @@ OPENVINO_TEST(${BACKEND_NAME}, onnx_model_deformable_conv_2d) { test_case.run(); } +OPENVINO_TEST(${BACKEND_NAME}, onnx_model_deformable_conv_2d_with_mask) { + auto model = convert_model("org.openvinotoolkit/deformable_conv_2d_with_mask.onnx"); + + auto test_case = ov::test::TestCase(model, s_device); + + // data + test_case.add_input( + {1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f, 8.0f, 9.0f, 10.0f, 11.0f, 12.0f, 13.0f, 14.0f, 15.0f, 16.0f}); + + // deformations + test_case.add_input({0.5f, -0.5f, 0.0f, 1.0f, 0.5f, -0.5f, 0.0f, 1.0f, 1.0f, 0.5f, -0.5f, 0.0f, + 1.0f, 0.5f, -0.5f, 0.0f, 1.0f, 1.0f, 0.5f, -0.5f, 0.0f, 1.0f, 0.5f, -0.5f, + 0.0f, 1.0f, 1.0f, 0.5f, -0.5f, 0.0f, 1.0f, 0.5f, -0.5f, 0.0f, 1.0f, 1.0f, + 0.5f, -0.5f, 0.0f, 1.0f, 0.5f, -0.5f, 0.0f, 1.0f, 1.0f, 0.5f, -0.5f, 0.0f, + 1.0f, 0.5f, -0.5f, 0.0f, 1.0f, 1.0f, 0.5f, -0.5f, 0.0f, 1.0f, 0.5f, -0.5f, + 0.0f, 1.0f, 1.0f, 0.5f, -0.5f, 0.0f, 1.0f, 0.5f, -0.5f, 0.0f, 1.0f, 1.0f}); + + // mask + test_case.add_input({0.1f, 0.2f, 0.3f, 0.4f, 0.5f, 0.6f, 0.7f, 0.8f, 0.9f, 1.0f, 1.1f, 1.2f, + 1.3f, 1.4f, 1.5f, 1.6f, 1.7f, 1.8f, 1.9f, 2.0f, 2.1f, 2.2f, 2.3f, 2.4f, + 2.5f, 2.6f, 2.7f, 2.8f, 2.9f, 3.0f, 3.1f, 3.2f, 3.3f, 3.4f, 3.5f, 3.6f}); + + test_case.add_expected_output(Shape{1, 1, 3, 3}, + {14.7299995f, + 7.3200006f, + 15.0600004f, + 31.1000004f, + 28.9899998f, + 20.5800018f, + 32.6200027f, + 6.6400003f, + 1.4399999f}); + test_case.run(); +} + OPENVINO_TEST(${BACKEND_NAME}, onnx_model_generate_proposals) { auto model = convert_model("org.openvinotoolkit/generate_proposals.onnx"); diff --git a/src/frontends/onnx/tests/runtime/interpreter/unit_test.manifest b/src/frontends/onnx/tests/runtime/interpreter/unit_test.manifest index cd97b44b2f3d82..82fa651aaf6e94 100644 --- a/src/frontends/onnx/tests/runtime/interpreter/unit_test.manifest +++ b/src/frontends/onnx/tests/runtime/interpreter/unit_test.manifest @@ -45,6 +45,7 @@ onnx_controlflow_loop_power # No evaluator for DeformableConv2D onnx_model_deformable_conv_2d +onnx_model_deformable_conv_2d_with_mask # New fails onnx_model_quant_conv_linear_3d