Merge branch 'master' into tj/core/serialize-validation

docs/requirements.txt

@@ -8,7 +8,7 @@ certifi==2024.7.4
 colorama==0.4.6
 Cython==0.29.33
 docutils==0.20
-idna==3.4
+idna==3.7
 imagesize==1.3.0
 importlib-metadata==4.8.0
 iniconfig==1.1.1

docs/sphinx_setup/conf.py

@@ -113,8 +113,6 @@
     "show_prev_next": False,
 }
 
-snippet_root = os.getenv("SNIPPET_ROOT", "")
-
 html_sidebars = {
     "**": ["search-field.html", "sidebar-nav-bs.html", "sidebar-ethical-ads.html"]
 }
@@ -123,7 +121,8 @@
     'current_language': 'English',
     #'languages': (('English', '/latest'), ('Chinese', '/cn/latest')),
     'doxygen_mapping_file': '@DOXYGEN_MAPPING_FILE@',
-    'doxygen_snippet_root': snippet_root,
+    # go back fours 3 steps down in directory to reach openvino dir
+    'doxygen_snippet_root': os.path.abspath(os.path.join(os.path.dirname(__file__), '../../../')),
     'default_mode': 'light'
 }
 

src/common/transformations/include/transformations/op_conversions/convert_scatter_nd_update15_downgrade.hpp

@@ -0,0 +1,24 @@
+// Copyright (C) 2018-2023 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+//
+
+#pragma once
+
+#include "openvino/pass/graph_rewrite.hpp"
+#include "transformations_visibility.hpp"
+
+namespace ov {
+namespace pass {
+/**
+ * @ingroup ov_transformation_common_api
+ * @brief Converts ScatterNDUpdate version 15 to ScatterNDUpdate version 3 if ScatterNDUpdate reduction attribute is set
+ * to None.
+ */
+class TRANSFORMATIONS_API ConvertScatterNDUpdate15ToScatterNDUpdate3 : public MatcherPass {
+public:
+    OPENVINO_RTTI("ConvertScatterNDUpdate15ToScatterNDUpdate3", "0");
+    ConvertScatterNDUpdate15ToScatterNDUpdate3();
+};
+
+}  // namespace pass
+}  // namespace ov

src/common/transformations/src/transformations/common_optimizations/common_optimizations.cpp

@@ -92,6 +92,7 @@
 #include "transformations/op_conversions/convert_roi_align_v3_to_v9.hpp"
 #include "transformations/op_conversions/convert_roi_align_v9_to_v3.hpp"
 #include "transformations/op_conversions/convert_scatter_elements_update12_downgrade.hpp"
+#include "transformations/op_conversions/convert_scatter_nd_update15_downgrade.hpp"
 #include "transformations/op_conversions/convert_slice_to_strided_slice.hpp"
 #include "transformations/op_conversions/convert_softmax_downgrade.hpp"
 #include "transformations/op_conversions/convert_softmax_upgrade.hpp"
@@ -231,6 +232,7 @@ bool ov::pass::CommonOptimizations::run_on_model(const std::shared_ptr<ov::Model
     REGISTER_PASS(manager, ConvertAvgPool14ToAvgPool1)
     REGISTER_PASS(manager, ConvertEmbeddingBagOffsets15ToEmbeddingBagOffsetsSum3)
     REGISTER_PASS(manager, ConvertEmbeddingBagPacked15ToEmbeddingBagPackedSum3)
+    REGISTER_PASS(manager, ConvertScatterNDUpdate15ToScatterNDUpdate3)
 
     auto fq_fusions = manager.register_pass<GraphRewrite>();
     ADD_MATCHER(fq_fusions, FakeQuantizeMulFusion)

src/common/transformations/src/transformations/common_optimizations/nop_elimination.cpp

@@ -789,8 +789,10 @@ pass::EliminateEltwise::EliminateEltwise() {
 
 pass::EliminateScatterUpdate::EliminateScatterUpdate() {
     MATCHER_SCOPE(EliminateScatterUpdate);
-    auto scatter_pattern =
-        pattern::wrap_type<ov::op::v3::ScatterUpdate, ov::op::v3::ScatterNDUpdate, ov::op::v3::ScatterElementsUpdate>();
+    auto scatter_pattern = pattern::wrap_type<ov::op::v3::ScatterUpdate,
+                                              ov::op::v3::ScatterNDUpdate,
+                                              ov::op::v15::ScatterNDUpdate,
+                                              ov::op::v3::ScatterElementsUpdate>();
 
     matcher_pass_callback callback = [=](pattern::Matcher& m) {
         auto scatter = m.get_match_root();

src/common/transformations/src/transformations/op_conversions/convert_scatter_nd_update15_downgrade.cpp

@@ -0,0 +1,39 @@
+// Copyright (C) 2018-2023 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+//
+
+#include "transformations/op_conversions/convert_scatter_nd_update15_downgrade.hpp"
+
+#include "itt.hpp"
+#include "openvino/core/rt_info.hpp"
+#include "openvino/op/scatter_nd_update.hpp"
+#include "openvino/pass/pattern/op/wrap_type.hpp"
+#include "transformations/utils/utils.hpp"
+
+ov::pass::ConvertScatterNDUpdate15ToScatterNDUpdate3::ConvertScatterNDUpdate15ToScatterNDUpdate3() {
+    MATCHER_SCOPE(ConvertScatterNDUpdate15ToScatterNDUpdate3);
+
+    const auto scatter_v15_pattern = pattern::wrap_type<ov::op::v15::ScatterNDUpdate>();
+
+    const matcher_pass_callback callback = [OV_CAPTURE_CPY_AND_THIS](pattern::Matcher& m) {
+        const auto scatter_v15 = std::dynamic_pointer_cast<ov::op::v15::ScatterNDUpdate>(m.get_match_root());
+        if (!scatter_v15 || transformation_callback(scatter_v15)) {
+            return false;
+        }
+        if (scatter_v15->get_reduction() != ov::op::v15::ScatterNDUpdate::Reduction::NONE) {
+            return false;
+        }
+        const auto scatter_v3 = std::make_shared<ov::op::v3::ScatterNDUpdate>(scatter_v15->input_value(0),
+                                                                              scatter_v15->input_value(1),
+                                                                              scatter_v15->input_value(2));
+
+        scatter_v3->set_friendly_name(scatter_v15->get_friendly_name());
+        copy_runtime_info(scatter_v15, scatter_v3);
+        replace_node(scatter_v15, scatter_v3);
+
+        return true;
+    };
+
+    auto m = std::make_shared<pattern::Matcher>(scatter_v15_pattern, matcher_name);
+    register_matcher(m, callback);
+}

src/common/transformations/tests/common_optimizations/nop_elimination.cpp

@@ -1746,3 +1746,25 @@ TEST_F(TransformationTestsF, TransposeElimination) {
         model_ref = std::make_shared<ov::Model>(OutputVector{result}, ParameterVector{data});
     }
 }
+
+TEST_F(TransformationTestsF, ScatterNDUpdates15Elimination) {
+    {
+        auto data = std::make_shared<op::v0::Parameter>(element::f32, PartialShape{100, 256, 10, 15});
+        auto indices = std::make_shared<op::v0::Parameter>(element::i32, PartialShape{25, 0, 3});
+        auto updates = std::make_shared<op::v0::Parameter>(element::f32, PartialShape{25, 0, 15});
+        auto relu = std::make_shared<op::v0::Relu>(data);
+        auto scatter = std::make_shared<op::v15::ScatterNDUpdate>(relu, indices, updates);
+
+        auto result = std::make_shared<op::v0::Result>(scatter);
+        model = std::make_shared<ov::Model>(OutputVector{result}, ParameterVector{data, indices, updates});
+        manager.register_pass<ov::pass::EliminateScatterUpdate>();
+    }
+    {
+        auto data = std::make_shared<op::v0::Parameter>(element::f32, PartialShape{100, 256, 10, 15});
+        auto indices = std::make_shared<op::v0::Parameter>(element::i32, PartialShape{25, 0, 3});
+        auto updates = std::make_shared<op::v0::Parameter>(element::f32, PartialShape{25, 0, 15});
+        auto relu = std::make_shared<op::v0::Relu>(data);
+        auto result = std::make_shared<op::v0::Result>(relu);
+        model_ref = std::make_shared<ov::Model>(OutputVector{result}, ParameterVector{data, indices, updates});
+    }
+}

src/common/transformations/tests/op_conversions/convert_scatter_nd_update15_downgrade_test.cpp

@@ -0,0 +1,54 @@
+// Copyright (C) 2018-2024 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+//
+
+#include "transformations/op_conversions/convert_scatter_nd_update15_downgrade.hpp"
+
+#include <gtest/gtest.h>
+
+#include <memory>
+
+#include "common_test_utils/ov_test_utils.hpp"
+#include "openvino/opsets/opset15.hpp"
+#include "openvino/opsets/opset4.hpp"
+#include "openvino/pass/manager.hpp"
+#include "transformations/utils/utils.hpp"
+using namespace ov;
+using namespace testing;
+
+namespace {
+using Reduction = ov::opset15::ScatterNDUpdate::Reduction;
+
+std::shared_ptr<ov::Model> create_v15_model(const Reduction reduction_type) {
+    const auto data = std::make_shared<ov::opset15::Parameter>(ov::element::f32, ov::Shape{1000, 256, 10, 15});
+    const auto indices = std::make_shared<ov::opset15::Parameter>(ov::element::i32, ov::Shape{25, 125, 3});
+    const auto updates = std::make_shared<ov::opset15::Parameter>(ov::element::f32, ov::Shape{25, 125, 15});
+    const auto scatter_nd = std::make_shared<ov::opset15::ScatterNDUpdate>(data, indices, updates, reduction_type);
+    scatter_nd->set_friendly_name("scatter_nd15");
+    return std::make_shared<ov::Model>(scatter_nd->outputs(), ov::ParameterVector{data, indices, updates});
+}
+
+std::shared_ptr<ov::Model> create_v3_model() {
+    const auto data = std::make_shared<ov::opset15::Parameter>(ov::element::f32, ov::Shape{1000, 256, 10, 15});
+    const auto indices = std::make_shared<ov::opset15::Parameter>(ov::element::i32, ov::Shape{25, 125, 3});
+    const auto updates = std::make_shared<ov::opset15::Parameter>(ov::element::f32, ov::Shape{25, 125, 15});
+    const auto scatter_nd = std::make_shared<ov::opset4::ScatterNDUpdate>(data, indices, updates);
+    scatter_nd->set_friendly_name("scatter_nd15");
+
+    return std::make_shared<ov::Model>(scatter_nd->outputs(), ov::ParameterVector{data, indices, updates});
+}
+
+}  // namespace
+
+TEST_F(TransformationTestsF, ConvertScatterNDUpdate15ToScatterNDUpdate3_no_reduction) {
+    manager.register_pass<ov::pass::ConvertScatterNDUpdate15ToScatterNDUpdate3>();
+    model = create_v15_model(Reduction::NONE);
+    model_ref = create_v3_model();
+    comparator.enable(FunctionsComparator::CmpValues::CONST_VALUES);
+    comparator.enable(FunctionsComparator::CmpValues::ATTRIBUTES);
+}
+
+TEST_F(TransformationTestsF, ConvertScatterNDUpdate15ToScatterNDUpdate3_reduction) {
+    manager.register_pass<ov::pass::ConvertScatterNDUpdate15ToScatterNDUpdate3>();
+    model = create_v15_model(Reduction::PROD);
+}

src/core/dev_api/openvino/core/meta_data.hpp

@@ -6,6 +6,10 @@
 
 #include "openvino/core/any.hpp"
 
+namespace pugi {
+class xml_node;
+}
+
 namespace ov {
 
 /**
@@ -32,4 +36,14 @@ class OPENVINO_API Meta {
     virtual ~Meta() = default;
 };
 
+class MetaDataWithPugixml : public Meta {
+public:
+    /**
+     * @brief Returns meta unchanged meta information. Throws ov::Exception if the meta was potentially changed
+     *
+     * @return const pugi::xml_node& with meta information
+     */
+    virtual const pugi::xml_node& get_pugi_node() const = 0;
+};
+
 }  // namespace ov

src/core/src/pass/serialize.cpp

@@ -916,11 +916,23 @@ void serialize_rt_info(pugi::xml_node& root, const std::string& name, const ov::
         child.append_attribute("name").set_value(name.c_str());
     }
     if (data.is<std::shared_ptr<ov::Meta>>()) {
-        std::shared_ptr<ov::Meta> meta = data.as<std::shared_ptr<ov::Meta>>();
-        ov::AnyMap& map = *meta;
-        for (const auto& it : map) {
-            serialize_rt_info(child, it.first, it.second);
-        }
+        auto meta = data.as<std::shared_ptr<ov::Meta>>();
+        do {
+            if (auto meta_with_pugixml_node = std::dynamic_pointer_cast<ov::MetaDataWithPugixml>(meta)) {
+                if (auto pugi_node = meta_with_pugixml_node->get_pugi_node()) {
+                    root.remove_child(child);
+                    auto added_node = root.append_copy(pugi_node);
+                    OPENVINO_ASSERT(added_node, "Cannot add pugixml node with name: ", name);
+                    added_node.set_name(name.c_str());
+                    break;
+                }
+            }
+            // Meta in ov::Meta cannot be accessed by MetaDataWithPugixml::get_pugi_node. Read it as ov::AnyMap
+            ov::AnyMap& map = *meta;
+            for (const auto& it : map) {
+                serialize_rt_info(child, it.first, it.second);
+            }
+        } while (false);
     } else if (data.is<ov::AnyMap>()) {
         const ov::AnyMap& any_map = data.as<ov::AnyMap>();
         for (const auto& it : any_map) {

src/core/tests/pass/serialization/rt_info_serialization.cpp

@@ -10,6 +10,7 @@
 #include "openvino/frontend/manager.hpp"
 #include "openvino/opsets/opset8.hpp"
 #include "openvino/pass/manager.hpp"
+#include "openvino/runtime/core.hpp"
 #include "transformations/rt_info/attributes.hpp"
 
 class RTInfoSerializationTest : public ov::test::TestsCommon {
@@ -221,3 +222,80 @@ TEST_F(RTInfoSerializationTest, tag_names_verification) {
                       ASSERT_EQ(model_rt_info[item.first], item.second);
                   });
 }
+
+TEST(OvSerializationTests, SerializeRawMeta) {
+    std::string ir_with_rt_info = R"V0G0N(<?xml version="1.0"?>
+<net name="Model0" version="11">
+	<layers>
+		<layer id="0" name="Parameter_1" type="Parameter" version="opset1">
+			<data shape="3,20,20" element_type="u8" />
+			<output>
+				<port id="0" precision="U8">
+					<dim>3</dim>
+					<dim>20</dim>
+					<dim>20</dim>
+				</port>
+			</output>
+		</layer>
+		<layer id="1" name="Result_2" type="Result" version="opset1">
+			<input>
+				<port id="0" precision="U8">
+					<dim>3</dim>
+					<dim>20</dim>
+					<dim>20</dim>
+				</port>
+			</input>
+		</layer>
+	</layers>
+	<edges>
+		<edge from-layer="0" from-port="0" to-layer="1" to-port="0" />
+	</edges>
+	<rt_info>
+		<custom_rt_info1>
+			<item0 value="testvalue1" />
+		</custom_rt_info1>
+		<custom_rt_info2>
+			<item0 value="testvalue2" />
+		</custom_rt_info2>
+	</rt_info>
+</net>
+)V0G0N";
+    ov::Core core;
+    {
+        // Don't read meta data. Copy raw pugixml::node from MetaDataWithPugixml to serialized model
+        auto model = core.read_model(ir_with_rt_info, ov::Tensor());
+
+        std::stringstream model_ss, weights_ss;
+        ov::pass::Serialize(model_ss, weights_ss).run_on_model(model);
+
+        auto serialized_model = model_ss.str();
+        EXPECT_EQ(0, serialized_model.compare(ir_with_rt_info));
+    }
+
+    {
+        // Don't read meta data. Fully serialize AnyMap with meta
+        auto model = core.read_model(ir_with_rt_info, ov::Tensor());
+        auto custom_rt_info1_value = model->get_rt_info<std::string>("custom_rt_info1", "item0");
+        EXPECT_EQ(0, custom_rt_info1_value.compare("testvalue1"));
+        auto custom_rt_info2_value = model->get_rt_info<std::string>("custom_rt_info2", "item0");
+        EXPECT_EQ(0, custom_rt_info2_value.compare("testvalue2"));
+
+        std::stringstream model_ss, weights_ss;
+        ov::pass::Serialize(model_ss, weights_ss).run_on_model(model);
+
+        auto serialized_model = model_ss.str();
+        EXPECT_EQ(0, serialized_model.compare(ir_with_rt_info));
+    }
+
+    {
+        auto model = core.read_model(ir_with_rt_info, ov::Tensor());
+        auto custom_rt_info1_value = model->get_rt_info<std::string>("custom_rt_info1", "item0");
+        EXPECT_EQ(0, custom_rt_info1_value.compare("testvalue1"));
+
+        std::stringstream model_ss, weights_ss;
+        ov::pass::Serialize(model_ss, weights_ss).run_on_model(model);
+
+        auto serialized_model = model_ss.str();
+        EXPECT_EQ(0, serialized_model.compare(ir_with_rt_info));
+    }
+}