From 2db27a50f8bb5cbde25e2f5833346370389899e8 Mon Sep 17 00:00:00 2001 From: Feiyue Chen Date: Mon, 3 Jun 2024 03:09:52 +0000 Subject: [PATCH 1/2] Add gather/squeeze/unsqueeze/tile etc ops --- .../vsinpu/vsinpu_provider_factory.h | 44 +++--- onnxruntime/core/framework/node_unit.cc | 8 +- .../vsinpu/builders/impl/base_op_builder.cc | 18 ++- .../vsinpu/builders/impl/cast_op_builder.h | 43 ++++++ .../vsinpu/builders/impl/clip_op_builder.h | 4 +- .../builders/impl/dequantize_op_builder.h | 1 - .../vsinpu/builders/impl/gather_op_builder.h | 82 ++++++++++ .../vsinpu/builders/impl/norm_op_builder.h | 2 +- .../vsinpu/builders/impl/reduce_op_builder.h | 24 +-- .../vsinpu/builders/impl/resize_op_builder.h | 146 ++++++++++++++++++ .../vsinpu/builders/impl/squeeze_op_builder.h | 85 ++++++++++ .../vsinpu/builders/impl/tile_op_builder.h | 68 ++++++++ .../builders/impl/unsqueeze_op_builder.h | 63 ++++++++ .../vsinpu/builders/op_builder_factory.h | 13 +- .../vsinpu/patches/AccuracyCorrection.patch | 26 ++++ .../vsinpu/patches/int8_checker_hack.patch | 22 --- ...g.patch => local_testing_record_res.patch} | 0 .../test_scripts/compare_cosine_sim.py | 6 +- .../patches/test_scripts/compare_topn.py | 6 +- .../core/providers/vsinpu/vsinpu_ep_graph.cc | 8 +- .../core/providers/vsinpu/vsinpu_ep_graph.h | 14 +- .../vsinpu/vsinpu_execution_provider.cc | 21 +-- .../vsinpu/vsinpu_execution_provider.h | 2 + .../vsinpu/vsinpu_provider_factory.cc | 4 +- .../core/providers/vsinpu/vsinpu_util.cc | 30 ++-- .../core/providers/vsinpu/vsinpu_util.h | 3 + onnxruntime/test/util/default_providers.cc | 3 +- 27 files changed, 640 insertions(+), 106 deletions(-) create mode 100644 onnxruntime/core/providers/vsinpu/builders/impl/cast_op_builder.h create mode 100644 onnxruntime/core/providers/vsinpu/builders/impl/gather_op_builder.h create mode 100644 onnxruntime/core/providers/vsinpu/builders/impl/resize_op_builder.h create mode 100644 onnxruntime/core/providers/vsinpu/builders/impl/squeeze_op_builder.h create mode 100644 onnxruntime/core/providers/vsinpu/builders/impl/tile_op_builder.h create mode 100644 onnxruntime/core/providers/vsinpu/builders/impl/unsqueeze_op_builder.h create mode 100644 onnxruntime/core/providers/vsinpu/patches/AccuracyCorrection.patch delete mode 100644 onnxruntime/core/providers/vsinpu/patches/int8_checker_hack.patch rename onnxruntime/core/providers/vsinpu/patches/{hack_for_testing.patch => local_testing_record_res.patch} (100%) diff --git a/include/onnxruntime/core/providers/vsinpu/vsinpu_provider_factory.h b/include/onnxruntime/core/providers/vsinpu/vsinpu_provider_factory.h index bbd6fe4dfd868..a84067a19aa8a 100644 --- a/include/onnxruntime/core/providers/vsinpu/vsinpu_provider_factory.h +++ b/include/onnxruntime/core/providers/vsinpu/vsinpu_provider_factory.h @@ -1,26 +1,26 @@ /**************************************************************************** -* -* Copyright (c) 2023 Vivante Corporation -* -* Permission is hereby granted, free of charge, to any person obtaining a -* copy of this software and associated documentation files (the "Software"), -* to deal in the Software without restriction, including without limitation -* the rights to use, copy, modify, merge, publish, distribute, sublicense, -* and/or sell copies of the Software, and to permit persons to whom the -* Software is furnished to do so, subject to the following conditions: -* -* The above copyright notice and this permission notice shall be included in -* all copies or substantial portions of the Software. -* -* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -* DEALINGS IN THE SOFTWARE. -* -*****************************************************************************/ + * + * Copyright (c) 2023 Vivante Corporation + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + * + *****************************************************************************/ #include "onnxruntime_c_api.h" #ifdef __cplusplus diff --git a/onnxruntime/core/framework/node_unit.cc b/onnxruntime/core/framework/node_unit.cc index 174942b9033d0..30412125fe587 100644 --- a/onnxruntime/core/framework/node_unit.cc +++ b/onnxruntime/core/framework/node_unit.cc @@ -284,7 +284,7 @@ void NodeUnit::InitForSingleNode() { const auto& input_defs = target_node_.InputDefs(); const auto& output_defs = target_node_.OutputDefs(); auto qlinear_type = GetQLinearOpType(target_node_); - if (qlinear_type == QLinearOpType::Unknown || IsVariadicQLinearOp(qlinear_type)) { // TODO, add variadic support + if (qlinear_type == QLinearOpType::Unknown) { // Not a Qlinear op, add all inputs / outputs auto add_all_io = [](std::vector& defs, const ConstPointerContainer>& node_defs) { @@ -334,6 +334,12 @@ void NodeUnit::InitForSingleNode() { outputs_.push_back(NodeUnitIODef{*output_defs[0], NodeUnitIODef::QuantParam{*input_defs[1], input_defs.size() == 3 ? input_defs[2] : nullptr}}); + } else if (IsVariadicQLinearOp(qlinear_type)) { + size_t input_num = (input_defs.size() - 2) / 3; + for (size_t i = 0; i < input_num; i++) { + inputs_.push_back(NodeUnitIODef{*input_defs[3 * i + 2], NodeUnitIODef::QuantParam{*input_defs[3 * i + 3], input_defs[3 * i + 4]}}); + } + outputs_.push_back(NodeUnitIODef{*output_defs[0], NodeUnitIODef::QuantParam{*input_defs[0], input_defs[1]}}); } else { ORT_THROW("The QLinear op [", static_cast(qlinear_type), "] is not supported"); } diff --git a/onnxruntime/core/providers/vsinpu/builders/impl/base_op_builder.cc b/onnxruntime/core/providers/vsinpu/builders/impl/base_op_builder.cc index b3a3fa40a51c7..ba00dbf4f6baf 100644 --- a/onnxruntime/core/providers/vsinpu/builders/impl/base_op_builder.cc +++ b/onnxruntime/core/providers/vsinpu/builders/impl/base_op_builder.cc @@ -49,13 +49,13 @@ bool BaseOpBuilder::HasSupportedInputOutputs(const InitializedTensorSet& initial return false; } - // We do not support dynamic shape input yet + // We do not support dynamic shape input yet, but resize op's second input can be empty cause we not care about this value for (const auto& dim : shape_proto->dim()) { if (!dim.has_dim_value()) { LOGS_DEFAULT(WARNING) << "Dynamic shape is not supported for now, for input:" << node_arg.Name(); return false; } - if (dim.dim_value() == 0) { + if (dim.dim_value() == 0 && op_type != "Resize") { LOGS_DEFAULT(WARNING) << "Zero in shape is not supported for now, for input:" << node_arg.Name(); return false; } @@ -91,6 +91,10 @@ bool BaseOpBuilder::HasSupportedInputOutputs(const InitializedTensorSet& initial return false; if (!has_initialized_quant_param(*input.quant_param->zero_point, initializers)) return false; + if (input.quant_param->zero_point->Type() != input.node_arg.Type()) { + LOGS_DEFAULT(ERROR) << "Invalid input type because the data type mismatch with its' quant param type."; + return false; + } } } } @@ -115,7 +119,7 @@ bool BaseOpBuilder::HasSupportedInputOutputs(const InitializedTensorSet& initial bool BaseOpBuilder::HasSupportedInputOutputsImpl( const InitializedTensorSet& /* initializers */, const NodeUnit& node_unit) const { - // Check input data type, int64 is generally unsupported + // Check input/output data type, int64 is generally unsupported // specific op builder can override this if the int64 input corresponds to VSINPU param for (const auto& input : node_unit.Inputs()) { auto input_type = input.node_arg.Type(); @@ -125,6 +129,14 @@ bool BaseOpBuilder::HasSupportedInputOutputsImpl( return false; } } + for (const auto& output : node_unit.Outputs()) { + auto output_type = output.node_arg.Type(); + if (*output_type == "tensor(int64)" || !util::IsTypeSupported(&output.node_arg)) { + LOGS_DEFAULT(WARNING) << node_unit.OpType() << " has unsupported output type : " + << *output_type; + return false; + } + } return true; } diff --git a/onnxruntime/core/providers/vsinpu/builders/impl/cast_op_builder.h b/onnxruntime/core/providers/vsinpu/builders/impl/cast_op_builder.h new file mode 100644 index 0000000000000..68c9702c8d9ba --- /dev/null +++ b/onnxruntime/core/providers/vsinpu/builders/impl/cast_op_builder.h @@ -0,0 +1,43 @@ +/**************************************************************************** + * + * Copyright (c) 2024 Vivante Corporation + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + * + *****************************************************************************/ +#include "core/providers/shared/utils/utils.h" +#include "core/providers/vsinpu/builders/impl/base_op_builder.h" +namespace onnxruntime { +namespace vsi { +namespace npu { +class CastOpBuilder : public BaseOpBuilder { + protected: + bool HandleBuildOp(vsi::npu::GraphEP* graph_ep, std::vector>& inputs, + std::vector>& outputs, const NodeUnit& node_unit) override { + LOGS_DEFAULT(VERBOSE) << "Creating Cast Op."; + NodeAttrHelper helper(node_unit.GetNode()); + auto op = graph_ep->GetGraph()->CreateOperation(); + (*op).BindInput(inputs[0]).BindOutputs(outputs); + return true; + } +}; + +} // namespace npu +} // namespace vsi +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/vsinpu/builders/impl/clip_op_builder.h b/onnxruntime/core/providers/vsinpu/builders/impl/clip_op_builder.h index a91bd6ecf5d3e..9ad1ae6c82009 100644 --- a/onnxruntime/core/providers/vsinpu/builders/impl/clip_op_builder.h +++ b/onnxruntime/core/providers/vsinpu/builders/impl/clip_op_builder.h @@ -28,9 +28,9 @@ namespace onnxruntime { namespace vsi { namespace npu { class ClipOpBuilder final : public BaseOpBuilder { - bool IsOpSupported(const onnxruntime::GraphViewer& graph_viewer, + bool IsOpSupported(const onnxruntime::GraphViewer& graph_viewer, const Node* node) const override { - if (node->SinceVersion() > 6) { + if (node->SinceVersion() > 6) { if (node->InputDefs().size() > 1 && !Contains(graph_viewer.GetAllInitializedTensors(), node->InputDefs()[1]->Name())) { LOGS_DEFAULT(WARNING) << "Min/Max value must be const input or attribute."; return false; diff --git a/onnxruntime/core/providers/vsinpu/builders/impl/dequantize_op_builder.h b/onnxruntime/core/providers/vsinpu/builders/impl/dequantize_op_builder.h index 9774dc81ece88..760aed1fd3ecc 100644 --- a/onnxruntime/core/providers/vsinpu/builders/impl/dequantize_op_builder.h +++ b/onnxruntime/core/providers/vsinpu/builders/impl/dequantize_op_builder.h @@ -36,7 +36,6 @@ class DequantizeLinearOpBuilder : public BaseOpBuilder { }; bool HasSupportedInputOutputsImpl(const InitializedTensorSet& initializers, const NodeUnit& node_unit) const override { - auto input_type = node_unit.Inputs()[0].node_arg.Type(); if (*input_type == "tensor(int64)" || !util::IsTypeSupported(&node_unit.Inputs()[0].node_arg)) { LOGS_DEFAULT(WARNING) << node_unit.OpType() << " has unsupported input type : " diff --git a/onnxruntime/core/providers/vsinpu/builders/impl/gather_op_builder.h b/onnxruntime/core/providers/vsinpu/builders/impl/gather_op_builder.h new file mode 100644 index 0000000000000..97818927322e4 --- /dev/null +++ b/onnxruntime/core/providers/vsinpu/builders/impl/gather_op_builder.h @@ -0,0 +1,82 @@ +/**************************************************************************** + * + * Copyright (c) 2024 Vivante Corporation + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + * + *****************************************************************************/ +#include "core/providers/vsinpu/builders/impl/base_op_builder.h" +#include "core/providers/shared/utils/utils.h" + +namespace onnxruntime { +namespace vsi { +namespace npu { +class GatherOpBuilder : public BaseOpBuilder { + bool HasSupportedInputOutputsImpl(const InitializedTensorSet& initializers, + const NodeUnit& node_unit) const override { + auto input = node_unit.Inputs()[0]; + auto indices = node_unit.Inputs()[1]; + if (util::IsTypeSupported(&input.node_arg) && util::IsTypeSupported(&indices.node_arg)) { + if (*input.node_arg.Type() == "tensor(int64)") { + LOGS_DEFAULT(WARNING) << "Only support indices tensor to be int64 type in gather op."; + return false; + } + if (*indices.node_arg.Type() != "tensor(int64)" && *indices.node_arg.Type() != "tensor(int32)") { + LOGS_DEFAULT(WARNING) << "Unsupported indices tensor type in gather op."; + return false; + } + if (*indices.node_arg.Type() == "tensor(int64)" && !Contains(initializers, indices.node_arg.Name())) { + LOGS_DEFAULT(WARNING) << "Only support const attribute if indice tensor is in int64 type."; + return false; + } + return true; + } + return false; + } + + bool HandleBuildOp(vsi::npu::GraphEP* graph_ep, + std::vector>& inputs, + std::vector>& outputs, + const NodeUnit& node_unit) override { + LOGS_DEFAULT(VERBOSE) << "Creating Gather Op."; + NodeAttrHelper helper(node_unit.GetNode()); + auto axis = helper.Get("axis", 0); + axis = util::ReverseAxis(axis, inputs[0]->GetShape().size()); + auto op = graph_ep->GetGraph()->CreateOperation(axis, 0); + + bool is_i64_indices = inputs[1]->GetDataType() == tim::vx::DataType::INT64; + if (!is_i64_indices) { + (*op).BindInputs(inputs).BindOutputs(outputs); + } else { + std::vector origin_data(inputs[1]->GetSpec().GetElementNum()); + inputs[1]->CopyDataFromTensor(origin_data.data()); + std::vector transformed_data(origin_data.begin(), origin_data.end()); + auto transformed_indices = graph_ep->GetGraph()->CreateTensor( + inputs[1]->GetSpec().SetAttribute(tim::vx::TensorAttribute::INPUT).SetDataType(tim::vx::DataType::INT32), transformed_data.data()); + (*op).BindInput(inputs[0]).BindInput(transformed_indices).BindOutput(outputs[0]); + } + graph_ep->GetOps().push_back(std::move(op)); + return true; + } +}; + +} // namespace npu + +} // namespace vsi +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/vsinpu/builders/impl/norm_op_builder.h b/onnxruntime/core/providers/vsinpu/builders/impl/norm_op_builder.h index 52c85e9ff698c..f9d7dbe7183fc 100644 --- a/onnxruntime/core/providers/vsinpu/builders/impl/norm_op_builder.h +++ b/onnxruntime/core/providers/vsinpu/builders/impl/norm_op_builder.h @@ -36,7 +36,7 @@ class BatchNormOpBuilder : public BaseOpBuilder { mean_tensor = 3, var_tensor = 4 }; - int GetMinSupportedOpSet(const NodeUnit& /* node_unit */) const override{ return 9; } + int GetMinSupportedOpSet(const NodeUnit& /* node_unit */) const override { return 9; } bool IsOpSupported(const onnxruntime::GraphViewer& graph_viewer, const Node* node) const override { diff --git a/onnxruntime/core/providers/vsinpu/builders/impl/reduce_op_builder.h b/onnxruntime/core/providers/vsinpu/builders/impl/reduce_op_builder.h index 607040cfdaf3a..0ddf106da75b6 100644 --- a/onnxruntime/core/providers/vsinpu/builders/impl/reduce_op_builder.h +++ b/onnxruntime/core/providers/vsinpu/builders/impl/reduce_op_builder.h @@ -38,9 +38,9 @@ class ReduceMeanOpBuilder : public BaseOpBuilder { return true; } bool HandleBuildOp(vsi::npu::GraphEP* graph_ep, - std::vector>& inputs, - std::vector>& outputs, - const NodeUnit& node_unit) override { + std::vector>& inputs, + std::vector>& outputs, + const NodeUnit& node_unit) override { LOGS_DEFAULT(INFO) << "Creating ReduceMean Op."; NodeAttrHelper helper(node_unit.GetNode()); @@ -48,22 +48,22 @@ class ReduceMeanOpBuilder : public BaseOpBuilder { auto input_shape_size = inputs[0]->GetShape().size(); if (node_unit.SinceVersion() < 18 && helper.HasAttr("axes")) { - def_axes = helper.Get("axes", def_axes); + def_axes = helper.Get("axes", def_axes); } else if (inputs.size() > 1) { - def_axes.resize(inputs[1]->GetSpec().GetElementNum()); - inputs[1]->CopyDataFromTensor(def_axes.data()); + def_axes.resize(inputs[1]->GetSpec().GetElementNum()); + inputs[1]->CopyDataFromTensor(def_axes.data()); } else { - for (int64_t i = 0; i < input_shape_size; ++i) { - def_axes.push_back(i); - } + for (int64_t i = 0; i < input_shape_size; ++i) { + def_axes.push_back(i); + } } std::vector axes(def_axes.begin(), def_axes.end()); axes = util::ReverseAxis(axes, input_shape_size); if (helper.HasAttr("noop_with_empty_axes") && inputs.size() == 1 && helper.Get("noop_with_empty_axes", 0) == 1) { - outputs[0] = inputs[0]; - return true; + outputs[0] = inputs[0]; + return true; } bool keepdims = helper.Get("keepdims", 1) == 1; @@ -71,7 +71,7 @@ class ReduceMeanOpBuilder : public BaseOpBuilder { (*op).BindInput(inputs[0]).BindOutputs(outputs); graph_ep->GetOps().push_back(std::move(op)); return true; -} + } }; } // namespace npu diff --git a/onnxruntime/core/providers/vsinpu/builders/impl/resize_op_builder.h b/onnxruntime/core/providers/vsinpu/builders/impl/resize_op_builder.h new file mode 100644 index 0000000000000..cc49461940aed --- /dev/null +++ b/onnxruntime/core/providers/vsinpu/builders/impl/resize_op_builder.h @@ -0,0 +1,146 @@ +/**************************************************************************** + * + * Copyright (c) 2024 Vivante Corporation + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + * + *****************************************************************************/ +#include "core/providers/vsinpu/builders/impl/base_op_builder.h" +#include "core/providers/shared/utils/utils.h" + +namespace onnxruntime { +namespace vsi { +namespace npu { +class ResizeOpBuilder : public BaseOpBuilder { + bool HasSupportedInputOutputsImpl(const InitializedTensorSet& initializers, + const NodeUnit& node_unit) const override { + auto input_type = node_unit.Inputs()[0].node_arg.Type(); + if (*input_type == "tensor(int64)" || !util::IsTypeSupported(&node_unit.Inputs()[0].node_arg)) { + LOGS_DEFAULT(WARNING) << node_unit.OpType() << " has unsupported input type : " + << *input_type; + return false; + } + if (node_unit.SinceVersion() > 10) { + if (node_unit.Inputs().size() > 2 && !Contains(initializers, node_unit.Inputs()[2].node_arg.Name())) { + LOGS_DEFAULT(WARNING) << "Scale tensor must be constant."; + return false; + } + if (node_unit.Inputs().size() > 3 && !Contains(initializers, node_unit.Inputs()[3].node_arg.Name())) { + LOGS_DEFAULT(WARNING) << "Size tensor must be constant."; + return false; + } + } else { + if (!Contains(initializers, node_unit.Inputs()[1].node_arg.Name())) { + LOGS_DEFAULT(WARNING) << "Scale tensor must be constant."; + return false; + } + } + return true; + } + bool IsOpSupported(const onnxruntime::GraphViewer& graph_viewer, const Node* node) const override { + auto shape = vsi::npu::util::GetTensorShape(*node->InputDefs()[0]); + if (shape.NumDimensions() > 4) { + LOGS_DEFAULT(WARNING) << "3D or more dimesions resize is not supported."; + return false; + } + + NodeAttrHelper helper(*node); + if (helper.Get("antialiax", 0) != 0) { + LOGS_DEFAULT(WARNING) << "Antialias attribute is not supported."; + return false; + } + auto& cooridinate = helper.Get("coordinate_transoformation_mode", "half_pixel"); + if (cooridinate != "align_corners" && cooridinate != "half_pixel") { + LOGS_DEFAULT(WARNING) << "Only support half_pixel and align_corners attributes now."; + return false; + } + if (helper.Get("keep_aspect_ratio_policy", "stretch") != "stretch") { + LOGS_DEFAULT(WARNING) << "Not support to keep aspect ratio."; + return false; + } + if (helper.Get("mode", "nearest") == "cubic") { + LOGS_DEFAULT(WARNING) << "Not support the cubic resize type yet."; + return false; + } + if (helper.HasAttr("axes")) { + LOGS_DEFAULT(WARNING) << "Axes-specifying is not support."; + return false; + } + return true; + } + + bool HandleBuildOp(vsi::npu::GraphEP* graph_ep, + std::vector>& inputs, + std::vector>& outputs, + const NodeUnit& node_unit) override { + LOGS_DEFAULT(VERBOSE) << "Creating Resize Op."; + auto inputs_num = inputs.size(); + bool is_1dresize = inputs[0]->GetShape().size() == 1; + NodeAttrHelper helper(node_unit.GetNode()); + auto onnx_mode = helper.Get("mode", "nearest"); + auto coordinate_transformation = helper.Get("coordinate_transformation_mode", "half_pixel"); + bool is_size_set = helper.HasAttr("size"); + int32_t scale_index = node_unit.SinceVersion() > 10 ? 2 : 1; + + auto resize_type = onnx_mode == "nearest" ? tim::vx::ResizeType::NEAREST_NEIGHBOR : tim::vx::ResizeType::BILINEAR; + bool align_corners = coordinate_transformation == "align_corners"; + bool half_pixel_center = coordinate_transformation == "half_pixel"; + std::shared_ptr op = nullptr; + if (is_1dresize) { + int target_size; + if (is_size_set) { + int64_t onnx_size; + inputs[3]->CopyDataFromTensor(&onnx_size); + target_size = static_cast(onnx_size); + op = graph_ep->GetGraph()->CreateOperation(resize_type, 0.0f, align_corners, half_pixel_center, target_size); + } else { + float scale; + inputs[scale_index]->CopyDataFromTensor(&scale); + op = graph_ep->GetGraph()->CreateOperation(resize_type, scale, align_corners, half_pixel_center, 0); + } + } else { + int target_height, target_width; + if (is_size_set) { + std::vector onnx_sizes(inputs[3]->GetShape().size()); + inputs[3]->CopyDataFromTensor(onnx_sizes.data()); + target_height = static_cast(onnx_sizes[1]); + target_width = static_cast(onnx_sizes[0]); + op = graph_ep->GetGraph()->CreateOperation(resize_type, 0.0f, align_corners, half_pixel_center, target_height, target_width); + } else { + auto input_shape = inputs[0]->GetShape(); + std::vector scales(input_shape.size()); + std::vector out_shape(input_shape.size()); + inputs[scale_index]->CopyDataFromTensor(scales.data()); + for (int i = 0; i < input_shape.size(); i++) { + out_shape[i] = input_shape[i] * scales[input_shape.size() - 1 - i]; + } + op = graph_ep->GetGraph()->CreateOperation(resize_type, 0, align_corners, half_pixel_center, out_shape[1], out_shape[0]); + } + } + + (*op).BindInput(inputs[0]).BindOutputs(outputs); + graph_ep->GetOps().push_back(std::move(op)); + return true; + } +}; + +} // namespace npu + +} // namespace vsi +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/vsinpu/builders/impl/squeeze_op_builder.h b/onnxruntime/core/providers/vsinpu/builders/impl/squeeze_op_builder.h new file mode 100644 index 0000000000000..54174060c6381 --- /dev/null +++ b/onnxruntime/core/providers/vsinpu/builders/impl/squeeze_op_builder.h @@ -0,0 +1,85 @@ +/**************************************************************************** + * + * Copyright (c) 2024 Vivante Corporation + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + * + *****************************************************************************/ +#include "core/providers/vsinpu/builders/impl/base_op_builder.h" +#include "core/providers/shared/utils/utils.h" + +namespace onnxruntime { +namespace vsi { +namespace npu { +class SqueezeOpBuilder : public BaseOpBuilder { + bool HasSupportedInputOutputsImpl(const InitializedTensorSet& initializers, + const NodeUnit& node_unit) const override { + auto input_type = node_unit.Inputs()[0].node_arg.Type(); + if (*input_type == "tensor(int64)" || !util::IsTypeSupported(&node_unit.Inputs()[0].node_arg)) { + LOGS_DEFAULT(WARNING) << node_unit.OpType() << " has unsupported input type : " + << *input_type; + return false; + } + if (node_unit.SinceVersion() > 11) { + if (node_unit.Inputs().size() > 1 && !Contains(initializers, node_unit.Inputs()[1].node_arg.Name())) { + LOGS_DEFAULT(WARNING) << "Only support const axes in Squeeze op."; + return false; + } + } + return true; + } + + bool HandleBuildOp(vsi::npu::GraphEP* graph_ep, + std::vector>& inputs, + std::vector>& outputs, + const NodeUnit& node_unit) override { + LOGS_DEFAULT(INFO) << "Creating Squeeze Op."; + + NodeAttrHelper helper(node_unit.GetNode()); + std::vector def_axes; + auto input_shape_size = inputs[0]->GetShape().size(); + + if (node_unit.SinceVersion() < 13 && helper.HasAttr("axes")) { + def_axes = helper.Get("axes", def_axes); + } else if (inputs.size() > 1) { + def_axes.resize(inputs[1]->GetSpec().GetElementNum()); + inputs[1]->CopyDataFromTensor(def_axes.data()); + } else { // if axes is empty from onnx, check input shape to determine + for (int64_t i = 0; i < input_shape_size; ++i) { + if (inputs[0]->GetShape()[i] == 1) { + def_axes.push_back(i); + } + } + } + + std::vector axes(def_axes.begin(), def_axes.end()); + axes = util::ReverseAxis(axes, input_shape_size); + + std::vector timvx_axes(axes.begin(), axes.end()); + + auto op = graph_ep->GetGraph()->CreateOperation(timvx_axes); + (*op).BindInput(inputs[0]).BindOutputs(outputs); + graph_ep->GetOps().push_back(std::move(op)); + return true; + } +}; +} // namespace npu + +} // namespace vsi +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/vsinpu/builders/impl/tile_op_builder.h b/onnxruntime/core/providers/vsinpu/builders/impl/tile_op_builder.h new file mode 100644 index 0000000000000..cb81ab82f7c88 --- /dev/null +++ b/onnxruntime/core/providers/vsinpu/builders/impl/tile_op_builder.h @@ -0,0 +1,68 @@ +/**************************************************************************** + * + * Copyright (c) 2024 Vivante Corporation + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + * + *****************************************************************************/ +#include "core/providers/vsinpu/builders/impl/base_op_builder.h" + +namespace onnxruntime { +namespace vsi { +namespace npu { +class TileOpBuilder : public BaseOpBuilder { + int GetMinSupportedOpSet(const NodeUnit& /* node_unit */) const override { return 6; } + + bool HasSupportedInputOutputsImpl(const InitializedTensorSet& initializers, + const NodeUnit& node_unit) const override { + auto input = node_unit.Inputs()[0]; + auto multipliers = node_unit.Inputs()[1]; + if (initializers.end() == initializers.find(multipliers.node_arg.Name())) { + LOGS_DEFAULT(WARNING) << "Multipliers of tile op must be known."; + return false; + } + if (util::IsTypeSupported(&input.node_arg) && util::IsTypeSupported(&multipliers.node_arg)) { + if (*input.node_arg.Type() != "tensor(int64)") { + return true; + } + } + LOGS_DEFAULT(WARNING) << "Input type not supported."; + return false; + } + + bool HandleBuildOp(vsi::npu::GraphEP* graph_ep, + std::vector>& inputs, + std::vector>& outputs, + const NodeUnit& node_unit) override { + LOGS_DEFAULT(VERBOSE) << "Creating Tile Op."; + std::vector multipliers(inputs[1]->GetShape()[0]); + inputs[1]->CopyDataFromTensor(multipliers.data()); + std::reverse(multipliers.begin(), multipliers.end()); + std::vector timvx_multipliers(multipliers.begin(), multipliers.end()); + auto op = graph_ep->GetGraph()->CreateOperation(timvx_multipliers); + (*op).BindInput(inputs[0]).BindOutputs(outputs); + graph_ep->GetOps().push_back(std::move(op)); + return true; + } +}; + +} // namespace npu + +} // namespace vsi +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/vsinpu/builders/impl/unsqueeze_op_builder.h b/onnxruntime/core/providers/vsinpu/builders/impl/unsqueeze_op_builder.h new file mode 100644 index 0000000000000..40fe93c6b8454 --- /dev/null +++ b/onnxruntime/core/providers/vsinpu/builders/impl/unsqueeze_op_builder.h @@ -0,0 +1,63 @@ +#include "core/providers/vsinpu/builders/impl/base_op_builder.h" +#include "core/providers/shared/utils/utils.h" + +namespace onnxruntime { +namespace vsi { +namespace npu { +class UnsqueezeOpBuilder : public BaseOpBuilder { + bool HasSupportedInputOutputsImpl(const InitializedTensorSet& initializers, + const NodeUnit& node_unit) const override { + auto input_type = node_unit.Inputs()[0].node_arg.Type(); + if (*input_type == "tensor(int64)" || !util::IsTypeSupported(&node_unit.Inputs()[0].node_arg)) { + LOGS_DEFAULT(WARNING) << node_unit.OpType() << " has unsupported input type : " + << *input_type; + return false; + } + if (node_unit.SinceVersion() > 11 && !Contains(initializers, node_unit.Inputs()[1].node_arg.Name())) { + LOGS_DEFAULT(WARNING) << "Only support const axes in Unsqueeze op."; + return false; + } + return true; + } + + bool HandleBuildOp(vsi::npu::GraphEP* graph_ep, + std::vector>& inputs, + std::vector>& outputs, + const NodeUnit& node_unit) override { + LOGS_DEFAULT(INFO) << "Creating Unsqueeze Op."; + + NodeAttrHelper helper(node_unit.GetNode()); + std::vector def_axes; + auto input_shape_size = inputs[0]->GetShape().size(); + + if (node_unit.SinceVersion() < 13 && helper.HasAttr("axes")) { + def_axes = helper.Get("axes", def_axes); + } else if (inputs.size() > 1) { + def_axes.resize(inputs[1]->GetSpec().GetElementNum()); + inputs[1]->CopyDataFromTensor(def_axes.data()); + } else { // if axes is empty from onnx, check input shape to determine + for (int64_t i = 0; i < input_shape_size; ++i) { + if (inputs[0]->GetShape()[i] == 1) { + def_axes.push_back(i); + } + } + } + + std::vector axes(def_axes.begin(), def_axes.end()); + axes = util::ReverseAxis(axes, input_shape_size + axes.size()); + + std::vector timvx_axes(inputs[0]->GetShape().begin(), inputs[0]->GetShape().end()); + for (int32_t dim : axes) { + timvx_axes.insert(timvx_axes.begin() + dim, 1); + } + + auto op = graph_ep->GetGraph()->CreateOperation(timvx_axes); + (*op).BindInput(inputs[0]).BindOutputs(outputs); + graph_ep->GetOps().push_back(std::move(op)); + return true; + } +}; +} // namespace npu + +} // namespace vsi +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/vsinpu/builders/op_builder_factory.h b/onnxruntime/core/providers/vsinpu/builders/op_builder_factory.h index 3ab865d18f3a3..6538ee3078505 100644 --- a/onnxruntime/core/providers/vsinpu/builders/op_builder_factory.h +++ b/onnxruntime/core/providers/vsinpu/builders/op_builder_factory.h @@ -42,7 +42,12 @@ #include "impl/qlinearmatmul_op_builder.h" #include "impl/qlinear_binary_op_builder.h" #include "impl/qlinearconcat_op_builder.h" - +#include "impl/gather_op_builder.h" +#include "impl/tile_op_builder.h" +#include "impl/squeeze_op_builder.h" +#include "impl/unsqueeze_op_builder.h" +#include "impl/resize_op_builder.h" +#include "impl/cast_op_builder.h" namespace onnxruntime { namespace vsi { namespace npu { @@ -94,6 +99,12 @@ static const std::map reg = { REGISTER_OP_BUILDER("QLinearAdd", QLinearAddOpBuilder), REGISTER_OP_BUILDER("QLinearMul", QLinearMulOpBuilder), REGISTER_OP_BUILDER("QLinearConcat", QLinearConcatOpBuilder), + REGISTER_OP_BUILDER("Gather", GatherOpBuilder), + REGISTER_OP_BUILDER("Tile", TileOpBuilder), + REGISTER_OP_BUILDER("Squeeze", SqueezeOpBuilder), + REGISTER_OP_BUILDER("Unsqueeze", UnsqueezeOpBuilder), + REGISTER_OP_BUILDER("Resize", ResizeOpBuilder), + REGISTER_OP_BUILDER("Cast", CastOpBuilder), #undef REGISTER_OP_BUILDER }; diff --git a/onnxruntime/core/providers/vsinpu/patches/AccuracyCorrection.patch b/onnxruntime/core/providers/vsinpu/patches/AccuracyCorrection.patch new file mode 100644 index 0000000000000..d44190101d9fa --- /dev/null +++ b/onnxruntime/core/providers/vsinpu/patches/AccuracyCorrection.patch @@ -0,0 +1,26 @@ +diff --git a/onnxruntime/test/providers/checkers.cc b/onnxruntime/test/providers/checkers.cc +index 47c18c478d..93b44501cd 100644 +--- a/onnxruntime/test/providers/checkers.cc ++++ b/onnxruntime/test/providers/checkers.cc +@@ -195,7 +195,7 @@ struct TensorCheck { + // For any other EPs, we still expect an exact match for the results + // TODO: Verify if DML can possibly have a ROUNDING_MODE parameter and conform to the other EPs #41968513 + if ((provider_type == kNnapiExecutionProvider || provider_type == kDmlExecutionProvider || +- provider_type == kXnnpackExecutionProvider) && ++ provider_type == kXnnpackExecutionProvider || provider_type == kVSINPUExecutionProvider) && + (has_abs_err || has_rel_err)) { + double threshold = has_abs_err ? *(params.absolute_error) + : 0.0; +diff --git a/onnxruntime/test/providers/cpu/nn/qlinearconv_op_test.cc b/onnxruntime/test/providers/cpu/nn/qlinearconv_op_test.cc +index 2bc0df5e36..7beb78c2ff 100644 +--- a/onnxruntime/test/providers/cpu/nn/qlinearconv_op_test.cc ++++ b/onnxruntime/test/providers/cpu/nn/qlinearconv_op_test.cc +@@ -498,7 +498,7 @@ class QLinearConvOpTester { + // NOTE, for now the tolerance will only apply if the NNAPI is actually used, + // if for any reason the execution falls back to CPU, we still expect an exact match + // See, 'void Check(...' in onnxruntime/test/providers/provider_test_utils.cc +-#if defined(USE_NNAPI) || defined(USE_DML) ++#if defined(USE_NNAPI) || defined(USE_DML) || defined(USE_VSINPU) + // TODO: Verify if DML can possibly have a ROUNDING_MODE parameter and conform to the other EPs #41968513 + abs_error = 1.0f; + #endif diff --git a/onnxruntime/core/providers/vsinpu/patches/int8_checker_hack.patch b/onnxruntime/core/providers/vsinpu/patches/int8_checker_hack.patch deleted file mode 100644 index a94bf4363e168..0000000000000 --- a/onnxruntime/core/providers/vsinpu/patches/int8_checker_hack.patch +++ /dev/null @@ -1,22 +0,0 @@ -diff --git a/onnxruntime/test/providers/checkers.cc b/onnxruntime/test/providers/checkers.cc -index f1a7240ea3..436031dfa8 100644 ---- a/onnxruntime/test/providers/checkers.cc -+++ b/onnxruntime/test/providers/checkers.cc -@@ -154,6 +154,7 @@ struct TensorCheck { - } - - const bool has_abs_err = params.absolute_error.has_value(); -+ const int8_t default_abs_err = 1; - if (has_abs_err) { - double threshold = *(params.absolute_error); - -@@ -162,7 +163,8 @@ struct TensorCheck { - } - } else { - for (int i = 0; i < size; ++i) { -- EXPECT_EQ(cur_expected[i], cur_actual[i]) << "i:" << i; -+ // EXPECT_EQ(cur_expected[i], cur_actual[i]) << "i:" << i; -+ EXPECT_NEAR(cur_expected[i], cur_actual[i], default_abs_err) << "i:" << i; - } - } - } diff --git a/onnxruntime/core/providers/vsinpu/patches/hack_for_testing.patch b/onnxruntime/core/providers/vsinpu/patches/local_testing_record_res.patch similarity index 100% rename from onnxruntime/core/providers/vsinpu/patches/hack_for_testing.patch rename to onnxruntime/core/providers/vsinpu/patches/local_testing_record_res.patch diff --git a/onnxruntime/core/providers/vsinpu/patches/test_scripts/compare_cosine_sim.py b/onnxruntime/core/providers/vsinpu/patches/test_scripts/compare_cosine_sim.py index 1fe0bb02289e3..e4e9b44fdc252 100644 --- a/onnxruntime/core/providers/vsinpu/patches/test_scripts/compare_cosine_sim.py +++ b/onnxruntime/core/providers/vsinpu/patches/test_scripts/compare_cosine_sim.py @@ -1,15 +1,19 @@ import sys + import numpy as np from numpy.linalg import norm + def read_values(filename): - with open(filename, 'r') as file: + with open(filename) as file: values = np.array([float(line.strip()) for line in file]) return values + def cosine_similarity(vec1, vec2): return np.dot(vec1, vec2) / (norm(vec1) * norm(vec2)) + if __name__ == "__main__": if len(sys.argv) != 3: print("Usage: python cosine_similarity.py ") diff --git a/onnxruntime/core/providers/vsinpu/patches/test_scripts/compare_topn.py b/onnxruntime/core/providers/vsinpu/patches/test_scripts/compare_topn.py index 5de023e3c6e65..cde75b7f18c1e 100644 --- a/onnxruntime/core/providers/vsinpu/patches/test_scripts/compare_topn.py +++ b/onnxruntime/core/providers/vsinpu/patches/test_scripts/compare_topn.py @@ -1,13 +1,16 @@ import sys + def read_values(filename): - with open(filename, 'r') as file: + with open(filename) as file: values = [(float(line.strip()), i + 1) for i, line in enumerate(file)] return values + def top_n(values, N): return sorted(values, key=lambda x: x[0], reverse=True)[:N] + def compare_files(cpu_file, npu_file, N): cpu_values = read_values(cpu_file) npu_values = read_values(npu_file) @@ -18,6 +21,7 @@ def compare_files(cpu_file, npu_file, N): print(f"Top-{N} values in {cpu_file}: {cpu_topn}") print(f"Top-{N} values in {npu_file}: {npu_topn}") + if __name__ == "__main__": if len(sys.argv) != 4: print("Usage: python compare_topn.py ") diff --git a/onnxruntime/core/providers/vsinpu/vsinpu_ep_graph.cc b/onnxruntime/core/providers/vsinpu/vsinpu_ep_graph.cc index 0b8408dd4814a..e51b0713ea41d 100644 --- a/onnxruntime/core/providers/vsinpu/vsinpu_ep_graph.cc +++ b/onnxruntime/core/providers/vsinpu/vsinpu_ep_graph.cc @@ -22,9 +22,10 @@ * DEALINGS IN THE SOFTWARE. * *****************************************************************************/ -#include "vsinpu_ep_graph.h" -#include "builders/op_builder_factory.h" -#include "vsinpu_util.h" +#include +#include "core/providers/vsinpu/vsinpu_ep_graph.h" +#include "core/providers/vsinpu/builders/op_builder_factory.h" +#include "core/providers/vsinpu/vsinpu_util.h" #include "core/framework/node_unit.h" #include "core/optimizer/qdq_transformer/selectors_actions/qdq_selectors.h" #include "core/optimizer/qdq_transformer/selectors_actions/shared/utils.h" @@ -66,7 +67,6 @@ bool GraphEP::Prepare() { add_quantized_input(*node_unit, 2); } } - } // All quantized inputs is recorded return true; } diff --git a/onnxruntime/core/providers/vsinpu/vsinpu_ep_graph.h b/onnxruntime/core/providers/vsinpu/vsinpu_ep_graph.h index 6dbf0c64b9b30..bd0f377b820b0 100644 --- a/onnxruntime/core/providers/vsinpu/vsinpu_ep_graph.h +++ b/onnxruntime/core/providers/vsinpu/vsinpu_ep_graph.h @@ -25,7 +25,9 @@ #pragma once #include #include - +#include +#include +#include #include "builders/op_builder.h" #include "tim/vx/context.h" #include "tim/vx/graph.h" @@ -49,8 +51,8 @@ struct NodeIOInfo { class GraphEP { public: - GraphEP(const GraphViewer& graph_viewer); - ~GraphEP(){}; + explicit GraphEP(const GraphViewer& graph_viewer); + ~GraphEP() {} bool Prepare(); @@ -65,18 +67,18 @@ class GraphEP { bool& GetCompiled() { return compiled_; } std::shared_ptr& GetGraph() { return graph_; } - std::vector>& GetOps() { return ops_;} + std::vector>& GetOps() { return ops_; } std::map>& GetTensors() { return tensors_; } std::vector>& GetGraphInputs() { return graph_inputs_; - }; + } std::vector>& GetGraphOutputs() { return graph_outputs_; - }; + } void UpdateTensorMap(const std::string& name, const std::shared_ptr& dst_tensor); diff --git a/onnxruntime/core/providers/vsinpu/vsinpu_execution_provider.cc b/onnxruntime/core/providers/vsinpu/vsinpu_execution_provider.cc index 5617ef5052935..7444dcfec09a2 100644 --- a/onnxruntime/core/providers/vsinpu/vsinpu_execution_provider.cc +++ b/onnxruntime/core/providers/vsinpu/vsinpu_execution_provider.cc @@ -21,20 +21,20 @@ * DEALINGS IN THE SOFTWARE. * *****************************************************************************/ +#include +#include +#include #include "core/framework/compute_capability.h" -#include "vsinpu_execution_provider.h" -#include "vsinpu_ep_graph.h" -#include "builders/op_builder.h" -#include "builders/op_builder_factory.h" +#include "core/providers/vsinpu/vsinpu_execution_provider.h" +#include "core/providers/vsinpu/vsinpu_ep_graph.h" +#include "core/providers/vsinpu/builders/op_builder.h" +#include "core/providers/vsinpu/builders/op_builder_factory.h" +#include "core/providers/vsinpu/vsinpu_util.h" #include "core/framework/kernel_registry.h" #include "core/framework/node_unit.h" #include "core/optimizer/qdq_transformer/selectors_actions/qdq_selectors.h" #include "core/optimizer/qdq_transformer/selectors_actions/shared/utils.h" #include "core/providers/partitioning_utils.h" -#include "vsinpu_util.h" - -using namespace ONNX_NAMESPACE; -using namespace ::onnxruntime::logging; namespace onnxruntime { VSINPUExecutionProvider::VSINPUExecutionProvider(const VSINPUExecutionProviderInfo& info) @@ -197,7 +197,7 @@ Status ComputeStateFunc(vsi::npu::GraphEP* graph_ep, auto out_shape = graph_ep->GetGraphOutputs()[i]->shape.GetDims(); auto onnx_output_tensor = ctx.GetOutput(i, out_shape.data(), out_shape.size()); - timvx_tensor->CopyDataFromTensor((void*)onnx_output_tensor.GetTensorRawData()); + timvx_tensor->CopyDataFromTensor(const_cast(onnx_output_tensor.GetTensorRawData())); } return Status::OK(); @@ -242,8 +242,9 @@ Status VSINPUExecutionProvider::Compile(const std::vector& fu graph_ep->GetCompiled() = graph_ep->GetGraph()->Compile(); if (!graph_ep->GetCompiled()) { LOGS_DEFAULT(ERROR) << "Failed to verify graph."; - } else + } else { LOGS_DEFAULT(INFO) << "Graph has been verified successfully."; + } NodeComputeInfo compute_info; compute_info.create_state_func = [graph_ep](ComputeContext* /*context*/, diff --git a/onnxruntime/core/providers/vsinpu/vsinpu_execution_provider.h b/onnxruntime/core/providers/vsinpu/vsinpu_execution_provider.h index 7efac91a34b52..44318c332fdd0 100644 --- a/onnxruntime/core/providers/vsinpu/vsinpu_execution_provider.h +++ b/onnxruntime/core/providers/vsinpu/vsinpu_execution_provider.h @@ -22,6 +22,8 @@ * *****************************************************************************/ #pragma once +#include +#include #include "core/framework/execution_provider.h" #include "core/session/abi_session_options_impl.h" diff --git a/onnxruntime/core/providers/vsinpu/vsinpu_provider_factory.cc b/onnxruntime/core/providers/vsinpu/vsinpu_provider_factory.cc index 9297983f10057..5f2f961d95c09 100644 --- a/onnxruntime/core/providers/vsinpu/vsinpu_provider_factory.cc +++ b/onnxruntime/core/providers/vsinpu/vsinpu_provider_factory.cc @@ -23,8 +23,8 @@ *****************************************************************************/ #include "core/framework/compute_capability.h" #include "core/providers/vsinpu/vsinpu_provider_factory.h" -#include "vsinpu_provider_factory_creator.h" -#include "vsinpu_execution_provider.h" +#include "core/providers/vsinpu/vsinpu_provider_factory_creator.h" +#include "core/providers/vsinpu/vsinpu_execution_provider.h" namespace onnxruntime { diff --git a/onnxruntime/core/providers/vsinpu/vsinpu_util.cc b/onnxruntime/core/providers/vsinpu/vsinpu_util.cc index 111db391cd8f7..8008ec1f436a4 100644 --- a/onnxruntime/core/providers/vsinpu/vsinpu_util.cc +++ b/onnxruntime/core/providers/vsinpu/vsinpu_util.cc @@ -22,7 +22,11 @@ * *****************************************************************************/ -#include "vsinpu_util.h" +#include +#include +#include +#include +#include "core/providers/vsinpu/vsinpu_util.h" #include "core/optimizer/initializer.h" #include "core/providers/shared/utils/utils.h" @@ -52,7 +56,7 @@ tim::vx::DataType OnnxDtypeToTIMVXDtype(const int32_t dtype) { case onnx::TensorProto_DataType_UINT16: return tim::vx::DataType::UINT16; case onnx::TensorProto_DataType_BOOL: - return tim::vx::DataType::INT8; + return tim::vx::DataType::BOOL8; default: LOGS_DEFAULT(WARNING) << "Unsupported data type: " << dtype; break; @@ -70,7 +74,7 @@ tim::vx::DataType OnnxDtypeToTIMVXDtype(const ONNX_NAMESPACE::DataType type) { {"tensor(int16)", tim::vx::DataType::INT16}, {"tensor(uint16)", tim::vx::DataType::UINT16}, {"tensor(int64)", tim::vx::DataType::INT64}, - {"tensor(bool)", tim::vx::DataType::INT8}, + {"tensor(bool)", tim::vx::DataType::BOOL8}, }; auto search = type_table.find(*type); if (search != type_table.end()) { @@ -267,7 +271,6 @@ bool IsTypeSupported(const NodeArg* node_arg) { case ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_FLOAT16: case ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_INT8: case ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_UINT8: - case ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_UINT16: case ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_INT32: case ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_INT64: return true; @@ -445,13 +448,11 @@ void GetQuantizationScaleAndZeroPoint( auto zps = unpacked_tensor.DataAsSpan(); std::vector zps_vec(zps.begin(), zps.end()); pcq_zps = onnxruntime::make_optional(std::move(zps_vec)); - } - else if (is_int32_zp) { + } else if (is_int32_zp) { auto zps = unpacked_tensor.DataAsByteSpan(); std::vector zps_vec(zps.begin(), zps.end()); pcq_zps = onnxruntime::make_optional(std::move(zps_vec)); - } - else { + } else { auto zps = unpacked_tensor.DataAsSpan(); std::vector zps_vec(zps.begin(), zps.end()); pcq_zps = onnxruntime::make_optional(std::move(zps_vec)); @@ -467,13 +468,12 @@ static bool IsInternalQuantizedNodeUnit(const NodeUnit& node_unit) { // These operators can use uint8 input without specific QLinear version of it // However, the mode has to be internal to the graph/partition (they cannot consume graph inputs) - static const std::unordered_set internal_quantized_op_types = - { - "Transpose", - "Resize", - "Concat", - "MaxPool", - }; + static const std::unordered_set internal_quantized_op_types = { + "Transpose", + "Resize", + "Concat", + "MaxPool", + }; const auto& node = node_unit.GetNode(); if (!Contains(internal_quantized_op_types, node.OpType())) diff --git a/onnxruntime/core/providers/vsinpu/vsinpu_util.h b/onnxruntime/core/providers/vsinpu/vsinpu_util.h index 4d7ff73071f21..2b72b3a312d39 100644 --- a/onnxruntime/core/providers/vsinpu/vsinpu_util.h +++ b/onnxruntime/core/providers/vsinpu/vsinpu_util.h @@ -23,6 +23,9 @@ *****************************************************************************/ #pragma once +#include +#include +#include #include "core/framework/op_kernel.h" #include "core/framework/tensor_type_and_shape.h" #include "core/framework/tensorprotoutils.h" diff --git a/onnxruntime/test/util/default_providers.cc b/onnxruntime/test/util/default_providers.cc index d8109cdbc2a61..579774eb2abb8 100644 --- a/onnxruntime/test/util/default_providers.cc +++ b/onnxruntime/test/util/default_providers.cc @@ -186,9 +186,8 @@ std::unique_ptr DefaultNnapiExecutionProvider() { } std::unique_ptr DefaultVSINPUExecutionProvider() { - #if defined(USE_VSINPU) - return VSINPUProviderFactoryCreator::Create()->CreateProvider(); + return VSINPUProviderFactoryCreator::Create()->CreateProvider(); #else return nullptr; #endif From 1b57ed0488ab7eed66e301cc39acc81039735f1e Mon Sep 17 00:00:00 2001 From: Feiyue Chen Date: Fri, 7 Jun 2024 02:20:56 +0000 Subject: [PATCH 2/2] Fixed cpplint warning/errors --- onnxruntime/core/framework/node_unit.cc | 3 ++- .../builders/impl/activation_op_builder.h | 3 +++ .../vsinpu/builders/impl/base_op_builder.cc | 1 + .../vsinpu/builders/impl/base_op_builder.h | 5 ++-- .../vsinpu/builders/impl/cast_op_builder.h | 4 +++ .../vsinpu/builders/impl/clip_op_builder.cc | 2 ++ .../vsinpu/builders/impl/clip_op_builder.h | 2 ++ .../vsinpu/builders/impl/concat_op_builder.h | 3 +++ .../vsinpu/builders/impl/conv_op_builder.h | 4 +++ .../builders/impl/dequantize_op_builder.h | 3 +++ .../builders/impl/elementwise_op_builder.h | 4 ++- .../vsinpu/builders/impl/flatten_op_builder.h | 7 +++-- .../vsinpu/builders/impl/gather_op_builder.h | 6 ++++- .../vsinpu/builders/impl/gemm_op_builder.h | 3 +++ .../vsinpu/builders/impl/matmul_op_builder.h | 3 +++ .../vsinpu/builders/impl/norm_op_builder.h | 3 +++ .../vsinpu/builders/impl/pool_op_builder.h | 5 +++- .../builders/impl/qlinear_binary_op_builder.h | 3 +++ .../builders/impl/qlinearconcat_op_builder.h | 4 +++ .../builders/impl/qlinearconv_op_builder.h | 5 +++- .../builders/impl/qlinearmatmul_op_builder.h | 8 ++++-- .../builders/impl/quantize_op_builder.h | 3 +++ .../vsinpu/builders/impl/reduce_op_builder.h | 3 +++ .../vsinpu/builders/impl/resize_op_builder.h | 15 ++++++++--- .../vsinpu/builders/impl/softmax_op_builder.h | 4 +++ .../vsinpu/builders/impl/squeeze_op_builder.h | 3 +++ .../vsinpu/builders/impl/tensor_op_builder.h | 3 +++ .../vsinpu/builders/impl/tile_op_builder.h | 3 +++ .../builders/impl/unsqueeze_op_builder.h | 26 +++++++++++++++++++ .../vsinpu/builders/op_builder_factory.h | 5 +++- 30 files changed, 130 insertions(+), 16 deletions(-) diff --git a/onnxruntime/core/framework/node_unit.cc b/onnxruntime/core/framework/node_unit.cc index 30412125fe587..fea89496002cd 100644 --- a/onnxruntime/core/framework/node_unit.cc +++ b/onnxruntime/core/framework/node_unit.cc @@ -337,7 +337,8 @@ void NodeUnit::InitForSingleNode() { } else if (IsVariadicQLinearOp(qlinear_type)) { size_t input_num = (input_defs.size() - 2) / 3; for (size_t i = 0; i < input_num; i++) { - inputs_.push_back(NodeUnitIODef{*input_defs[3 * i + 2], NodeUnitIODef::QuantParam{*input_defs[3 * i + 3], input_defs[3 * i + 4]}}); + inputs_.push_back(NodeUnitIODef{*input_defs[3 * i + 2], NodeUnitIODef::QuantParam{*input_defs[3 * i + 3], + input_defs[3 * i + 4]}}); } outputs_.push_back(NodeUnitIODef{*output_defs[0], NodeUnitIODef::QuantParam{*input_defs[0], input_defs[1]}}); } else { diff --git a/onnxruntime/core/providers/vsinpu/builders/impl/activation_op_builder.h b/onnxruntime/core/providers/vsinpu/builders/impl/activation_op_builder.h index 87881585cfe21..9a59d90365f64 100644 --- a/onnxruntime/core/providers/vsinpu/builders/impl/activation_op_builder.h +++ b/onnxruntime/core/providers/vsinpu/builders/impl/activation_op_builder.h @@ -21,6 +21,9 @@ * DEALINGS IN THE SOFTWARE. * *****************************************************************************/ +#include +#include +#include #include "core/providers/shared/utils/utils.h" #include "core/providers/vsinpu/builders/impl/base_op_builder.h" diff --git a/onnxruntime/core/providers/vsinpu/builders/impl/base_op_builder.cc b/onnxruntime/core/providers/vsinpu/builders/impl/base_op_builder.cc index ba00dbf4f6baf..5755b21b0ecd8 100644 --- a/onnxruntime/core/providers/vsinpu/builders/impl/base_op_builder.cc +++ b/onnxruntime/core/providers/vsinpu/builders/impl/base_op_builder.cc @@ -21,6 +21,7 @@ * DEALINGS IN THE SOFTWARE. * *****************************************************************************/ +#include #include "core/providers/vsinpu/builders/impl/base_op_builder.h" namespace onnxruntime { diff --git a/onnxruntime/core/providers/vsinpu/builders/impl/base_op_builder.h b/onnxruntime/core/providers/vsinpu/builders/impl/base_op_builder.h index c664c44d86e7a..95bc9c5569d1d 100644 --- a/onnxruntime/core/providers/vsinpu/builders/impl/base_op_builder.h +++ b/onnxruntime/core/providers/vsinpu/builders/impl/base_op_builder.h @@ -22,7 +22,8 @@ * *****************************************************************************/ #pragma once - +#include +#include #include "core/providers/vsinpu/builders/op_builder.h" #include "core/providers/vsinpu/vsinpu_ep_graph.h" #include "core/providers/vsinpu/vsinpu_util.h" @@ -54,7 +55,7 @@ class BaseOpBuilder : public IOpBuilder { const InitializedTensorSet& initializers, const NodeUnit& node_unit) const; // TODO:Check if this node_unit's type is supported - virtual bool IsNodeUnitTypeSupported(const NodeUnit& node_unit) const { return true; }; + virtual bool IsNodeUnitTypeSupported(const NodeUnit& node_unit) const { return true; } virtual bool HandleBuildOp( vsi::npu::GraphEP* graph_ep, diff --git a/onnxruntime/core/providers/vsinpu/builders/impl/cast_op_builder.h b/onnxruntime/core/providers/vsinpu/builders/impl/cast_op_builder.h index 68c9702c8d9ba..6579f0ca9045f 100644 --- a/onnxruntime/core/providers/vsinpu/builders/impl/cast_op_builder.h +++ b/onnxruntime/core/providers/vsinpu/builders/impl/cast_op_builder.h @@ -21,6 +21,9 @@ * DEALINGS IN THE SOFTWARE. * *****************************************************************************/ +#include +#include +#include #include "core/providers/shared/utils/utils.h" #include "core/providers/vsinpu/builders/impl/base_op_builder.h" namespace onnxruntime { @@ -34,6 +37,7 @@ class CastOpBuilder : public BaseOpBuilder { NodeAttrHelper helper(node_unit.GetNode()); auto op = graph_ep->GetGraph()->CreateOperation(); (*op).BindInput(inputs[0]).BindOutputs(outputs); + graph_ep->GetOps().push_back(std::move(op)); return true; } }; diff --git a/onnxruntime/core/providers/vsinpu/builders/impl/clip_op_builder.cc b/onnxruntime/core/providers/vsinpu/builders/impl/clip_op_builder.cc index 92b560499d549..f3cccd69e3cf1 100644 --- a/onnxruntime/core/providers/vsinpu/builders/impl/clip_op_builder.cc +++ b/onnxruntime/core/providers/vsinpu/builders/impl/clip_op_builder.cc @@ -21,6 +21,8 @@ * DEALINGS IN THE SOFTWARE. * *****************************************************************************/ +#include +#include #include "core/providers/vsinpu/builders/impl/clip_op_builder.h" namespace onnxruntime { diff --git a/onnxruntime/core/providers/vsinpu/builders/impl/clip_op_builder.h b/onnxruntime/core/providers/vsinpu/builders/impl/clip_op_builder.h index 9ad1ae6c82009..8c056c5a2481a 100644 --- a/onnxruntime/core/providers/vsinpu/builders/impl/clip_op_builder.h +++ b/onnxruntime/core/providers/vsinpu/builders/impl/clip_op_builder.h @@ -21,6 +21,8 @@ * DEALINGS IN THE SOFTWARE. * *****************************************************************************/ +#include +#include #include "core/providers/vsinpu/builders/impl/base_op_builder.h" #include "core/providers/shared/utils/utils.h" diff --git a/onnxruntime/core/providers/vsinpu/builders/impl/concat_op_builder.h b/onnxruntime/core/providers/vsinpu/builders/impl/concat_op_builder.h index 1dfd115632586..4d3fc658b7bef 100644 --- a/onnxruntime/core/providers/vsinpu/builders/impl/concat_op_builder.h +++ b/onnxruntime/core/providers/vsinpu/builders/impl/concat_op_builder.h @@ -21,6 +21,9 @@ * DEALINGS IN THE SOFTWARE. * *****************************************************************************/ +#include +#include +#include #include "core/providers/vsinpu/builders/impl/base_op_builder.h" #include "core/providers/common.h" #include "core/providers/shared/utils/utils.h" diff --git a/onnxruntime/core/providers/vsinpu/builders/impl/conv_op_builder.h b/onnxruntime/core/providers/vsinpu/builders/impl/conv_op_builder.h index 8e6b0112d1312..d4776910a3ac1 100644 --- a/onnxruntime/core/providers/vsinpu/builders/impl/conv_op_builder.h +++ b/onnxruntime/core/providers/vsinpu/builders/impl/conv_op_builder.h @@ -21,6 +21,10 @@ * DEALINGS IN THE SOFTWARE. * *****************************************************************************/ +#include +#include +#include +#include #include "core/providers/shared/utils/utils.h" #include "core/providers/vsinpu/builders/impl/base_op_builder.h" namespace onnxruntime { diff --git a/onnxruntime/core/providers/vsinpu/builders/impl/dequantize_op_builder.h b/onnxruntime/core/providers/vsinpu/builders/impl/dequantize_op_builder.h index 760aed1fd3ecc..b4d1f6b19963f 100644 --- a/onnxruntime/core/providers/vsinpu/builders/impl/dequantize_op_builder.h +++ b/onnxruntime/core/providers/vsinpu/builders/impl/dequantize_op_builder.h @@ -21,6 +21,9 @@ * DEALINGS IN THE SOFTWARE. * *****************************************************************************/ +#include +#include +#include #include "core/providers/vsinpu/builders/impl/base_op_builder.h" #include "core/providers/common.h" #include "core/providers/shared/utils/utils.h" diff --git a/onnxruntime/core/providers/vsinpu/builders/impl/elementwise_op_builder.h b/onnxruntime/core/providers/vsinpu/builders/impl/elementwise_op_builder.h index 3f26282fc078f..01c48699d61ca 100644 --- a/onnxruntime/core/providers/vsinpu/builders/impl/elementwise_op_builder.h +++ b/onnxruntime/core/providers/vsinpu/builders/impl/elementwise_op_builder.h @@ -22,6 +22,9 @@ * DEALINGS IN THE SOFTWARE. * *****************************************************************************/ +#include +#include +#include #include "core/providers/vsinpu/builders/impl/base_op_builder.h" namespace onnxruntime { namespace vsi { @@ -85,7 +88,6 @@ class PowOpBuilder : public BaseOpBuilder { (*op).BindInputs(inputs).BindOutputs(outputs); graph_ep->GetOps().push_back(std::move(op)); return true; - ; } }; diff --git a/onnxruntime/core/providers/vsinpu/builders/impl/flatten_op_builder.h b/onnxruntime/core/providers/vsinpu/builders/impl/flatten_op_builder.h index c92b2f73b00e0..dfb0bb9c1b99f 100644 --- a/onnxruntime/core/providers/vsinpu/builders/impl/flatten_op_builder.h +++ b/onnxruntime/core/providers/vsinpu/builders/impl/flatten_op_builder.h @@ -21,6 +21,9 @@ * DEALINGS IN THE SOFTWARE. * *****************************************************************************/ +#include +#include +#include #include "core/providers/vsinpu/builders/impl/base_op_builder.h" #include "core/providers/common.h" #include "core/providers/shared/utils/utils.h" @@ -35,9 +38,9 @@ class FlattenOpBuilder : public BaseOpBuilder { const NodeUnit& node_unit) override { LOGS_DEFAULT(VERBOSE) << "Creating Flatten Op."; std::vector reshape_param; - if (outputs[0]->GetShape().size() == 2) + if (outputs[0]->GetShape().size() == 2) { reshape_param = outputs[0]->GetShape(); - else { + } else { auto input_shape = inputs[0]->GetShape(); NodeAttrHelper helper(node_unit.GetNode()); int64_t axis = helper.Get("axis", 1); diff --git a/onnxruntime/core/providers/vsinpu/builders/impl/gather_op_builder.h b/onnxruntime/core/providers/vsinpu/builders/impl/gather_op_builder.h index 97818927322e4..bd568873cd6d1 100644 --- a/onnxruntime/core/providers/vsinpu/builders/impl/gather_op_builder.h +++ b/onnxruntime/core/providers/vsinpu/builders/impl/gather_op_builder.h @@ -21,6 +21,9 @@ * DEALINGS IN THE SOFTWARE. * *****************************************************************************/ +#include +#include +#include #include "core/providers/vsinpu/builders/impl/base_op_builder.h" #include "core/providers/shared/utils/utils.h" @@ -68,7 +71,8 @@ class GatherOpBuilder : public BaseOpBuilder { inputs[1]->CopyDataFromTensor(origin_data.data()); std::vector transformed_data(origin_data.begin(), origin_data.end()); auto transformed_indices = graph_ep->GetGraph()->CreateTensor( - inputs[1]->GetSpec().SetAttribute(tim::vx::TensorAttribute::INPUT).SetDataType(tim::vx::DataType::INT32), transformed_data.data()); + inputs[1]->GetSpec().SetAttribute(tim::vx::TensorAttribute::INPUT).SetDataType(tim::vx::DataType::INT32), + transformed_data.data()); (*op).BindInput(inputs[0]).BindInput(transformed_indices).BindOutput(outputs[0]); } graph_ep->GetOps().push_back(std::move(op)); diff --git a/onnxruntime/core/providers/vsinpu/builders/impl/gemm_op_builder.h b/onnxruntime/core/providers/vsinpu/builders/impl/gemm_op_builder.h index 5fc0406a3f94e..6f5b484ce3a20 100644 --- a/onnxruntime/core/providers/vsinpu/builders/impl/gemm_op_builder.h +++ b/onnxruntime/core/providers/vsinpu/builders/impl/gemm_op_builder.h @@ -21,6 +21,9 @@ * DEALINGS IN THE SOFTWARE. * *****************************************************************************/ +#include +#include +#include #include "core/providers/shared/utils/utils.h" #include "core/providers/vsinpu/builders/impl/base_op_builder.h" namespace onnxruntime { diff --git a/onnxruntime/core/providers/vsinpu/builders/impl/matmul_op_builder.h b/onnxruntime/core/providers/vsinpu/builders/impl/matmul_op_builder.h index 48ca4d86e6d75..8cdf72906b644 100644 --- a/onnxruntime/core/providers/vsinpu/builders/impl/matmul_op_builder.h +++ b/onnxruntime/core/providers/vsinpu/builders/impl/matmul_op_builder.h @@ -21,6 +21,9 @@ * DEALINGS IN THE SOFTWARE. * *****************************************************************************/ +#include +#include +#include #include "core/providers/vsinpu/builders/impl/base_op_builder.h" namespace onnxruntime { diff --git a/onnxruntime/core/providers/vsinpu/builders/impl/norm_op_builder.h b/onnxruntime/core/providers/vsinpu/builders/impl/norm_op_builder.h index f9d7dbe7183fc..98149dc8de63d 100644 --- a/onnxruntime/core/providers/vsinpu/builders/impl/norm_op_builder.h +++ b/onnxruntime/core/providers/vsinpu/builders/impl/norm_op_builder.h @@ -21,6 +21,9 @@ * DEALINGS IN THE SOFTWARE. * *****************************************************************************/ +#include +#include +#include #include "core/providers/vsinpu/builders/impl/base_op_builder.h" #include "core/providers/shared/utils/utils.h" diff --git a/onnxruntime/core/providers/vsinpu/builders/impl/pool_op_builder.h b/onnxruntime/core/providers/vsinpu/builders/impl/pool_op_builder.h index 3575bbbc5194d..63d9a0d79f7de 100644 --- a/onnxruntime/core/providers/vsinpu/builders/impl/pool_op_builder.h +++ b/onnxruntime/core/providers/vsinpu/builders/impl/pool_op_builder.h @@ -21,6 +21,9 @@ * DEALINGS IN THE SOFTWARE. * *****************************************************************************/ +#include +#include +#include #include "core/providers/vsinpu/builders/impl/base_op_builder.h" #include "core/providers/shared/utils/utils.h" @@ -29,7 +32,7 @@ namespace vsi { namespace npu { class BasePoolOpBuilder : public BaseOpBuilder { public: - BasePoolOpBuilder(tim::vx::PoolType pool_type) : pool_type_(pool_type) {} + explicit BasePoolOpBuilder(tim::vx::PoolType pool_type) : pool_type_(pool_type) {} protected: bool IsOpSupported(const onnxruntime::GraphViewer& graph_viewer, const Node* node) const override { diff --git a/onnxruntime/core/providers/vsinpu/builders/impl/qlinear_binary_op_builder.h b/onnxruntime/core/providers/vsinpu/builders/impl/qlinear_binary_op_builder.h index c76bbbb4a7f55..def37b1ec1019 100644 --- a/onnxruntime/core/providers/vsinpu/builders/impl/qlinear_binary_op_builder.h +++ b/onnxruntime/core/providers/vsinpu/builders/impl/qlinear_binary_op_builder.h @@ -21,6 +21,9 @@ * DEALINGS IN THE SOFTWARE. * *****************************************************************************/ +#include +#include +#include #include "core/providers/vsinpu/builders/impl/base_op_builder.h" namespace onnxruntime { diff --git a/onnxruntime/core/providers/vsinpu/builders/impl/qlinearconcat_op_builder.h b/onnxruntime/core/providers/vsinpu/builders/impl/qlinearconcat_op_builder.h index eb5f79ee60ff9..dc51e99730c15 100644 --- a/onnxruntime/core/providers/vsinpu/builders/impl/qlinearconcat_op_builder.h +++ b/onnxruntime/core/providers/vsinpu/builders/impl/qlinearconcat_op_builder.h @@ -21,6 +21,9 @@ * DEALINGS IN THE SOFTWARE. * *****************************************************************************/ +#include +#include +#include #include "core/providers/shared/utils/utils.h" #include "core/providers/vsinpu/builders/impl/base_op_builder.h" namespace onnxruntime { @@ -35,6 +38,7 @@ class QLinearConcatOpBuilder : public BaseOpBuilder { axis = util::ReverseAxis(axis, inputs[0]->GetShape().size()); auto op = graph_ep->GetGraph()->CreateOperation(axis, inputs.size()); (*op).BindInputs(inputs).BindOutputs(outputs); + graph_ep->GetOps().push_back(std::move(op)); return true; } }; diff --git a/onnxruntime/core/providers/vsinpu/builders/impl/qlinearconv_op_builder.h b/onnxruntime/core/providers/vsinpu/builders/impl/qlinearconv_op_builder.h index cbb1d878089ce..c9b4721dd160a 100644 --- a/onnxruntime/core/providers/vsinpu/builders/impl/qlinearconv_op_builder.h +++ b/onnxruntime/core/providers/vsinpu/builders/impl/qlinearconv_op_builder.h @@ -21,10 +21,13 @@ * DEALINGS IN THE SOFTWARE. * *****************************************************************************/ +#include +#include +#include +#include #include "core/providers/shared/utils/utils.h" #include "core/providers/vsinpu/builders/impl/base_op_builder.h" #include "core/framework/tensorprotoutils.h" -#include namespace onnxruntime { namespace vsi { namespace npu { diff --git a/onnxruntime/core/providers/vsinpu/builders/impl/qlinearmatmul_op_builder.h b/onnxruntime/core/providers/vsinpu/builders/impl/qlinearmatmul_op_builder.h index f38c9955ae48e..7447c8b6b0b91 100644 --- a/onnxruntime/core/providers/vsinpu/builders/impl/qlinearmatmul_op_builder.h +++ b/onnxruntime/core/providers/vsinpu/builders/impl/qlinearmatmul_op_builder.h @@ -21,6 +21,9 @@ * DEALINGS IN THE SOFTWARE. * *****************************************************************************/ +#include +#include +#include #include "core/providers/vsinpu/builders/impl/base_op_builder.h" namespace onnxruntime { @@ -44,9 +47,9 @@ class QLinearMatMulOpBuilder : public BaseOpBuilder { auto A_def = input_defs[matrixA]; auto B_def = input_defs[matrixB]; for (auto def : input_defs) { - if (def->Name() == A_def->Name() || def->Name() == B_def->Name()) + if (def->Name() == A_def->Name() || def->Name() == B_def->Name()) { continue; - else { + } else { if (!graph_viewer.IsConstantInitializer(def->Name(), true)) { LOGS_DEFAULT(WARNING) << "Scale and zero point must be known before setting graph."; return false; @@ -70,6 +73,7 @@ class QLinearMatMulOpBuilder : public BaseOpBuilder { LOGS_DEFAULT(INFO) << "Creating QLinearMatmul Op."; auto op = graph_ep->GetGraph()->CreateOperation(); (*op).BindInputs(inputs).BindOutputs(outputs); + graph_ep->GetOps().push_back(std::move(op)); return true; } }; diff --git a/onnxruntime/core/providers/vsinpu/builders/impl/quantize_op_builder.h b/onnxruntime/core/providers/vsinpu/builders/impl/quantize_op_builder.h index 9dbfe32947e1b..155dedbc44f4c 100644 --- a/onnxruntime/core/providers/vsinpu/builders/impl/quantize_op_builder.h +++ b/onnxruntime/core/providers/vsinpu/builders/impl/quantize_op_builder.h @@ -21,6 +21,9 @@ * DEALINGS IN THE SOFTWARE. * *****************************************************************************/ +#include +#include +#include #include "core/providers/vsinpu/builders/impl/base_op_builder.h" #include "core/providers/common.h" #include "core/providers/shared/utils/utils.h" diff --git a/onnxruntime/core/providers/vsinpu/builders/impl/reduce_op_builder.h b/onnxruntime/core/providers/vsinpu/builders/impl/reduce_op_builder.h index 0ddf106da75b6..3b0a282c5de89 100644 --- a/onnxruntime/core/providers/vsinpu/builders/impl/reduce_op_builder.h +++ b/onnxruntime/core/providers/vsinpu/builders/impl/reduce_op_builder.h @@ -21,6 +21,9 @@ * DEALINGS IN THE SOFTWARE. * *****************************************************************************/ +#include +#include +#include #include "core/providers/vsinpu/builders/impl/base_op_builder.h" #include "core/providers/shared/utils/utils.h" diff --git a/onnxruntime/core/providers/vsinpu/builders/impl/resize_op_builder.h b/onnxruntime/core/providers/vsinpu/builders/impl/resize_op_builder.h index cc49461940aed..4ce8786e28d85 100644 --- a/onnxruntime/core/providers/vsinpu/builders/impl/resize_op_builder.h +++ b/onnxruntime/core/providers/vsinpu/builders/impl/resize_op_builder.h @@ -21,6 +21,9 @@ * DEALINGS IN THE SOFTWARE. * *****************************************************************************/ +#include +#include +#include #include "core/providers/vsinpu/builders/impl/base_op_builder.h" #include "core/providers/shared/utils/utils.h" @@ -108,11 +111,13 @@ class ResizeOpBuilder : public BaseOpBuilder { int64_t onnx_size; inputs[3]->CopyDataFromTensor(&onnx_size); target_size = static_cast(onnx_size); - op = graph_ep->GetGraph()->CreateOperation(resize_type, 0.0f, align_corners, half_pixel_center, target_size); + op = graph_ep->GetGraph()->CreateOperation(resize_type, 0.0f, align_corners, + half_pixel_center, target_size); } else { float scale; inputs[scale_index]->CopyDataFromTensor(&scale); - op = graph_ep->GetGraph()->CreateOperation(resize_type, scale, align_corners, half_pixel_center, 0); + op = graph_ep->GetGraph()->CreateOperation(resize_type, scale, align_corners, + half_pixel_center, 0); } } else { int target_height, target_width; @@ -121,7 +126,8 @@ class ResizeOpBuilder : public BaseOpBuilder { inputs[3]->CopyDataFromTensor(onnx_sizes.data()); target_height = static_cast(onnx_sizes[1]); target_width = static_cast(onnx_sizes[0]); - op = graph_ep->GetGraph()->CreateOperation(resize_type, 0.0f, align_corners, half_pixel_center, target_height, target_width); + op = graph_ep->GetGraph()->CreateOperation(resize_type, 0.0f, align_corners, + half_pixel_center, target_height, target_width); } else { auto input_shape = inputs[0]->GetShape(); std::vector scales(input_shape.size()); @@ -130,7 +136,8 @@ class ResizeOpBuilder : public BaseOpBuilder { for (int i = 0; i < input_shape.size(); i++) { out_shape[i] = input_shape[i] * scales[input_shape.size() - 1 - i]; } - op = graph_ep->GetGraph()->CreateOperation(resize_type, 0, align_corners, half_pixel_center, out_shape[1], out_shape[0]); + op = graph_ep->GetGraph()->CreateOperation(resize_type, 0, align_corners, + half_pixel_center, out_shape[1], out_shape[0]); } } diff --git a/onnxruntime/core/providers/vsinpu/builders/impl/softmax_op_builder.h b/onnxruntime/core/providers/vsinpu/builders/impl/softmax_op_builder.h index 1509077cced44..b8a0b7488710a 100644 --- a/onnxruntime/core/providers/vsinpu/builders/impl/softmax_op_builder.h +++ b/onnxruntime/core/providers/vsinpu/builders/impl/softmax_op_builder.h @@ -21,6 +21,10 @@ * DEALINGS IN THE SOFTWARE. * *****************************************************************************/ +#include +#include +#include +#include #include "core/providers/vsinpu/builders/impl/base_op_builder.h" #include "core/providers/common.h" #include "core/providers/shared/utils/utils.h" diff --git a/onnxruntime/core/providers/vsinpu/builders/impl/squeeze_op_builder.h b/onnxruntime/core/providers/vsinpu/builders/impl/squeeze_op_builder.h index 54174060c6381..2e1837384618d 100644 --- a/onnxruntime/core/providers/vsinpu/builders/impl/squeeze_op_builder.h +++ b/onnxruntime/core/providers/vsinpu/builders/impl/squeeze_op_builder.h @@ -21,6 +21,9 @@ * DEALINGS IN THE SOFTWARE. * *****************************************************************************/ +#include +#include +#include #include "core/providers/vsinpu/builders/impl/base_op_builder.h" #include "core/providers/shared/utils/utils.h" diff --git a/onnxruntime/core/providers/vsinpu/builders/impl/tensor_op_builder.h b/onnxruntime/core/providers/vsinpu/builders/impl/tensor_op_builder.h index 8fc6b8b0b531c..427457b521b61 100644 --- a/onnxruntime/core/providers/vsinpu/builders/impl/tensor_op_builder.h +++ b/onnxruntime/core/providers/vsinpu/builders/impl/tensor_op_builder.h @@ -21,6 +21,9 @@ * DEALINGS IN THE SOFTWARE. * *****************************************************************************/ +#include +#include +#include #include "core/providers/vsinpu/builders/impl/base_op_builder.h" #include "core/providers/shared/utils/utils.h" diff --git a/onnxruntime/core/providers/vsinpu/builders/impl/tile_op_builder.h b/onnxruntime/core/providers/vsinpu/builders/impl/tile_op_builder.h index cb81ab82f7c88..d42624c31557c 100644 --- a/onnxruntime/core/providers/vsinpu/builders/impl/tile_op_builder.h +++ b/onnxruntime/core/providers/vsinpu/builders/impl/tile_op_builder.h @@ -21,6 +21,9 @@ * DEALINGS IN THE SOFTWARE. * *****************************************************************************/ +#include +#include +#include #include "core/providers/vsinpu/builders/impl/base_op_builder.h" namespace onnxruntime { diff --git a/onnxruntime/core/providers/vsinpu/builders/impl/unsqueeze_op_builder.h b/onnxruntime/core/providers/vsinpu/builders/impl/unsqueeze_op_builder.h index 40fe93c6b8454..c49c93008b25a 100644 --- a/onnxruntime/core/providers/vsinpu/builders/impl/unsqueeze_op_builder.h +++ b/onnxruntime/core/providers/vsinpu/builders/impl/unsqueeze_op_builder.h @@ -1,3 +1,29 @@ +/**************************************************************************** + * + * Copyright (c) 2024 Vivante Corporation + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + * + *****************************************************************************/ +#include +#include +#include #include "core/providers/vsinpu/builders/impl/base_op_builder.h" #include "core/providers/shared/utils/utils.h" diff --git a/onnxruntime/core/providers/vsinpu/builders/op_builder_factory.h b/onnxruntime/core/providers/vsinpu/builders/op_builder_factory.h index 6538ee3078505..3a9190d8cb03a 100644 --- a/onnxruntime/core/providers/vsinpu/builders/op_builder_factory.h +++ b/onnxruntime/core/providers/vsinpu/builders/op_builder_factory.h @@ -22,7 +22,10 @@ * *****************************************************************************/ #pragma once - +#include +#include +#include +#include #include "impl/activation_op_builder.h" #include "impl/conv_op_builder.h" #include "impl/elementwise_op_builder.h"