Added VSINPU ExecutionProvider for onnxruntime with tag v1.18.0

Added VSINPU files in onnxruntime at branch rel/1.18

Type: New Feature
Signed-off-by: Feiyue Chen <Feiyue.Chen@verisilicon.com>

cmake/CMakeLists.txt

@@ -96,6 +96,7 @@ option(onnxruntime_BUILD_OBJC "Build Objective-C library" OFF)
 option(onnxruntime_USE_PREINSTALLED_EIGEN "Use pre-installed EIGEN. Need to provide eigen_SOURCE_PATH if turn this on." OFF)
 option(onnxruntime_BUILD_BENCHMARKS "Build ONNXRuntime micro-benchmarks" OFF)
 option(onnxruntime_USE_LLVM "Build TVM with LLVM" OFF)
+option(onnxruntime_USE_VSINPU "Build with VSINPU support" OFF)
 
 cmake_dependent_option(onnxruntime_USE_FLASH_ATTENTION "Build flash attention kernel for scaled dot product attention" ON "NOT WIN32; onnxruntime_USE_CUDA" OFF)
 option(onnxruntime_USE_MEMORY_EFFICIENT_ATTENTION "Build memory efficient attention kernel for scaled dot product attention" ON)
@@ -777,6 +778,11 @@ if (onnxruntime_USE_RKNPU)
     list(APPEND ORT_PROVIDER_CMAKE_FLAGS -Donnxruntime_USE_RKNPU=1)
     list(APPEND ONNXRUNTIME_PROVIDER_NAMES rknpu)
 endif()
+if (onnxruntime_USE_VSINPU)
+    list(APPEND ORT_PROVIDER_FLAGS -DUSE_VSINPU=1)
+    list(APPEND ORT_PROVIDER_CMAKE_FLAGS -Donnxruntime_USE_VSINPU=1)
+    list(APPEND ONNXRUNTIME_PROVIDER_NAMES vsinpu)
+endif()
 if (onnxruntime_USE_NNAPI_BUILTIN)
     list(APPEND ORT_PROVIDER_FLAGS -DUSE_NNAPI=1)
     list(APPEND ORT_PROVIDER_CMAKE_FLAGS -Donnxruntime_USE_NNAPI_BUILTIN=1)

cmake/onnxruntime.cmake

@@ -189,6 +189,7 @@ set(onnxruntime_INTERNAL_LIBRARIES
   ${PROVIDERS_SNPE}
   ${PROVIDERS_TVM}
   ${PROVIDERS_RKNPU}
+  ${PROVIDERS_VSINPU}
   ${PROVIDERS_XNNPACK}
   ${PROVIDERS_WEBNN}
   ${PROVIDERS_AZURE}

cmake/onnxruntime_providers.cmake

@@ -80,6 +80,9 @@ endif()
 if(onnxruntime_USE_RKNPU)
   set(PROVIDERS_RKNPU onnxruntime_providers_rknpu)
 endif()
+if(onnxruntime_USE_VSINPU)
+  set(PROVIDERS_VSINPU onnxruntime_providers_vsinpu)
+endif()
 if(onnxruntime_USE_DML)
   set(PROVIDERS_DML onnxruntime_providers_dml)
 endif()
@@ -188,6 +191,35 @@ if (onnxruntime_USE_TVM)
   include(onnxruntime_providers_tvm.cmake)
 endif()
 
+if (onnxruntime_USE_VSINPU)
+  add_definitions(-DUSE_VSINPU=1)
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused-parameter")
+  file(GLOB_RECURSE onnxruntime_providers_vsinpu_srcs
+    "${ONNXRUNTIME_ROOT}/core/providers/vsinpu/builders/*.h"
+    "${ONNXRUNTIME_ROOT}/core/providers/vsinpu/builders/*.cc"
+    "${ONNXRUNTIME_ROOT}/core/providers/vsinpu/*.h"
+    "${ONNXRUNTIME_ROOT}/core/providers/vsinpu/*.cc"
+    "${ONNXRUNTIME_ROOT}/core/providers/shared/utils/utils.h"
+    "${ONNXRUNTIME_ROOT}/core/providers/shared/utils/utils.cc"
+  )
+  source_group(TREE ${ONNXRUNTIME_ROOT}/core FILES ${onnxruntime_providers_vsinpu_srcs})
+  add_library(onnxruntime_providers_vsinpu ${onnxruntime_providers_vsinpu_srcs})
+  onnxruntime_add_include_to_target(onnxruntime_providers_vsinpu
+    onnxruntime_common onnxruntime_framework onnx onnx_proto protobuf::libprotobuf-lite flatbuffers Boost::mp11
+    safeint_interface nsync::nsync_cpp)
+  add_dependencies(onnxruntime_providers_vsinpu ${onnxruntime_EXTERNAL_DEPENDENCIES})
+  set_target_properties(onnxruntime_providers_vsinpu PROPERTIES FOLDER "ONNXRuntime" LINKER_LANGUAGE CXX)
+  target_include_directories(onnxruntime_providers_vsinpu PRIVATE ${ONNXRUNTIME_ROOT} $ENV{TIM_VX_INSTALL}/include)
+
+  find_library(TIMVX_LIBRARY NAMES tim-vx PATHS $ENV{TIM_VX_INSTALL}/lib NO_DEFAULT_PATH)
+  if(TIMVX_LIBRARY)
+    target_link_libraries(onnxruntime_providers_vsinpu PRIVATE ${TIMVX_LIBRARY})
+  else()
+    message(FATAL_ERROR "Cannot find TIM-VX library!")
+  endif()
+
+endif()
+
 if (onnxruntime_USE_XNNPACK)
   include(onnxruntime_providers_xnnpack.cmake)
 endif()

cmake/onnxruntime_unittests.cmake

@@ -546,6 +546,10 @@ if(onnxruntime_USE_NNAPI_BUILTIN)
   list(APPEND onnxruntime_test_providers_dependencies onnxruntime_providers_nnapi)
 endif()
 
+if(onnxruntime_USE_VSINPU)
+  list(APPEND onnxruntime_test_providers_dependencies onnxruntime_providers_vsinpu)
+endif()
+
 if(onnxruntime_USE_JSEP)
   list(APPEND onnxruntime_test_providers_dependencies onnxruntime_providers_js)
 endif()
@@ -593,6 +597,7 @@ set(ONNXRUNTIME_TEST_LIBS
     ${onnxruntime_libs}
     # CUDA, ROCM, TENSORRT, MIGRAPHX, DNNL, and OpenVINO are dynamically loaded at runtime
     ${PROVIDERS_NNAPI}
+    ${PROVIDERS_VSINPU}
     ${PROVIDERS_JS}
     ${PROVIDERS_QNN}
     ${PROVIDERS_SNPE}

docs/execution_providers/VSINPU-ExecutionProvider.md

@@ -0,0 +1,84 @@
+---
+title: Verisilicon - VSINPU
+description: Instructions to execute ONNX Runtime with VsiNpu
+parent: Execution Providers
+# nav_order: 15
+---
+{::options toc_levels="2" /}
+
+# VSINPU Execution Provider
+VsiNpu constructed with TIM-VX as an onnxruntime Execution Provider.
+
+[TIM-VX](https://github.com/VeriSilicon/TIM-VX) is a software integration module provided by VeriSilicon to facilitate deployment of Neural-Networks on VeriSilicon ML accelerators. It serves as the backend binding for runtime frameworks such as Android NN, Tensorflow-Lite, MLIR, TVM and more.
+
+## Contents
+{: .no_toc }
+
+* TOC placeholder
+{:toc}
+
+## Prepare
+
+The VSINPU Execution Provider (EP) requires that tim-vx has already been built building EP.
+
+See [here](https://github.com/VeriSilicon/TIM-VX?tab=readme-ov-file#build-and-run) for building tim-vx-build.
+
+## Build
+### Linux
+
+1. export envs
+```bash
+export TIM_VX_INSTALL=<your-path-to-tim-vx-build>/install
+export VIVANTE_SDK_DIR=<your_path_of_tim-vx>/prebuilt-sdk/x86_64_linux
+export LD_LIBRARY_PATH=$VIVANTE_SDK_DIR/lib:$TIM_VX_INSTALL/lib
+```
+
+2. build onnxruntime with "--use_vsinpu"
+
+```bash
+./build.sh --config Debug --build_shared_lib --use_vsinpu --skip_tests
+```
+
+## Test
+VSINPU EP can run onnx models (unit-tests)with onnx binary onnx_test_runner (onnxruntime_test_all).
+1. export envs
+```bash
+export TIM_VX_INSTALL=<your-path-to-tim-vx-build>/install
+export VIVANTE_SDK_DIR=<your_path_of_tim-vx>/prebuilt-sdk/x86_64_linux
+export LD_LIBRARY_PATH=$VIVANTE_SDK_DIR/lib:$TIM_VX_INSTALL/lib
+```
+2. run test
+```bash
+cd <your_path_to_onnxruntime_build>/Debug/
+./onnx_test_runner -e vsinpu <yout_path_to_onnx_model>
+
+cd <your_path_to_onnxruntime_build>/Debug/
+./onnxruntime_test_all --gtest_filter=ActivationOpTest.Sigmoid
+```
+If detailed log information is required, please add the "-v" parameter.
+
+## Supported ops
+Following ops are supported by the VSINPU Execution Provider, it should be noted that all operators do not support dynamic shape input or output.
+
+|Operator|Note|
+|--------|------|
+|ai.onnx:Abs||
+|ai.onnx:Add||
+|ai.onnx:Conv|Only 1D/2D Conv is supported yet.|
+|ai.onnx:Div||
+|ai.onnx:Exp||
+|ai.onnx:Floor||
+|ai.onnx:Gemm||
+|ai.onnx:GlobalAveragePool||
+|ai.onnx:HardSigmoid||
+|ai.onnx:HardSwish||
+|ai.onnx:LeakyRelu||
+|ai.onnx:Log||
+|ai.onnx:Mul||
+|ai.onnx:Pow||
+|ai.onnx:Relu||
+|ai.onnx:Sigmoid||
+|ai.onnx:Sin||
+|ai.onnx:Sqrt||
+|ai.onnx:Sub||
+|ai.onnx:Tanh||

include/onnxruntime/core/graph/constants.h

@@ -52,6 +52,7 @@ constexpr const char* kXnnpackExecutionProvider = "XnnpackExecutionProvider";
 constexpr const char* kWebNNExecutionProvider = "WebNNExecutionProvider";
 constexpr const char* kCannExecutionProvider = "CANNExecutionProvider";
 constexpr const char* kAzureExecutionProvider = "AzureExecutionProvider";
+constexpr const char* kVSINPUExecutionProvider = "VSINPUExecutionProvider";
 
 constexpr const char* kExecutionProviderSharedLibraryPath = "shared_lib_path";
 constexpr const char* kExecutionProviderSharedLibraryEntry = "provider_factory_entry_point";

include/onnxruntime/core/providers/vsinpu/vsinpu_provider_factory.h

@@ -0,0 +1,34 @@
+/****************************************************************************
+*
+*    Copyright (c) 2023 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************/
+#include "onnxruntime_c_api.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+ORT_API_STATUS(OrtSessionOptionsAppendExecutionProvider_VSINPU, _In_ OrtSessionOptions* options);
+
+#ifdef __cplusplus
+}
+#endif

onnxruntime/core/framework/utils.cc

@@ -61,6 +61,7 @@ bool ProviderIsCpuBased(const std::string& provider_type) {
          provider_type == onnxruntime::kVitisAIExecutionProvider ||
          provider_type == onnxruntime::kOpenVINOExecutionProvider ||
          provider_type == onnxruntime::kNnapiExecutionProvider ||
+         provider_type == onnxruntime::kVSINPUExecutionProvider ||
          provider_type == onnxruntime::kAclExecutionProvider ||
          provider_type == onnxruntime::kArmNNExecutionProvider ||
          provider_type == onnxruntime::kRknpuExecutionProvider ||

onnxruntime/core/providers/get_execution_providers.cc

@@ -98,6 +98,14 @@ constexpr ProviderInfo kProvidersInPriorityOrder[] =
             true,
 #else
             false,
+#endif
+        },
+        {
+            kVSINPUExecutionProvider,
+#ifdef USE_VSINPU
+            true,
+#else
+            false,
 #endif
         },
         {

onnxruntime/core/providers/provider_factory_creators.h

@@ -46,6 +46,10 @@
 #include "core/providers/nnapi/nnapi_provider_factory_creator.h"
 #endif
 
+#if defined(USE_VSINPU)
+#include "core/providers/vsinpu/vsinpu_provider_factory_creator.h"
+#endif
+
 #if defined(USE_JSEP)
 #include "core/providers/js/js_provider_factory_creator.h"
 #endif

onnxruntime/core/providers/vsinpu/builders/impl/activation_op_builder.h

@@ -0,0 +1,127 @@
+/****************************************************************************
+ *
+ *    Copyright (c) 2023 Vivante Corporation
+ *
+ *    Permission is hereby granted, free of charge, to any person obtaining a
+ *    copy of this software and associated documentation files (the "Software"),
+ *    to deal in the Software without restriction, including without limitation
+ *    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ *    and/or sell copies of the Software, and to permit persons to whom the
+ *    Software is furnished to do so, subject to the following conditions:
+ *
+ *    The above copyright notice and this permission notice shall be included in
+ *    all copies or substantial portions of the Software.
+ *
+ *    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ *    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ *    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ *    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ *    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ *    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ *    DEALINGS IN THE SOFTWARE.
+ *
+ *****************************************************************************/
+#include "core/providers/shared/utils/utils.h"
+#include "core/providers/vsinpu/builders/impl/base_op_builder.h"
+
+namespace onnxruntime {
+namespace vsi {
+namespace npu {
+class ReluOpBuilder : public BaseOpBuilder {
+ public:
+  bool HandleBuildOp(vsi::npu::GraphEP* graph_ep,
+                     std::vector<std::shared_ptr<tim::vx::Tensor>>& inputs,
+                     std::vector<std::shared_ptr<tim::vx::Tensor>>& outputs,
+                     const NodeUnit& node_unit) override {
+    LOGS_DEFAULT(VERBOSE) << "Creating Relu Activation.";
+    auto op = graph_ep->GetGraph()->CreateOperation<tim::vx::ops::Relu>();
+    (*op).BindInputs(inputs).BindOutputs(outputs);
+    graph_ep->GetOps().push_back(std::move(op));
+    return true;
+  }
+};
+class SigmoidOpBuilder : public BaseOpBuilder {
+ public:
+  bool HandleBuildOp(vsi::npu::GraphEP* graph_ep,
+                     std::vector<std::shared_ptr<tim::vx::Tensor>>& inputs,
+                     std::vector<std::shared_ptr<tim::vx::Tensor>>& outputs,
+                     const NodeUnit& node_unit) override {
+    LOGS_DEFAULT(VERBOSE) << "Creating Sigmoid Activation.";
+    auto op = graph_ep->GetGraph()->CreateOperation<tim::vx::ops::Sigmoid>();
+    (*op).BindInputs(inputs).BindOutputs(outputs);
+    graph_ep->GetOps().push_back(std::move(op));
+    return true;
+  }
+};
+class TanhOpBuilder : public BaseOpBuilder {
+ public:
+  bool HandleBuildOp(vsi::npu::GraphEP* graph_ep,
+                     std::vector<std::shared_ptr<tim::vx::Tensor>>& inputs,
+                     std::vector<std::shared_ptr<tim::vx::Tensor>>& outputs,
+                     const NodeUnit& node_unit) override {
+    LOGS_DEFAULT(VERBOSE) << "Creating Tanh activation.";
+    auto op = graph_ep->GetGraph()->CreateOperation<tim::vx::ops::Tanh>();
+    (*op).BindInputs(inputs).BindOutputs(outputs);
+    graph_ep->GetOps().push_back(std::move(op));
+    return true;
+  }
+};
+
+class LeakyReluOpBuilder : public BaseOpBuilder {
+ public:
+  bool HandleBuildOp(vsi::npu::GraphEP* graph_ep,
+                     std::vector<std::shared_ptr<tim::vx::Tensor>>& inputs,
+                     std::vector<std::shared_ptr<tim::vx::Tensor>>& outputs,
+                     const NodeUnit& node_unit) override {
+    LOGS_DEFAULT(VERBOSE) << "Creating LeakyRelu activation.";
+    const auto& node = node_unit.GetNode();
+    NodeAttrHelper helper(node);
+    auto alpha = helper.Get("alpha", 1.0f);
+    auto op =
+        graph_ep->GetGraph()->CreateOperation<tim::vx::ops::LeakyRelu>(alpha);
+    (*op).BindInputs(inputs).BindOutputs(outputs);
+    graph_ep->GetOps().push_back(std::move(op));
+    return true;
+  }
+};
+
+class EluOpBuilder : public BaseOpBuilder {
+ public:
+  bool HandleBuildOp(vsi::npu::GraphEP* graph_ep,
+                     std::vector<std::shared_ptr<tim::vx::Tensor>>& inputs,
+                     std::vector<std::shared_ptr<tim::vx::Tensor>>& outputs,
+                     const NodeUnit& node_unit) override {
+    LOGS_DEFAULT(VERBOSE) << "Creating Elu activation.";
+    const auto& node = node_unit.GetNode();
+    NodeAttrHelper helper(node);
+    auto alpha = helper.Get("alpha", 1.0f);
+    auto op =
+        graph_ep->GetGraph()->CreateOperation<tim::vx::ops::LeakyRelu>(alpha);
+    (*op).BindInputs(inputs).BindOutputs(outputs);
+    graph_ep->GetOps().push_back(std::move(op));
+    return true;
+  }
+};
+
+class HardSigmoidOpBuilder : public BaseOpBuilder {
+ public:
+  bool HandleBuildOp(vsi::npu::GraphEP* graph_ep,
+                     std::vector<std::shared_ptr<tim::vx::Tensor>>& inputs,
+                     std::vector<std::shared_ptr<tim::vx::Tensor>>& outputs,
+                     const NodeUnit& node_unit) override {
+    LOGS_DEFAULT(VERBOSE) << "Creating HardSigmoid activation.";
+    const auto& node = node_unit.GetNode();
+    NodeAttrHelper helper(node);
+    auto alpha = helper.Get("alpha", 1.0f);
+    auto beta = helper.Get("beta", 1.0f);
+    auto op = graph_ep->GetGraph()->CreateOperation<tim::vx::ops::HardSigmoid>(
+        alpha, beta);
+    (*op).BindInputs(inputs).BindOutputs(outputs);
+    graph_ep->GetOps().push_back(std::move(op));
+    return true;
+  }
+};
+}  // namespace npu
+
+}  // namespace vsi
+}  // namespace onnxruntime