Skip to content

Commit

Permalink
update: Enabled OV devices update parsing on runtime
Browse files Browse the repository at this point in the history
  • Loading branch information
@ankitm3k
ankitm3k committed Jul 19, 2024
1 parent 42b7ced commit 2915a0a
Show file tree
Hide file tree
Showing 4 changed files with 193 additions and 12 deletions.
2 changes: 1 addition & 1 deletion onnxruntime/core/providers/openvino/openvino_execution_provider.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ OpenVINOExecutionProvider::OpenVINOExecutionProvider(const OpenVINOExecutionProv
info.device_type_.find("AUTO") != std::string::npos) {
device_found = true;
} else {
for (std::string device : available_devices) {
for (const std::string& device : available_devices) {
if (device.rfind(info.device_type_, 0) == 0) {
if (info.device_type_.find("GPU") != std::string::npos && (info.precision_ == "FP32" ||
info.precision_ == "FP16" ||
Expand Down
44 changes: 35 additions & 9 deletions onnxruntime/core/providers/openvino/openvino_execution_provider.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,16 +16,23 @@

namespace onnxruntime {

struct OVDevices {
ov::Core core;
std::vector<std::string> get_ov_devices() const {
return core.get_available_devices();
}
};

static void print_build_options() {
std::cout << "[ERROR] INVALID DEVICE BUILD TYPE SPECIFIED" << std::endl;
std::cout << "Specify the keyword HETERO (or) MULTI (or) AUTO followed by the devices in the order of priority "
<< "you want to build"
<< std::endl;
std::cout << "The different hardware devices that can be added with HETERO/MULTI/AUTO build "
<< "are ['CPU','GPU','NPU']"
<< "are ['CPU','GPU','NPU','GPU.x'] where x = 0,1,2 and so on"
<< std::endl;
std::cout << "An example of how to specify the HETERO or MULTI or AUTO build type. "
<< "Ex: HETERO:GPU,CPU Ex: MULTI:GPU,CPU Ex: AUTO:GPU,CPU"
<< "Ex: HETERO:GPU,CPU Ex: MULTI:GPU,CPU Ex: AUTO:GPU,CPU Ex: AUTO:GPU.0,CPU Ex: AUTO:GPU.1,CPU"
<< std::endl;
}

Expand All @@ -40,7 +47,8 @@ static std::vector<std::string> split(const std::string& s, char delim) {
return result;
}

static std::vector<std::string> parseDevices(const std::string& device_string) {
static std::vector<std::string> parseDevices(const std::string& device_string,
const std::vector<std::string>& available_devices) {
std::string comma_separated_devices = device_string;
if (comma_separated_devices.find(":") != std::string::npos) {
comma_separated_devices = comma_separated_devices.substr(comma_separated_devices.find(":") + 1);
Expand All @@ -50,8 +58,15 @@ static std::vector<std::string> parseDevices(const std::string& device_string) {
print_build_options();
ORT_THROW("Invalid device string: " + device_string);
}
std::vector<std::string> dev_options = {"CPU", "GPU", "NPU"};
for (std::string dev : devices) {
std::set<std::string> dev_options = {"CPU", "GPU", "NPU"};

for (auto& device : available_devices) {
if (dev_options.find(device) == dev_options.end()) {
auto dev_options_update = dev_options.emplace(device);
}
}

for (const std::string& dev : devices) {
if (!std::count(dev_options.begin(), dev_options.end(), dev)) {
print_build_options();
ORT_THROW("Invalid device string: " + device_string);
Expand All @@ -78,8 +93,9 @@ struct OpenVINOExecutionProviderInfo {

OpenVINOExecutionProviderInfo() = delete;

explicit OpenVINOExecutionProviderInfo(std::string dev_type, std::string precision, bool enable_npu_fast_compile,
size_t num_of_threads, std::string cache_dir, std::string model_priority,
explicit OpenVINOExecutionProviderInfo(const std::string& dev_type, const std::string& precision,
bool enable_npu_fast_compile, size_t num_of_threads,
const std::string& cache_dir, const std::string& model_priority,
int num_streams, void* context, bool enable_opencl_throttling,
bool disable_dynamic_shapes, bool export_ep_ctx_blob,
bool enable_qdq_optimizer, bool disable_cpu_fallback)
Expand All @@ -97,6 +113,16 @@ struct OpenVINOExecutionProviderInfo {
disable_cpu_fallback_(disable_cpu_fallback) {
std::set<std::string> ov_supported_device_types = {"CPU", "GPU",
"GPU.0", "GPU.1", "NPU"};

OVDevices devices;
std::vector<std::string> available_devices = devices.get_ov_devices();

for (auto& device : available_devices) {
if (ov_supported_device_types.find(device) == ov_supported_device_types.end()) {
auto ov_supported_device_types_update = ov_supported_device_types.emplace(device);
}
}

if (dev_type == "") {
LOGS_DEFAULT(INFO) << "[OpenVINO-EP]"
<< "No runtime device selection option provided.";
Expand All @@ -116,7 +142,7 @@ struct OpenVINOExecutionProviderInfo {
dev_type = DEVICE;

if (dev_type.find("HETERO") == 0 || dev_type.find("MULTI") == 0 || dev_type.find("AUTO") == 0) {
std::vector<std::string> devices = parseDevices(dev_type);
std::vector<std::string> devices = parseDevices(dev_type, available_devices);
precision_ = "FP16";
if (devices[0] == "CPU") {
precision_ = "FP32";
Expand All @@ -127,7 +153,7 @@ struct OpenVINOExecutionProviderInfo {
} else if (ov_supported_device_types.find(dev_type) != ov_supported_device_types.end()) {
device_type_ = std::move(dev_type);
} else if (dev_type.find("HETERO") == 0 || dev_type.find("MULTI") == 0 || dev_type.find("AUTO") == 0) {
std::vector<std::string> devices = parseDevices(dev_type);
std::vector<std::string> devices = parseDevices(dev_type, available_devices);
device_type_ = dev_type;
} else {
ORT_THROW("Invalid device string: " + dev_type);
Expand Down
12 changes: 10 additions & 2 deletions onnxruntime/core/providers/openvino/openvino_provider_factory.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,14 @@ struct OpenVINO_Provider : Provider {
std::set<std::string> deprecated_device_types = {"CPU_FP32", "GPU_FP32",
"GPU.0_FP32", "GPU.1_FP32", "GPU_FP16",
"GPU.0_FP16", "GPU.1_FP16"};
OVDevices devices;
std::vector<std::string> available_devices = devices.get_ov_devices();

for (auto& device : available_devices) {
if (ov_supported_device_types.find(device) == ov_supported_device_types.end()) {
auto ov_supported_device_types_update = ov_supported_device_types.emplace(device);
}
}
if (deprecated_device_types.find(device_type) != deprecated_device_types.end()) {
std::string deprecated_device = device_type;
int delimit = device_type.find("_");
Expand All @@ -128,8 +136,8 @@ struct OpenVINO_Provider : Provider {
(device_type.find("MULTI:") == 0) ||
(device_type.find("AUTO:") == 0))) {
ORT_THROW(
"[ERROR] [OpenVINO] You have selcted wrong configuration value for the key 'device_type'. "
"Select from 'CPU', 'GPU', 'GPU.0', 'GPU.1', 'NPU' or from"
"[ERROR] [OpenVINO] You have selected wrong configuration value for the key 'device_type'. "
"Select from 'CPU', 'GPU', 'NPU', 'GPU.x' where x = 0,1,2 and so on or from"
" HETERO/MULTI/AUTO options available. \n");
}
}
Expand Down
147 changes: 147 additions & 0 deletions onnxruntime/test/perftest/ort_test_session.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -222,6 +222,153 @@ OnnxRuntimeTestSession::OnnxRuntimeTestSession(Ort::Env& env, std::random_device
session_options.AppendExecutionProvider_CUDA(cuda_options);
#else
ORT_THROW("TensorRT is not supported in this build\n");
#endif
} else if (provider_name_ == onnxruntime::kOpenVINOExecutionProvider) {
#ifdef USE_OPENVINO
#ifdef _MSC_VER
std::string ov_string = ToUTF8String(performance_test_config.run_config.ep_runtime_config_string);
#else
std::string ov_string = performance_test_config.run_config.ep_runtime_config_string;
#endif
std::unordered_map<std::string, std::string> ov_options;
std::istringstream ss(ov_string);
std::string token;
while (ss >> token) {
if (token == "") {
continue;
}
auto pos = token.find("|");
if (pos == std::string::npos || pos == 0 || pos == token.length()) {
ORT_THROW("[ERROR] [OpenVINO] Use a '|' to separate the key and value for the run-time option"
" you are trying to use.\n");
}

auto key = token.substr(0, pos);
auto value = token.substr(pos + 1);

if (key == "device_type") {
std::set<std::string> ov_supported_device_types = {"CPU", "GPU",
"GPU.0", "GPU.1", "NPU"};
std::set<std::string> deprecated_device_types = {"CPU_FP32", "GPU_FP32",
"GPU.0_FP32", "GPU.1_FP32", "GPU_FP16",
"GPU.0_FP16", "GPU.1_FP16"};
size_t num_gpus = 10;
for (size_t i = 0; i <= num_gpus; i++) {
ov_supported_device_types.emplace("GPU." + std::to_string(i));
}
if (ov_supported_device_types.find(value) != ov_supported_device_types.end()) {
ov_options[key] = value;
} else if (deprecated_device_types.find(value) != deprecated_device_types.end()) {
ov_options[key] = value;
} else if (value.find("HETERO:") == 0) {
ov_options[key] = value;
} else if (value.find("MULTI:") == 0) {
ov_options[key] = value;
} else if (value.find("AUTO:") == 0) {
ov_options[key] = value;
} else {
ORT_THROW(
"[ERROR] [OpenVINO] You have selected wrong configuration value for the key 'device_type'. "
"Select from 'CPU', 'GPU', 'NPU', 'GPU.x' where x = 0,1,2 and so on or from"
" HETERO/MULTI/AUTO options available. \n");
}
} else if (key == "device_id") {
if (value == "CPU" || value == "GPU" || value == "NPU") {
ov_options[key] = value;
} else {
ORT_THROW("[ERROR] [OpenVINO] Unsupported device_id is selected. Select from available options.");
}
} else if (key == "precision") {
auto device_type = ov_options["device_type"];
if (device_type.find("GPU") != std::string::npos) {
if (value == "") {
ov_options[key] = "FP16";
continue;
} else if (value == "ACCURACY" || value == "FP16" || value == "FP32") {
ov_options[key] = value;
continue;
} else {
ORT_THROW(
"[ERROR] [OpenVINO] Unsupported inference precision is selected. "
"GPU only supported FP32 / FP16. \n");
}
} else if (device_type.find("NPU") != std::string::npos) {
if (value == "" || value == "ACCURACY" || value == "FP16") {
ov_options[key] = "FP16";
continue;
} else {
ORT_THROW("[ERROR] [OpenVINO] Unsupported inference precision is selected. NPU only supported FP16. \n");
}
} else if (device_type.find("CPU") != std::string::npos) {
if (value == "" || value == "ACCURACY" || value == "FP32") {
ov_options[key] = "FP32";
continue;
} else {
ORT_THROW("[ERROR] [OpenVINO] Unsupported inference precision is selected. CPU only supports FP32 . \n");
}
}
} else if (key == "enable_npu_fast_compile") {
if (value == "true" || value == "True" ||
value == "false" || value == "False") {
ov_options[key] = value;
} else {
ORT_THROW("[ERROR] [OpenVINO] The value for the key 'enable_npu_fast_compile' should be a boolean"
" i.e. true or false. Default value is false.\n");
}
} else if (key == "enable_opencl_throttling") {
if (value == "true" || value == "True" ||
value == "false" || value == "False") {
ov_options[key] = value;
} else {
ORT_THROW("[ERROR] [OpenVINO] The value for the key 'enable_opencl_throttling' should be a boolean"
" i.e. true or false. Default value is false.\n");
}
} else if (key == "disable_dynamic_shapes") {
if (value == "true" || value == "True" ||
value == "false" || value == "False") {
ov_options[key] = value;
} else {
ORT_THROW(
"[ERROR] [OpenVINO] The value for the key 'enable_dynamic_shapes' "
"should be a boolean i.e. true or false. Default value is false.\n");
}
} else if (key == "num_of_threads") {
if (std::stoi(value) <= 0) {
ORT_THROW("[ERROR] [OpenVINO] The value for the key 'num_of_threads' should be greater than 0\n");
} else {
ov_options[key] = value;
}
} else if (key == "model_priority") {
ov_options[key] = value;
} else if (key == "cache_dir") {
ov_options[key] = value;
} else if (key == "context") {
ov_options[key] = value;
} else if (key == "num_streams") {
if (std::stoi(value) <= 0 && std::stoi(value) > 8) {
ORT_THROW("[ERROR] [OpenVINO] The value for the key 'num_streams' should be in the range of 1-8 \n");
} else {
ov_options[key] = value;
}
} else if (key == "export_ep_ctx_blob") {
if (value == "true" || value == "True" ||
value == "false" || value == "False") {
ov_options[key] = value;
} else {
ORT_THROW(
"[ERROR] [OpenVINO] The value for the key 'export_ep_ctx_blob' "
"should be a boolean i.e. true or false. Default value is false.\n");
}
} else {
ORT_THROW("[ERROR] [OpenVINO] wrong key type entered."
" Choose from the following runtime key options that are available for OpenVINO."
" ['device_type', 'device_id', 'enable_npu_fast_compile', 'num_of_threads', 'cache_dir',"
" 'num_streams', 'enable_opencl_throttling', 'disable_dynamic_shapes'] \n");
}
}
session_options.AppendExecutionProvider_OpenVINO_V2(ov_options);
#else
ORT_THROW("OpenVINO is not supported in this build\n");
#endif
} else if (provider_name_ == onnxruntime::kQnnExecutionProvider) {
#ifdef USE_QNN
Expand Down

0 comments on commit 2915a0a

Please sign in to comment.