Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Wenyi560_ov_runtime_GPU #2132

Closed
wants to merge 5 commits into from
Closed

Wenyi560_ov_runtime_GPU #2132

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
670e9d7
ov_runtime supports device configuration
wenyi5608 Nov 8, 2023
323d944
upgrade ov version to 2023.1.0
wenyi5608 Nov 8, 2023
954117f
ov runtime set device
wenyi5608 Nov 8, 2023
8cdaa76
ov runtime set device
wenyi5608 Nov 8, 2023
da2b532
support WeTextProcessing
wenyi5608 Nov 8, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 17 additions & 14 deletions runtime/core/cmake/openvino.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,33 +15,36 @@ endfunction()

if(OPENVINO)
include(gflags)
set(VINO_VERSION "2022.3.0")
set(VINO_VERSION "2023.1.0")
if(${CMAKE_SYSTEM_PROCESSOR} MATCHES "(x86)|(X86)|(amd64)|(AMD64)")
set (X86 TRUE)
else ()
set (X86 FALSE)
endif ()

if(${CMAKE_SYSTEM_NAME} STREQUAL "Windows")
set(VINO_URL "https://storage.openvinotoolkit.org/repositories/openvino/packages/2022.3/windows/w_openvino_toolkit_windows_${VINO_VERSION}.9052.9752fafe8eb_x86_64.zip")
set(URL_HASH "SHA256=adf42b4ffe4d8a39ffebcb476ea8b84e51eefbb5b900db70510412beed2cc722")
set(VINO_URL "https://storage.openvinotoolkit.org/repositories/openvino/packages/2023.1/windows/w_openvino_toolkit_windows_${VINO_VERSION}.12185.47b736f63ed_x86_64.zip")
set(URL_HASH "SHA256=1ec70bc8e05c7dbb77b8b96a6420595a45d2d52992b27a93f03821be2377e13a")
elseif(${CMAKE_SYSTEM_NAME} STREQUAL "Linux")
get_linux_lsb_release_information()
if(LSB_RELEASE_ID_SHORT STREQUAL "Ubuntu" AND LSB_RELEASE_VERSION_SHORT VERSION_GREATER_EQUAL 20 AND LSB_RELEASE_VERSION_SHORT VERSION_LESS 22)
set(VINO_URL "https://storage.openvinotoolkit.org/repositories/openvino/packages/2022.3/linux/l_openvino_toolkit_ubuntu20_${VINO_VERSION}.9052.9752fafe8eb_x86_64.tgz")
set(URL_HASH "SHA256=e421fc1d9c7e6cce4089aaf344c7f84dcfd8f81ebf4ef2130b145aeca48c4e71")
if(LSB_RELEASE_ID_SHORT STREQUAL "Ubuntu" AND LSB_RELEASE_VERSION_SHORT VERSION_GREATER_EQUAL 22 AND LSB_RELEASE_VERSION_SHORT VERSION_LESS 24)
set(VINO_URL "https://storage.openvinotoolkit.org/repositories/openvino/packages/2023.1/linux/l_openvino_toolkit_ubuntu22_${VINO_VERSION}.12185.47b736f63ed_x86_64.tgz")
set(URL_HASH "SHA256=7d7f1ba703dfe1fc89b49832c1eace31ed66f4a05dd5d07693c942f0e9bf56c3")
elseif(LSB_RELEASE_ID_SHORT STREQUAL "Ubuntu" AND LSB_RELEASE_VERSION_SHORT VERSION_GREATER_EQUAL 20 AND LSB_RELEASE_VERSION_SHORT VERSION_LESS 22)
set(VINO_URL "https://storage.openvinotoolkit.org/repositories/openvino/packages/2023.1/linux/l_openvino_toolkit_ubuntu20_${VINO_VERSION}.12185.47b736f63ed_x86_64.tgz")
set(URL_HASH "SHA256=1ca9abda5c99a21e46e3e996cc02e4493df1b97096f3a8e07d74ec11f8b40321")
elseif(LSB_RELEASE_ID_SHORT STREQUAL "Ubuntu" AND LSB_RELEASE_VERSION_SHORT VERSION_GREATER_EQUAL 18 AND LSB_RELEASE_VERSION_SHORT VERSION_LESS 20)
set(VINO_URL "https://storage.openvinotoolkit.org/repositories/openvino/packages/2022.3/linux/l_openvino_toolkit_ubuntu18_${VINO_VERSION}.9052.9752fafe8eb_x86_64.tgz")
set(URL_HASH "SHA256=8337579883d216336572e35066b6c8de52ea5a41f6771a3344ee41c019d27372")
set(VINO_URL "https://storage.openvinotoolkit.org/repositories/openvino/packages/2023.1/linux/l_openvino_toolkit_ubuntu18_${VINO_VERSION}.12185.47b736f63ed_x86_64.tgz")
set(URL_HASH "SHA256=ac5a6503875e922850f820cbc772cf2fc96474befe39a3993e9b1700cfe6670d")
elseif(LSB_RELEASE_ID_SHORT STREQUAL "CentOS" AND LSB_RELEASE_VERSION_SHORT VERSION_LESS 7.10)
set(VINO_URL "https://storage.openvinotoolkit.org/repositories/openvino/packages/2022.3/linux/l_openvino_toolkit_centos7_${VINO_VERSION}.9052.9752fafe8eb_x86_64.tgz")
set(URL_HASH "SHA256=93d6bd23d5e8306384bdc8f3746fe56baad1a0f87b97d0b384b4c0b158227d76")
set(VINO_URL "https://storage.openvinotoolkit.org/repositories/openvino/packages/2023.1/linux/l_openvino_toolkit_centos7_${VINO_VERSION}.12185.47b736f63ed_x86_64.tgz")
set(URL_HASH "SHA256=ac18d6c10f2e12040d3ff21dc2150343ca496188bf5abb6f7c67b7f12e4b888c")
elseif(LSB_RELEASE_ID_SHORT STREQUAL "CentOS" AND LSB_RELEASE_VERSION_SHORT VERSION_LESS 8.10)
set(VINO_URL "https://storage.openvinotoolkit.org/repositories/openvino/packages/2022.3/linux/l_openvino_toolkit_rhel8_${VINO_VERSION}.9052.9752fafe8eb_x86_64.tgz")
set(URL_HASH "SHA256=519cbdf95d73489c771357f27dbc5829057fcc3b23fd89be06e290b41a8ff934")
set(VINO_URL "https://storage.openvinotoolkit.org/repositories/openvino/packages/2023.1/linux/l_openvino_toolkit_rhel8_${VINO_VERSION}.12185.47b736f63ed_x86_64.tgz")
set(URL_HASH "SHA256=100230f4079dede9a67aa2d772b90e6e5378c244105bed1246143e9c54854993")
elseif(LSB_RELEASE_ID_SHORT STREQUAL "Debian" AND LSB_RELEASE_VERSION_SHORT VERSION_LESS_EQUAL 9.13 AND X86)
set(VINO_URL "https://storage.openvinotoolkit.org/repositories/openvino/packages/2022.3/linux/l_openvino_toolkit_debian9_${VINO_VERSION}.9052.9752fafe8eb_arm64.tgz")
set(URL_HASH "SHA256=4099bb9a257af46fbf7d5c30414ec4e308542704eac666b4b213362208bf8b16")
set(VINO_URL "https://storage.openvinotoolkit.org/repositories/openvino/packages/2023.1/linux/l_openvino_toolkit_debian9_${VINO_VERSION}.12185.47b736f63ed_arm64.tgz")
set(URL_HASH "SHA256=0eb1409d475ebb34631a834f535f5f81292cbcf0da6d4825852473960ce321d1")
else()
message(STATUS "Linux OS without exist binary, compile OpenVINO source.")
endif()
Expand Down
2 changes: 2 additions & 0 deletions runtime/core/decoder/params.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@ DEFINE_string(bpu_model_dir, "",
// OVAsrModel flags
DEFINE_string(openvino_dir, "", "directory where the OV model is saved");
DEFINE_int32(core_number, 1, "Core number of process");
DEFINE_string(openvino_device, "CPU", "set OpenVINO Device for ASR model");

// FeaturePipelineConfig flags
DEFINE_int32(num_bins, 80, "num mel bins for fbank feature");
Expand Down Expand Up @@ -191,6 +192,7 @@ std::shared_ptr<DecodeResource> InitDecodeResourceFromFlags() {
LOG(INFO) << "Read OpenVINO model ";
auto model = std::make_shared<OVAsrModel>();
model->InitEngineThreads(FLAGS_core_number);
model->SetDeviceName(FLAGS_openvino_device);
model->Read(FLAGS_openvino_dir);
resource->model = model;
#else
Expand Down
5 changes: 5 additions & 0 deletions runtime/openvino/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ option(TORCH "whether to build with Torch" OFF)
option(ONNX "whether to build with ONNX" OFF)
option(GPU "whether to build with GPU" OFF)
option(OPENVINO "whether to build with OpenVINO" ON)
option(ITN "whether to use WeTextProcessing" ON)

set(CMAKE_VERBOSE_MAKEFILE OFF)

Expand All @@ -41,6 +42,10 @@ include_directories(
${CMAKE_CURRENT_SOURCE_DIR}/kaldi
)

if(ITN)
include(wetextprocessing)
endif()

# Build all libraries
add_subdirectory(utils)
add_subdirectory(frontend)
Expand Down
19 changes: 12 additions & 7 deletions runtime/openvino/ov/ov_asr_model.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,9 +66,14 @@ static void PrintPerformanceCounts(

OVAsrModel::~OVAsrModel() {}

void OVAsrModel::SetDeviceName(const std::string& device_name){
device_name_=device_name;
}

void OVAsrModel::InitEngineThreads(int core_number) {
core_ = std::make_shared<ov::Core>();
if (core_) {
if (core_ && device_name_ == "CPU") {
std::cout << "OV device d " << device_name_ << std::endl;
core_->set_property("CPU", ov::num_streams(core_number));
core_->set_property("CPU", ov::affinity(ov::Affinity::NONE));
core_->set_property("CPU", ov::inference_num_threads(1));
Expand All @@ -87,11 +92,11 @@ void OVAsrModel::Read(const std::string& model_dir) {
std::shared_ptr<ov::Model> encoder_model =
core_->read_model(encoder_ir_path);
std::map<std::string, ov::AnyMap> config;
config["CPU"] = {};
config[device_name_] = {};
if (getenv("OPENVINO_PROFILE")) {
config["CPU"].emplace(ov::enable_profiling(true));
config[device_name_].emplace(ov::enable_profiling(true));
}
config["CPU"].emplace(
config[device_name_].emplace(
ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT));
// set some configurations if any
for (auto&& item : config) {
Expand Down Expand Up @@ -130,7 +135,7 @@ void OVAsrModel::Read(const std::string& model_dir) {
}

encoder_compile_model_ = std::make_shared<ov::CompiledModel>(std::move(
core_->compile_model(encoder_model, "CPU")));
core_->compile_model(encoder_model, device_name_)));
// {{"PERF_COUNT", "NO"} /* YES for profile */
// })));

Expand All @@ -145,7 +150,7 @@ void OVAsrModel::Read(const std::string& model_dir) {
std::shared_ptr<ov::Model> ctc_model = core_->read_model(ctc_ir_path);
if (ctc_model) {
ctc_compile_model_ = std::make_shared<ov::CompiledModel>(std::move(
core_->compile_model(ctc_model, "CPU")));
core_->compile_model(ctc_model, device_name_)));
// {{"PERFORMANCE_HINT", "THROUGHPUT"},
// {"PERFORMANCE_HINT_NUM_REQUESTS", 1}})));

Expand All @@ -161,7 +166,7 @@ void OVAsrModel::Read(const std::string& model_dir) {
core_->read_model(rescore_ir_path);
if (rescore_model) {
rescore_compile_model_ = std::make_shared<ov::CompiledModel>(std::move(
core_->compile_model(rescore_model, "CPU")));
core_->compile_model(rescore_model, device_name_)));
// {{"PERFORMANCE_HINT", "THROUGHPUT"},
// {"PERFORMANCE_HINT_NUM_REQUESTS", 1}})));

Expand Down
2 changes: 2 additions & 0 deletions runtime/openvino/ov/ov_asr_model.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ class OVAsrModel : public AsrModel {
float reverse_weight,
std::vector<float>* rescoring_score) override;
std::shared_ptr<AsrModel> Copy() const override;
void SetDeviceName(const std::string& device_name);

protected:
void ForwardEncoderFunc(const std::vector<std::vector<float>>& chunk_feats,
Expand All @@ -44,6 +45,7 @@ class OVAsrModel : public AsrModel {
int num_blocks_ = 0;
int cnn_module_kernel_ = 0;
int head_ = 0;
std::string device_name_;

std::shared_ptr<ov::Core> core_;

Expand Down
Loading