diff --git a/Makefile b/Makefile index 96d50d81ba..71bac7bd74 100644 --- a/Makefile +++ b/Makefile @@ -40,6 +40,7 @@ run_hello_ovms: MEDIAPIPE_UNSTABLE_TESTS_REGEX="MuxInputStreamHandlerTest.RemovesUnusedDataStreamPackets" run_unit_tests: docker run -e http_proxy=$(HTTP_PROXY) -e https_proxy=$(HTTPS_PROXY) $(OVMS_MEDIA_DOCKER_IMAGE):$(OVMS_MEDIA_IMAGE_TAG) bazel test --define=MEDIAPIPE_DISABLE_GPU=1 --test_output=streamed --test_filter="-${MEDIAPIPE_UNSTABLE_TESTS_REGEX}" //mediapipe/framework/... + docker run -e http_proxy=$(HTTP_PROXY) -e https_proxy=$(HTTPS_PROXY) $(OVMS_MEDIA_DOCKER_IMAGE):$(OVMS_MEDIA_IMAGE_TAG) bazel test --define=MEDIAPIPE_DISABLE_GPU=1 --test_output=streamed //mediapipe/calculators/ovms:all run_hello_world: docker run $(OVMS_MEDIA_DOCKER_IMAGE):$(OVMS_MEDIA_IMAGE_TAG) bazel-bin/mediapipe/examples/desktop/hello_world/hello_world diff --git a/mediapipe/calculators/ovms/BUILD b/mediapipe/calculators/ovms/BUILD index 880e49d727..f867c98269 100644 --- a/mediapipe/calculators/ovms/BUILD +++ b/mediapipe/calculators/ovms/BUILD @@ -47,7 +47,7 @@ cc_library( "modelapiovmsadapter.hpp" ], deps = [ - "//mediapipe/framework/port:logging", # TODO remove logs but need better error handling/reporting in Model API + "//mediapipe/framework/port:logging", "@ovms//src:ovms_header", "@model_api//:model_api", "@linux_openvino//:openvino", @@ -145,8 +145,8 @@ cc_test( "test_data/config.json", "test_data/dummy/1/dummy.bin", "test_data/dummy/1/dummy.xml", - "test_data/add_two_inputs_model/1/dummy.bin", - "test_data/add_two_inputs_model/1/dummy.xml", + "test_data/add_two_inputs_model/1/add.bin", + "test_data/add_two_inputs_model/1/add.xml", ], copts = ["-Iexternal/ovms/src","-Isrc"], ) diff --git a/mediapipe/calculators/ovms/modelapiovmsadapter.cc b/mediapipe/calculators/ovms/modelapiovmsadapter.cc index 1460771613..d3989530d8 100644 --- a/mediapipe/calculators/ovms/modelapiovmsadapter.cc +++ b/mediapipe/calculators/ovms/modelapiovmsadapter.cc @@ -67,14 +67,11 @@ using std::endl; #pragma GCC diagnostic ignored "-Wunused-function" using InferenceOutput = std::map; using InferenceInput = std::map; -// TODO -// * why std::map -// * no ret code from infer() -// * no ret code from load() + namespace ovms { static OVMS_DataType OVPrecision2CAPI(ov::element::Type_t datatype); static ov::element::Type_t CAPI2OVPrecision(OVMS_DataType datatype); -static ov::Tensor makeOvTensorO(OVMS_DataType datatype, const int64_t* shape, size_t dimCount, const void* voutputData, size_t bytesize); +static ov::Tensor makeOvTensor(OVMS_DataType datatype, const int64_t* shape, size_t dimCount, const void* voutputData, size_t bytesize); OVMSInferenceAdapter::OVMSInferenceAdapter(const std::string& servableName, uint32_t servableVersion, OVMS_Server* cserver) : servableName(servableName), @@ -103,31 +100,22 @@ InferenceOutput OVMSInferenceAdapter::infer(const InferenceInput& input) { // PREPARE EACH INPUT // extract single tensor for (const auto& [name, input_tensor] : input) { - // TODO validate existence of tag key in map - // or handle inference when there is no need for mapping const char* realInputName = name.c_str(); -#if 0 - const float* input_tensor_access = reinterpret_cast(input_tensor.data()); - std::stringstream ss; - ss << " Adapter received tensor: [ "; - for (int x = 0; x < 10; ++x) { - ss << input_tensor_access[x] << " "; - } - ss << " ]"; - LOG(INFO) << ss.str(); -#endif const auto& ovinputShape = input_tensor.get_shape(); - std::vector inputShape{ovinputShape.begin(), ovinputShape.end()}; // TODO error handling shape conversion + if (std::any_of(ovinputShape.begin(), ovinputShape.end(), [](size_t dim) { + return dim > std::numeric_limits::max();})) { + throw std::runtime_error("Cannot use C-API with dimension size greater than int64_t max value"); + } + std::vector inputShape{ovinputShape.begin(), ovinputShape.end()}; OVMS_DataType inputDataType = OVPrecision2CAPI(input_tensor.get_element_type()); - ASSERT_CAPI_STATUS_NULL(OVMS_InferenceRequestAddInput(request, realInputName, inputDataType, inputShape.data(), inputShape.size())); // TODO retcode + ASSERT_CAPI_STATUS_NULL(OVMS_InferenceRequestAddInput(request, realInputName, inputDataType, inputShape.data(), inputShape.size())); const uint32_t NOT_USED_NUM = 0; - // TODO handle hardcoded buffertype, notUsedNum additional options? side packets? ASSERT_CAPI_STATUS_NULL(OVMS_InferenceRequestInputSetData(request, realInputName, reinterpret_cast(input_tensor.data()), input_tensor.get_byte_size(), OVMS_BUFFERTYPE_CPU, - NOT_USED_NUM)); // TODO retcode + NOT_USED_NUM)); } ////////////////// // INFERENCE @@ -147,13 +135,10 @@ InferenceOutput OVMSInferenceAdapter::infer(const InferenceInput& input) { return output; } CREATE_GUARD(responseGuard, OVMS_InferenceResponse, response); - // verify GetOutputCount uint32_t outputCount = 42; ASSERT_CAPI_STATUS_NULL(OVMS_InferenceResponseOutputCount(response, &outputCount)); uint32_t parameterCount = 42; ASSERT_CAPI_STATUS_NULL(OVMS_InferenceResponseParameterCount(response, ¶meterCount)); - // TODO handle output filtering. Graph definition could suggest - // that we are not interested in all outputs from OVMS Inference const void* voutputData; size_t bytesize = 42; OVMS_DataType datatype = (OVMS_DataType)199; @@ -164,18 +149,19 @@ InferenceOutput OVMSInferenceAdapter::infer(const InferenceInput& input) { const char* outputName{nullptr}; for (size_t i = 0; i < outputCount; ++i) { ASSERT_CAPI_STATUS_NULL(OVMS_InferenceResponseOutput(response, i, &outputName, &datatype, &shape, &dimCount, &voutputData, &bytesize, &bufferType, &deviceId)); - output[outputName] = makeOvTensorO(datatype, shape, dimCount, voutputData, bytesize); // TODO optimize FIXME + output[outputName] = makeOvTensor(datatype, shape, dimCount, voutputData, bytesize); } return output; } + void OVMSInferenceAdapter::loadModel(const std::shared_ptr& model, ov::Core& core, const std::string& device, const ov::AnyMap& compilationConfig) { // no need to load but we need to extract metadata OVMS_ServableMetadata* servableMetadata = nullptr; ASSERT_CAPI_STATUS_NULL(OVMS_GetServableMetadata(cserver, servableName.c_str(), servableVersion, &servableMetadata)); + CREATE_GUARD(metadataGuard, OVMS_ServableMetadata, servableMetadata); uint32_t inputCount = 0; uint32_t outputCount = 0; - // TODO ensure Metadata object removal in all paths ASSERT_CAPI_STATUS_NULL(OVMS_ServableMetadataInputCount(servableMetadata, &inputCount)); ASSERT_CAPI_STATUS_NULL(OVMS_ServableMetadataOutputCount(servableMetadata, &outputCount)); @@ -190,7 +176,6 @@ void OVMSInferenceAdapter::loadModel(const std::shared_ptr& mod inputNames.emplace_back(tensorName); shape_min_max_t inputMinMax; for (size_t i = 0; i < dimCount; ++i) { - // TODO test adapter dynamic shapes inputMinMax.first.emplace_back(shapeMin[i]); inputMinMax.second.emplace_back(shapeMax[i]); } @@ -203,7 +188,6 @@ void OVMSInferenceAdapter::loadModel(const std::shared_ptr& mod const ov::AnyMap* servableMetadataRtInfo; ASSERT_CAPI_STATUS_NULL(OVMS_ServableMetadataInfo(servableMetadata, reinterpret_cast(&servableMetadataRtInfo))); this->modelConfig = *servableMetadataRtInfo; - OVMS_ServableMetadataDelete(servableMetadata); } ov::PartialShape OVMSInferenceAdapter::getInputShape(const std::string& inputName) const { @@ -294,7 +278,7 @@ static ov::element::Type_t CAPI2OVPrecision(OVMS_DataType datatype) { return it->second; } -static ov::Tensor makeOvTensorO(OVMS_DataType datatype, const int64_t* shape, size_t dimCount, const void* voutputData, size_t bytesize) { +static ov::Tensor makeOvTensor(OVMS_DataType datatype, const int64_t* shape, size_t dimCount, const void* voutputData, size_t bytesize) { ov::Shape ovShape; for (size_t i = 0; i < dimCount; ++i) { ovShape.push_back(shape[i]); diff --git a/mediapipe/calculators/ovms/modelapiovmsadapter.hpp b/mediapipe/calculators/ovms/modelapiovmsadapter.hpp index 3c677324ff..cb96dd51de 100644 --- a/mediapipe/calculators/ovms/modelapiovmsadapter.hpp +++ b/mediapipe/calculators/ovms/modelapiovmsadapter.hpp @@ -23,7 +23,7 @@ #include #include -#include // TODO fix path model_api/model_api/cpp/adapters/include/adapters/inference_adapter.h +#include // model_api/model_api/cpp/adapters/include/adapters/inference_adapter.h #include // here we need to decide if we have several calculators (1 for OVMS repository, 1-N inside mediapipe) @@ -37,8 +37,6 @@ namespace ovms { using InferenceOutput = std::map; using InferenceInput = std::map; -// TODO -// * why std::map using shape_border_t = std::vector; using shape_min_max_t = std::pair; using shapes_min_max_t = std::unordered_map; diff --git a/mediapipe/calculators/ovms/openvinoinferencecalculator.cc b/mediapipe/calculators/ovms/openvinoinferencecalculator.cc index 735a7e8405..4af8f23d9d 100644 --- a/mediapipe/calculators/ovms/openvinoinferencecalculator.cc +++ b/mediapipe/calculators/ovms/openvinoinferencecalculator.cc @@ -20,7 +20,7 @@ #include #include -#include // TODO fix path model_api/model_api/cpp/adapters/include/adapters/inference_adapter.h +#include // model_api/model_api/cpp/adapters/include/adapters/inference_adapter.h #include #include @@ -317,7 +317,7 @@ static ov::Tensor convertTFLiteTensor2OVTensor(const TfLiteTensor& t) { ov::Shape shape; // for some reason TfLite tensor does not have bs dim shape.emplace_back(1); - // TODO: Support scalars and no data tensors with 0-dim + // No support for scalars and no data tensors with 0-dim for (int i = 0; i < t.dims->size; ++i) { shape.emplace_back(t.dims->data[i]); } @@ -570,7 +570,6 @@ class OpenVINOInferenceCalculator : public CalculatorBase { cc->Outputs().Tag(tag).Add( tensors.release(), cc->InputTimestamp()); - //break; // TODO FIXME order of outputs // no need to break since we only have one tag // create concatenator calc } else if (startsWith(tag, MPTENSORS_TAG)) { @@ -599,11 +598,9 @@ class OpenVINOInferenceCalculator : public CalculatorBase { cc->Outputs().Tag(tag).Add( tensors.release(), cc->InputTimestamp()); - //break; // TODO FIXME order of outputs // no need to break since we only have one tag // create concatenator calc } else if (startsWith(tag, TFLITE_TENSORS_TAG)) { - // TODO FIXME use output_order_list LOG(INFO) << "OVMS calculator will process vector"; auto outputStreamTensors = std::vector(); if (!this->initialized) { @@ -619,7 +616,7 @@ class OpenVINOInferenceCalculator : public CalculatorBase { } interpreter_->SetTensorParametersReadWrite( tensorId, - kTfLiteFloat32, // TODO datatype + kTfLiteFloat32, name.c_str(), tfliteshape, TfLiteQuantization()); diff --git a/mediapipe/calculators/ovms/openvinomodelserversessioncalculator.proto b/mediapipe/calculators/ovms/openvinomodelserversessioncalculator.proto index eb63b15df0..73e7e23fd6 100644 --- a/mediapipe/calculators/ovms/openvinomodelserversessioncalculator.proto +++ b/mediapipe/calculators/ovms/openvinomodelserversessioncalculator.proto @@ -29,5 +29,7 @@ message OpenVINOModelServerSessionCalculatorOptions { optional string servable_version = 2; // service_url: "13.21.212.171:9718" optional string service_url = 3; + // config_path: "/models/config.json" + // when this field is used ensure that each calculator is using the same file optional string server_config = 4; } diff --git a/mediapipe/calculators/ovms/openvinomodelserversessioncalculator_test.cc b/mediapipe/calculators/ovms/openvinomodelserversessioncalculator_test.cc index 38955124e2..8d76d3ac0b 100644 --- a/mediapipe/calculators/ovms/openvinomodelserversessioncalculator_test.cc +++ b/mediapipe/calculators/ovms/openvinomodelserversessioncalculator_test.cc @@ -23,7 +23,7 @@ #include "mediapipe/framework/port/gtest.h" -#include // TODO fix path model_api/model_api/cpp/adapters/include/adapters/inference_adapter.h +#include // model_api/model_api/cpp/adapters/include/adapters/inference_adapter.h #include #include diff --git a/mediapipe/calculators/ovms/ovms_calculator.cc b/mediapipe/calculators/ovms/ovms_calculator.cc deleted file mode 100644 index 94249169ac..0000000000 --- a/mediapipe/calculators/ovms/ovms_calculator.cc +++ /dev/null @@ -1,281 +0,0 @@ -//***************************************************************************** -// Copyright 2023 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#include -#include -#include - -#include - -#include "../ovms.h" // NOLINT -#include "../stringutils.hpp" // TODO dispose - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wdeprecated-declarations" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/port/canonical_errors.h" -#pragma GCC diagnostic pop -#include "src/mediapipe_calculators/ovmscalculator.pb.h" -// here we need to decide if we have several calculators (1 for OVMS repository, 1-N inside mediapipe) -// for the one inside OVMS repo it makes sense to reuse code from ovms lib -namespace mediapipe { -using std::endl; - -namespace { -#define ASSERT_CAPI_STATUS_NULL(C_API_CALL) \ - { \ - auto* err = C_API_CALL; \ - if (err != nullptr) { \ - uint32_t code = 0; \ - const char* msg = nullptr; \ - OVMS_StatusCode(err, &code); \ - OVMS_StatusDetails(err, &msg); \ - LOG(INFO) << "Error encountred in OVMSCalculator:" << msg << " code: " << code; \ - OVMS_StatusDelete(err); \ - RET_CHECK(err == nullptr); \ - } \ - } -#define CREATE_GUARD(GUARD_NAME, CAPI_TYPE, CAPI_PTR) \ - std::unique_ptr GUARD_NAME(CAPI_PTR, &(CAPI_TYPE##Delete)); - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wunused-function" -static ov::element::Type_t CAPI2OVPrecision(OVMS_DataType datatype) { - static std::unordered_map precisionMap{ - {OVMS_DATATYPE_FP64, ov::element::Type_t::f64}, - {OVMS_DATATYPE_FP32, ov::element::Type_t::f32}, - {OVMS_DATATYPE_FP16, ov::element::Type_t::f16}, - {OVMS_DATATYPE_I64, ov::element::Type_t::i64}, - {OVMS_DATATYPE_I32, ov::element::Type_t::i32}, - {OVMS_DATATYPE_I16, ov::element::Type_t::i16}, - {OVMS_DATATYPE_I8, ov::element::Type_t::i8}, - {OVMS_DATATYPE_I4, ov::element::Type_t::i4}, - {OVMS_DATATYPE_U64, ov::element::Type_t::u64}, - {OVMS_DATATYPE_U32, ov::element::Type_t::u32}, - {OVMS_DATATYPE_U16, ov::element::Type_t::u16}, - {OVMS_DATATYPE_U8, ov::element::Type_t::u8}, - {OVMS_DATATYPE_U4, ov::element::Type_t::u4}, - {OVMS_DATATYPE_U1, ov::element::Type_t::u1}, - {OVMS_DATATYPE_BOOL, ov::element::Type_t::boolean}, - {OVMS_DATATYPE_BF16, ov::element::Type_t::bf16}, - {OVMS_DATATYPE_UNDEFINED, ov::element::Type_t::undefined}, - {OVMS_DATATYPE_DYNAMIC, ov::element::Type_t::dynamic} - // {OVMS_DATATYPE_MIXED, ov::element::Type_t::MIXED}, - // {OVMS_DATATYPE_Q78, ov::element::Type_t::Q78}, - // {OVMS_DATATYPE_BIN, ov::element::Type_t::BIN}, - // {OVMS_DATATYPE_CUSTOM, ov::element::Type_t::CUSTOM - }; - auto it = precisionMap.find(datatype); - if (it == precisionMap.end()) { - return ov::element::Type_t::undefined; - } - return it->second; -} - -static OVMS_DataType OVPrecision2CAPI(ov::element::Type_t datatype) { - static std::unordered_map precisionMap{ - {ov::element::Type_t::f64, OVMS_DATATYPE_FP64}, - {ov::element::Type_t::f32, OVMS_DATATYPE_FP32}, - {ov::element::Type_t::f16, OVMS_DATATYPE_FP16}, - {ov::element::Type_t::i64, OVMS_DATATYPE_I64}, - {ov::element::Type_t::i32, OVMS_DATATYPE_I32}, - {ov::element::Type_t::i16, OVMS_DATATYPE_I16}, - {ov::element::Type_t::i8, OVMS_DATATYPE_I8}, - {ov::element::Type_t::i4, OVMS_DATATYPE_I4}, - {ov::element::Type_t::u64, OVMS_DATATYPE_U64}, - {ov::element::Type_t::u32, OVMS_DATATYPE_U32}, - {ov::element::Type_t::u16, OVMS_DATATYPE_U16}, - {ov::element::Type_t::u8, OVMS_DATATYPE_U8}, - {ov::element::Type_t::u4, OVMS_DATATYPE_U4}, - {ov::element::Type_t::u1, OVMS_DATATYPE_U1}, - {ov::element::Type_t::boolean, OVMS_DATATYPE_BOOL}, - {ov::element::Type_t::bf16, OVMS_DATATYPE_BF16}, - {ov::element::Type_t::undefined, OVMS_DATATYPE_UNDEFINED}, - {ov::element::Type_t::dynamic, OVMS_DATATYPE_DYNAMIC} - // {ov::element::Type_t::, OVMS_DATATYPE_MIXEDMIXED}, - // {ov::element::Type_t::, OVMS_DATATYPE_Q78Q78}, - // {ov::element::Type_t::, OVMS_DATATYPE_BINBIN}, - // {ov::element::Type_t::, OVMS_DATATYPE_CUSTOMCUSTOM - }; - auto it = precisionMap.find(datatype); - if (it == precisionMap.end()) { - return OVMS_DATATYPE_UNDEFINED; - } - return it->second; -} - -static ov::Tensor* makeOvTensor(OVMS_DataType datatype, const int64_t* shape, uint32_t dimCount, const void* voutputData, size_t bytesize) { - ov::Shape ovShape; - for (size_t i = 0; i < dimCount; ++i) { - ovShape.push_back(shape[i]); - } - // here we make copy of underlying OVMS repsonse tensor - ov::Tensor* output = new ov::Tensor(CAPI2OVPrecision(datatype), ovShape); - std::memcpy(output->data(), voutputData, bytesize); - return output; -} -static ov::Tensor makeOvTensorO(OVMS_DataType datatype, const int64_t* shape, uint32_t dimCount, const void* voutputData, size_t bytesize) { - ov::Shape ovShape; - for (size_t i = 0; i < dimCount; ++i) { - ovShape.push_back(shape[i]); - } - // here we make copy of underlying OVMS repsonse tensor - ov::Tensor output(CAPI2OVPrecision(datatype), ovShape); - std::memcpy(output.data(), voutputData, bytesize); - return output; -} -} // namespace - -class OVMSOVCalculator : public CalculatorBase { - OVMS_Server* cserver{nullptr}; - OVMS_ServerSettings* _serverSettings{nullptr}; - OVMS_ModelsSettings* _modelsSettings{nullptr}; - std::unordered_map outputNameToTag; - -public: - static absl::Status GetContract(CalculatorContract* cc) { - RET_CHECK(!cc->Inputs().GetTags().empty()); - RET_CHECK(!cc->Outputs().GetTags().empty()); - for (const std::string& tag : cc->Inputs().GetTags()) { - cc->Inputs().Tag(tag).Set(); - } - for (const std::string& tag : cc->Outputs().GetTags()) { - cc->Outputs().Tag(tag).Set(); - } - const auto& options = cc->Options(); - RET_CHECK(!options.servable_name().empty()); - // TODO validate version from string - // TODO validate service url format - RET_CHECK(options.config_path().empty() || - options.service_url().empty()); - // TODO validate tag_to_tensor maps so that key fulfill regex - return absl::OkStatus(); - } - - absl::Status Close(CalculatorContext* cc) final { - const auto& options = cc->Options(); - if (!options.config_path().empty()) { - OVMS_ModelsSettingsDelete(_modelsSettings); - OVMS_ServerSettingsDelete(_serverSettings); - // Close is called on input node and output node in initial pipeline - // Commented out since for now this happens twice in 2 nodes graph. Server will close - // OVMS_ServerDelete(cserver); TODO this should happen onlif graph is used once - // moreover we may need several ovms calculators use in graph each providing its own model? how to handle then different model inputs, as wel as config? - } - return absl::OkStatus(); - } - absl::Status Open(CalculatorContext* cc) final { - for (CollectionItemId id = cc->Inputs().BeginId(); id < cc->Inputs().EndId(); ++id) { - if (!cc->Inputs().Get(id).Header().IsEmpty()) { - cc->Outputs().Get(id).SetHeader(cc->Inputs().Get(id).Header()); - } - } - if (cc->OutputSidePackets().NumEntries() != 0) { - for (CollectionItemId id = cc->InputSidePackets().BeginId(); id < cc->InputSidePackets().EndId(); ++id) { - cc->OutputSidePackets().Get(id).Set(cc->InputSidePackets().Get(id)); - } - } - cc->SetOffset(TimestampDiff(0)); - - const auto& options = cc->Options(); - OVMS_ServerNew(&cserver); - if (!options.config_path().empty()) { - OVMS_ServerSettingsNew(&_serverSettings); - OVMS_ModelsSettingsNew(&_modelsSettings); - OVMS_ModelsSettingsSetConfigPath(_modelsSettings, options.config_path().c_str()); - OVMS_ServerSettingsSetLogLevel(_serverSettings, OVMS_LOG_DEBUG); - OVMS_ServerStartFromConfigurationFile(cserver, _serverSettings, _modelsSettings); - } - for (const auto& [key, value] : options.tag_to_output_tensor_names()) { - outputNameToTag[value] = key; - } - return absl::OkStatus(); - } - - absl::Status Process(CalculatorContext* cc) final { - const auto& options = cc->Options(); - ///////////////////// - // PREPARE REQUEST - ///////////////////// - OVMS_InferenceRequest* request{nullptr}; - auto servableVersionOpt = ovms::stou32(options.servable_version().c_str()); - uint64_t servableVersion = servableVersionOpt.value_or(0); - ASSERT_CAPI_STATUS_NULL(OVMS_InferenceRequestNew(&request, cserver, options.servable_name().c_str(), servableVersion)); - CREATE_GUARD(requestGuard, OVMS_InferenceRequest, request); - - // PREPARE EACH INPUT - // extract single tensor - const auto inputTagInputMap = options.tag_to_input_tensor_names(); - const auto inputTagOutputMap = options.tag_to_output_tensor_names(); - for (const std::string& tag : cc->Inputs().GetTags()) { - // TODO validate existence of tag key in map - const char* realInputName = inputTagInputMap.at(tag).c_str(); - - auto& packet = cc->Inputs().Tag(tag).Get(); - ov::Tensor input_tensor(packet); - const float* input_tensor_access = reinterpret_cast(input_tensor.data()); - std::stringstream ss; - ss << "Calculator received tensor: [ "; - for (int x = 0; x < 10; ++x) { - ss << input_tensor_access[x] << " "; - } - ss << " ] timestamp: " << cc->InputTimestamp().DebugString(); - LOG(INFO) << ss.str(); - const auto& ovInputShape = input_tensor.get_shape(); - std::vector inputShape(ovInputShape.begin(), ovInputShape.end()); // TODO ensure ov tensors shapes conversions return error in all calcs - OVMS_DataType inputDataType = OVPrecision2CAPI(input_tensor.get_element_type()); - ASSERT_CAPI_STATUS_NULL(OVMS_InferenceRequestAddInput(request, realInputName, inputDataType, inputShape.data(), inputShape.size())); - const uint32_t notUsedNum = 0; - // TODO handle hardcoded buffertype, notUsedNum additional options? side packets? - ASSERT_CAPI_STATUS_NULL(OVMS_InferenceRequestInputSetData(request, - realInputName, - reinterpret_cast(input_tensor.data()), - input_tensor.get_byte_size(), - OVMS_BUFFERTYPE_CPU, - notUsedNum)); - } - ////////////////// - // INFERENCE - ////////////////// - OVMS_InferenceResponse* response = nullptr; - ASSERT_CAPI_STATUS_NULL(OVMS_Inference(cserver, request, &response)); - CREATE_GUARD(responseGuard, OVMS_InferenceResponse, response); - // verify GetOutputCount - uint32_t outputCount = 42; - ASSERT_CAPI_STATUS_NULL(OVMS_InferenceResponseOutputCount(response, &outputCount)); - RET_CHECK(outputCount == cc->Outputs().GetTags().size()); - uint32_t parameterCount = 42; - ASSERT_CAPI_STATUS_NULL(OVMS_InferenceResponseParameterCount(response, ¶meterCount)); - // TODO handle output filtering. Graph definition could suggest - // that we are not interested in all outputs from OVMS Inference - const void* voutputData; - size_t bytesize = 42; - OVMS_DataType datatype = (OVMS_DataType)199; - const int64_t* shape{nullptr}; - size_t dimCount = 42; - OVMS_BufferType bufferType = (OVMS_BufferType)199; - uint32_t deviceId = 42; - const char* outputName{nullptr}; - for (size_t i = 0; i < outputCount; ++i) { - ASSERT_CAPI_STATUS_NULL(OVMS_InferenceResponseOutput(response, i, &outputName, &datatype, &shape, &dimCount, &voutputData, &bytesize, &bufferType, &deviceId)); - ov::Tensor* outOvTensor = makeOvTensor(datatype, shape, dimCount, voutputData, bytesize); - cc->Outputs().Tag(outputNameToTag.at(outputName)).Add(outOvTensor, cc->InputTimestamp()); - } - return absl::OkStatus(); - } -}; -#pragma GCC diagnostic pop -REGISTER_CALCULATOR(OVMSOVCalculator); -} // namespace mediapipe diff --git a/mediapipe/calculators/ovms/ovmscalculator.proto b/mediapipe/calculators/ovms/ovmscalculator.proto deleted file mode 100644 index 3d2920560a..0000000000 --- a/mediapipe/calculators/ovms/ovmscalculator.proto +++ /dev/null @@ -1,49 +0,0 @@ -//***************************************************************************** -// Copyright 2023 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** - -syntax = "proto2"; -package mediapipe; - -import "mediapipe/framework/calculator.proto"; - -message OVMSCalculatorOptions { - extend mediapipe.CalculatorOptions { - // https://github.com/google/mediapipe/issues/634 have to be unique in app - // no rule to obtain this - optional OVMSCalculatorOptions ext = 113473742; - } - required string servable_name = 1; - // [1-9][0-9]* - required string servable_version = 2; // TODO decide if follow KFS proto or use int - // keep mappings separately in case input/output tensor have the same name -/* - tag_to_input_tensor_names {$ - key: "INPUT"$ - value: "input_tensor/input_features:0"$ - }$ - tag_to_input_tensor_names {$ - key: "DATA"$ - value: "data_tensor"$ - }$ -*/ - map tag_to_input_tensor_names = 3; - map tag_to_output_tensor_names = 4; - // service_url: "13.21.212.171:9718" - optional string service_url = 5; - // config_path: "/models/config.json" - // TODO decide if it should be at all possible since OVMS does not have a C-API way to stop and then restart - optional string config_path = 6; -} diff --git a/mediapipe/calculators/ovms/test_data/add_two_inputs_model/1/add.bin b/mediapipe/calculators/ovms/test_data/add_two_inputs_model/1/add.bin new file mode 100644 index 0000000000..e69de29bb2 diff --git a/mediapipe/calculators/ovms/test_data/add_two_inputs_model/1/add.xml b/mediapipe/calculators/ovms/test_data/add_two_inputs_model/1/add.xml new file mode 100644 index 0000000000..a901c2e546 --- /dev/null +++ b/mediapipe/calculators/ovms/test_data/add_two_inputs_model/1/add.xml @@ -0,0 +1,91 @@ + + + + + + + + 1 + 10 + + + + + + + + 1 + 10 + + + + + + + 1 + 10 + + + 1 + 10 + + + + + 1 + 10 + + + + + + + 1 + 10 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +