From 974ebc2d95b9af13d7fd12886c229816dda53e78 Mon Sep 17 00:00:00 2001 From: ismukhin Date: Mon, 9 Oct 2023 22:40:34 +0300 Subject: [PATCH 01/18] raw tvm inference --- src/inference/inference_tvm.py | 176 ++++++++++++++++++++++++++++++ src/inference/io_model_wrapper.py | 4 + src/inference/transformer.py | 4 + src/inference/tvm_auxiliary.py | 48 ++++++++ 4 files changed, 232 insertions(+) create mode 100644 src/inference/inference_tvm.py create mode 100644 src/inference/tvm_auxiliary.py diff --git a/src/inference/inference_tvm.py b/src/inference/inference_tvm.py new file mode 100644 index 000000000..8ee65085f --- /dev/null +++ b/src/inference/inference_tvm.py @@ -0,0 +1,176 @@ +import argparse +import json +import logging as log +import os +import sys +import traceback +import warnings +import mxnet +from pathlib import Path +from time import time + +import tvm + +import postprocessing_data as pp +from inference_tools.loop_tools import loop_inference, get_exec_time +from io_adapter import IOAdapter +from io_model_wrapper import TVMIOModelWrapper +from transformer import TVMTransformer +from reporter.report_writer import ReportWriter +from tvm_auxiliary import (TVMConverter, create_dict_for_converter, + prepare_output) + + +def cli_argument_parser(): + parser = argparse.ArgumentParser() + parser.add_argument('-mn', '--model_name', + help='Model name to download using packages from framework.', + type=str, + dest='model_name') + parser.add_argument('-m', '--model', + help='Path to an .json file with a trained model.', + type=str, + dest='model_path') + parser.add_argument('-w', '--weights', + help='Path to an .params file with a trained weights.', + type=str, + dest='model_params') + parser.add_argument('-d', '--device', + help='Specify the target device to infer on CPU or ' + 'NVIDIA_GPU (CPU by default)', + default='CPU', + type=str, + dest='device') + parser.add_argument('-f', '--framework', + help='Model name to download using GluonCV package.', + type=str, + dest='framework') + parser.add_argument('-ni', '--number_iter', + help='Number of inference iterations.', + default=1, + type=int, + dest='number_iter') + parser.add_argument('--output_names', + help='Name of the output tensors.', + default='output0', + type=str, + nargs='+', + dest='output_names') + parser.add_argument('-t', '--task', + help='Task type determines the type of output processing ' + 'method. Available values: feedforward - without' + 'postprocessing (by default), classification - output' + 'is a vector of probabilities.', + choices=['feedforward', 'classification'], + default='feedforward', + type=str, + dest='task') + parser.add_argument('-i', '--input', + help='Path to data.', + required=True, + type=str, + nargs='+', + dest='input') + parser.add_argument('-in', '--input_name', + help='Input name.', + default='data', + type=str, + dest='input_name') + parser.add_argument('-is', '--input_shape', + help='Input shape BxWxHxC, B is a batch size,' + 'W is an input tensor width,' + 'H is an input tensor height,' + 'C is an input tensor number of channels.', + required=True, + type=int, + nargs=4, + dest='input_shape') + parser.add_argument('--norm', + help='Flag to normalize input images' + '(use --mean and --std arguments to set' + 'required normalization parameters).', + action='store_true', + dest='norm') + parser.add_argument('--mean', + help='Mean values.', + default=[0, 0, 0], + type=float, + nargs=3, + dest='mean') + parser.add_argument('--std', + help='Standard deviation values.', + default=[1., 1., 1.], + type=float, + nargs=3, + dest='std') + parser.add_argument('-nt', '--number_top', + help='Number of top results to print', + default=5, + type=int, + dest='number_top') + parser.add_argument('-b', '--batch_size', + help='Batch size.', + default=1, + type=int, + dest='batch_size') + parser.add_argument('--channel_swap', + help='Parameter of channel swap (WxHxC to CxWxH by default).', + default=[2, 0, 1], + type=int, + nargs=3, + dest='channel_swap') + args = parser.parse_args() + return args + +def create_dict_for_transformer(args): + dictionary = { + 'channel_swap': args.channel_swap, + 'mean': args.mean, + 'std': args.std, + 'norm': args.norm, + 'input_shape': args.input_shape, + 'batch_size': args.batch_size, + } + return dictionary + + +def create_dict_for_modelwrapper(args): + dictionary = { + 'input_name': args.input_name, + 'input_shape': [args.batch_size] + args.input_shape[1:4], + 'model_name': args.model_name, + } + return dictionary + + +def inference_tvm(module, num_of_iteration, input_name, get_slice): + if num_of_iteration == 1: + slice_input = get_slice() + module.set_input(input_name, slice_input[input_name].asnumpy()) + module.run() + result = module.get_output(0) + return result + + +def main(): + log.basicConfig( + format='[ %(levelname)s ] %(message)s', + level=log.INFO, + stream=sys.stdout, + ) + args = cli_argument_parser() + report_writer = ReportWriter() + wrapper = TVMIOModelWrapper(create_dict_for_modelwrapper(args)) + transformer = TVMTransformer(create_dict_for_transformer(args)) + io = IOAdapter.get_io_adapter(args, wrapper, transformer) + converter = TVMConverter(create_dict_for_converter(args)) + graph_module = converter.convert_model_from_framework() + io.prepare_input(graph_module, args.input) + res = inference_tvm(graph_module, args.number_iter, args.input_name, io.get_slice_input_mxnet) + io.process_output(prepare_output(res, args.task, args.output_names), log) + + + + +if __name__ == '__main__': + sys.exit(main() or 0) \ No newline at end of file diff --git a/src/inference/io_model_wrapper.py b/src/inference/io_model_wrapper.py index ff31569b4..7d3a8ec47 100644 --- a/src/inference/io_model_wrapper.py +++ b/src/inference/io_model_wrapper.py @@ -252,3 +252,7 @@ def get_input_layer_dtype(self, model, layer_name): from numpy import float32 return float32 + + +class TVMIOModelWrapper(MXNetIOModelWrapper): + pass diff --git a/src/inference/transformer.py b/src/inference/transformer.py index 99ac3dde4..32340b35e 100644 --- a/src/inference/transformer.py +++ b/src/inference/transformer.py @@ -264,6 +264,10 @@ class ONNXRuntimeTransformer(TensorFlowLiteTransformer): pass +class TVMTransformer(MXNetTransformer): + pass + + class OnnxRuntimeTransformerCpp(Transformer): def __init__(self, model): self._model = model diff --git a/src/inference/tvm_auxiliary.py b/src/inference/tvm_auxiliary.py new file mode 100644 index 000000000..e38291d7f --- /dev/null +++ b/src/inference/tvm_auxiliary.py @@ -0,0 +1,48 @@ +import tvm +import logging as log +from scipy.special import softmax + + +class TVMConverter: + def __init__(self, args): + self.args = args + self.net = None + + def convert_model_from_framework(self): + if self.args['framework'] == 'mxnet': + import mxnet, gluoncv + if self.args['device'] == 'CPU': + context = mxnet.cpu() + target = 'llvm' + dev = tvm.device(target, 0) + model_name = self.args['model_name'] + log.info(f'Loading network \"{model_name}\" from GluonCV model zoo') + net = gluoncv.model_zoo.get_model(model_name, pretrained=True, ctx=context) + shape_dict = {self.args['input_name']: self.args['input_shape']} + model, params = tvm.relay.frontend.from_mxnet(net, shape_dict) + with tvm.transform.PassContext(opt_level=3): + lib = tvm.relay.build(model, target=target, params=params) + module = tvm.contrib.graph_executor.GraphModule(lib["default"](dev)) + return module + + +def create_dict_for_converter(args): + dictionary = { + 'input_name': args.input_name, + 'input_shape': [args.batch_size] + args.input_shape[1:4], + 'model_name': args.model_name, + 'model_path': args.model_path, + 'model_params': args.model_params, + 'framework': args.framework, + 'device': args.device, + } + return dictionary + + +def prepare_output(result, task, output_names): + if task == 'classification': + return {output_names[0]: softmax(result.numpy())} + + + + \ No newline at end of file From b1cbed784ec9ba4534d216f491639d5ac810164d Mon Sep 17 00:00:00 2001 From: ismukhin Date: Sat, 14 Oct 2023 16:47:42 +0300 Subject: [PATCH 02/18] pytorch, onnx and time measurements --- ...nference_tvm.py => inference_tvm_mxnet.py} | 85 ++++++++++--------- src/inference/io_model_wrapper.py | 20 ++++- src/inference/transformer.py | 42 ++++++++- src/inference/tvm_auxiliary.py | 74 ++++++++++++++-- 4 files changed, 171 insertions(+), 50 deletions(-) rename src/inference/{inference_tvm.py => inference_tvm_mxnet.py} (72%) diff --git a/src/inference/inference_tvm.py b/src/inference/inference_tvm_mxnet.py similarity index 72% rename from src/inference/inference_tvm.py rename to src/inference/inference_tvm_mxnet.py index 8ee65085f..1aaef87e6 100644 --- a/src/inference/inference_tvm.py +++ b/src/inference/inference_tvm_mxnet.py @@ -6,25 +6,25 @@ import traceback import warnings import mxnet + from pathlib import Path from time import time -import tvm - import postprocessing_data as pp from inference_tools.loop_tools import loop_inference, get_exec_time from io_adapter import IOAdapter from io_model_wrapper import TVMIOModelWrapper from transformer import TVMTransformer from reporter.report_writer import ReportWriter -from tvm_auxiliary import (TVMConverter, create_dict_for_converter, - prepare_output) +from tvm_auxiliary import (TVMConverter, create_dict_for_converter_mxnet, + prepare_output, create_dict_for_modelwrapper, + create_dict_for_transformer) def cli_argument_parser(): parser = argparse.ArgumentParser() parser.add_argument('-mn', '--model_name', - help='Model name to download using packages from framework.', + help='Model name to download using packages from GluonCV.', type=str, dest='model_name') parser.add_argument('-m', '--model', @@ -41,10 +41,6 @@ def cli_argument_parser(): default='CPU', type=str, dest='device') - parser.add_argument('-f', '--framework', - help='Model name to download using GluonCV package.', - type=str, - dest='framework') parser.add_argument('-ni', '--number_iter', help='Number of inference iterations.', default=1, @@ -76,6 +72,9 @@ def cli_argument_parser(): default='data', type=str, dest='input_name') + parser.add_argument('--time', required=False, default=0, type=int, + dest='time', + help='Optional. Time in seconds to execute topology.') parser.add_argument('-is', '--input_shape', help='Input shape BxWxHxC, B is a batch size,' 'W is an input tensor width,' @@ -119,37 +118,42 @@ def cli_argument_parser(): type=int, nargs=3, dest='channel_swap') + parser.add_argument('--report_path', + type=Path, + default=Path(__file__).parent / 'tvm_inference_report.json', + dest='report_path') args = parser.parse_args() return args -def create_dict_for_transformer(args): - dictionary = { - 'channel_swap': args.channel_swap, - 'mean': args.mean, - 'std': args.std, - 'norm': args.norm, - 'input_shape': args.input_shape, - 'batch_size': args.batch_size, - } - return dictionary +def inference_tvm(module, num_of_iterations, input_name, get_slice, test_duration): + result = None + time_infer = [] + if num_of_iterations == 1: + slice_input = get_slice() + t0 = time() + module.set_input(input_name, slice_input[input_name]) + module.run() + result = module.get_output(0) + t1 = time() + time_infer.append(t1 - t0) + else: + time_infer = loop_inference(num_of_iterations, test_duration)(inference_iteration)(get_slice, input_name, module) + return result, time_infer -def create_dict_for_modelwrapper(args): - dictionary = { - 'input_name': args.input_name, - 'input_shape': [args.batch_size] + args.input_shape[1:4], - 'model_name': args.model_name, - } - return dictionary +def inference_iteration(get_slice, input_name, module): + slice_input = get_slice() + _, exec_time = infer_slice(input_name, module, slice_input) + return exec_time -def inference_tvm(module, num_of_iteration, input_name, get_slice): - if num_of_iteration == 1: - slice_input = get_slice() - module.set_input(input_name, slice_input[input_name].asnumpy()) - module.run() - result = module.get_output(0) - return result + +@get_exec_time() +def infer_slice(input_name, module, slice_input): + module.set_input(input_name, slice_input[input_name]) + module.run() + res = module.get_output(0) + return res def main(): @@ -163,12 +167,17 @@ def main(): wrapper = TVMIOModelWrapper(create_dict_for_modelwrapper(args)) transformer = TVMTransformer(create_dict_for_transformer(args)) io = IOAdapter.get_io_adapter(args, wrapper, transformer) - converter = TVMConverter(create_dict_for_converter(args)) - graph_module = converter.convert_model_from_framework() + converter = TVMConverter(create_dict_for_converter_mxnet(args)) + graph_module = converter.convert_model_from_framework('mxnet') io.prepare_input(graph_module, args.input) - res = inference_tvm(graph_module, args.number_iter, args.input_name, io.get_slice_input_mxnet) - io.process_output(prepare_output(res, args.task, args.output_names), log) - + result, infer_time = inference_tvm(graph_module, args.number_iter, args.input_name, io.get_slice_input, args.time) + if args.number_iter == 1: + io.process_output(prepare_output(result, args.task, args.output_names), log) + inference_result = pp.calculate_performance_metrics_sync_mode(args.batch_size, infer_time) + report_writer.update_execution_results(**inference_result) + report_writer.write_report(args.report_path) + log.info('Performance results') + log.info(f'Performance results:\n{json.dumps(inference_result, indent=4)}') diff --git a/src/inference/io_model_wrapper.py b/src/inference/io_model_wrapper.py index 7d3a8ec47..4c5bb2e19 100644 --- a/src/inference/io_model_wrapper.py +++ b/src/inference/io_model_wrapper.py @@ -254,5 +254,21 @@ def get_input_layer_dtype(self, model, layer_name): return float32 -class TVMIOModelWrapper(MXNetIOModelWrapper): - pass +class TVMIOModelWrapper(IOModelWrapper): + def __init__(self, args): + self._input_names = [args['input_name']] + self._input_shapes = [args['input_shape']] + self._model_name = args['model_name'] + + def get_input_layer_names(self, model): + return self._input_names + + def get_input_layer_shape(self, model, layer_name): + return self._input_shapes[0] + + def get_input_layer_dtype(self, model, layer_name): + import numpy as np + return np.float32 + + def get_model_name(self): + return self._model_name diff --git a/src/inference/transformer.py b/src/inference/transformer.py index 32340b35e..ed64742f1 100644 --- a/src/inference/transformer.py +++ b/src/inference/transformer.py @@ -188,7 +188,6 @@ def __init__(self, converting): def __set_norm(self, image): import mxnet - if self._converting['norm'] is True: mean = mxnet.nd.array([self._converting['mean'][0], self._converting['mean'][1], @@ -264,8 +263,45 @@ class ONNXRuntimeTransformer(TensorFlowLiteTransformer): pass -class TVMTransformer(MXNetTransformer): - pass +class TVMTransformer(Transformer): + def __init__(self, converting): + self._converting = converting + + def __set_norm(self, image): + std = np.array([self._converting['std'][0], + self._converting['std'][1], + self._converting['std'][2]]) + mean = np.array([self._converting['mean'][0], + self._converting['mean'][1], + self._converting['mean'][2]]) + for i in range(image.shape[2]): + image[:, :, i] /= 255 + image[:, :, i] -= mean[i] + image[:, :, i] /= std[i] + return image + + def __set_channel_swap(self, image): + if self._converting['channel_swap'] is not None: + transposing_form = (self._converting['channel_swap'][0], + self._converting['channel_swap'][1], + self._converting['channel_swap'][2]) + transposed_image = image.transpose(transposing_form) + return transposed_image + return image + + def _transform(self, image, element_type): + transformed_image = np.copy(image).astype(element_type) + normalized_image = self.__set_norm(transformed_image) + transposed_image = self.__set_channel_swap(normalized_image) + return transposed_image + + def transform_images(self, images, shape, element_type, input_name): + dataset_size = images.shape[0] + new_shape = [dataset_size] + shape[1:] + transformed_images = np.zeros(shape=new_shape, dtype=element_type) + for i in range(dataset_size): + transformed_images[i] = self._transform(images[i], element_type) + return transformed_images class OnnxRuntimeTransformerCpp(Transformer): diff --git a/src/inference/tvm_auxiliary.py b/src/inference/tvm_auxiliary.py index e38291d7f..0522e6b3d 100644 --- a/src/inference/tvm_auxiliary.py +++ b/src/inference/tvm_auxiliary.py @@ -8,13 +8,20 @@ def __init__(self, args): self.args = args self.net = None - def convert_model_from_framework(self): - if self.args['framework'] == 'mxnet': + def _get_target_device(self): + if self.args['device'] == 'CPU': + target = tvm.target.Target('llvm') + dev = tvm.cpu(0) + return target, dev + + def convert_model_from_framework(self, framework): + + target, dev = self._get_target_device() + + if framework == 'mxnet': import mxnet, gluoncv if self.args['device'] == 'CPU': context = mxnet.cpu() - target = 'llvm' - dev = tvm.device(target, 0) model_name = self.args['model_name'] log.info(f'Loading network \"{model_name}\" from GluonCV model zoo') net = gluoncv.model_zoo.get_model(model_name, pretrained=True, ctx=context) @@ -24,24 +31,77 @@ def convert_model_from_framework(self): lib = tvm.relay.build(model, target=target, params=params) module = tvm.contrib.graph_executor.GraphModule(lib["default"](dev)) return module + + elif framework == 'pytorch': + import torch, torchvision + model_name = self.args['model_name'] + pt_model = getattr(torchvision.models, model_name)(pretrained=True) + pt_model = pt_model.eval() + input_shape = self.args['input_shape'] + input_data = torch.randn(input_shape) + scripted_model = torch.jit.trace(pt_model, input_data).eval() + input_name = self.args['input_name'] + shape_list = [(input_name, input_shape)] + model, params = tvm.relay.frontend.from_pytorch(scripted_model, shape_list) + with tvm.transform.PassContext(opt_level=3): + lib = tvm.relay.build(model, target=target, params=params) + module = tvm.contrib.graph_executor.GraphModule(lib["default"](dev)) + return module + + elif framework == 'onnx': + import onnx + model_path = self.args['model_name'] + model_onnx = onnx.load(model_path) + shape_dict = {self.args['input_name']: self.args['input_shape']} + model, params = tvm.relay.frontend.from_onnx(model_onnx, shape_dict) + with tvm.transform.PassContext(opt_level=3): + lib = tvm.relay.build(model, target=target, params=params) + module = tvm.contrib.graph_executor.GraphModule(lib["default"](dev)) + return module + + + -def create_dict_for_converter(args): + +def create_dict_for_converter_mxnet(args): dictionary = { 'input_name': args.input_name, 'input_shape': [args.batch_size] + args.input_shape[1:4], 'model_name': args.model_name, 'model_path': args.model_path, 'model_params': args.model_params, - 'framework': args.framework, 'device': args.device, } return dictionary +def create_dict_for_transformer(args): + dictionary = { + 'channel_swap': args.channel_swap, + 'mean': args.mean, + 'std': args.std, + 'norm': args.norm, + 'input_shape': args.input_shape, + 'batch_size': args.batch_size, + } + return dictionary + + +def create_dict_for_modelwrapper(args): + dictionary = { + 'input_name': args.input_name, + 'input_shape': [args.batch_size] + args.input_shape[1:4], + 'model_name': args.model_name, + } + return dictionary + + def prepare_output(result, task, output_names): + if task == 'feedforward': + return {} if task == 'classification': - return {output_names[0]: softmax(result.numpy())} + return {output_names[0]: softmax(result.asnumpy())} From e7692687aae08418c53134610323a045737e16f7 Mon Sep 17 00:00:00 2001 From: ismukhin Date: Wed, 18 Oct 2023 19:04:41 +0300 Subject: [PATCH 03/18] template for tvm benchmark --- src/benchmark/config_parser_factory.py | 3 +++ src/benchmark/frameworks/known_frameworks.py | 1 + src/benchmark/frameworks/tvm/__init__.py | 0 src/benchmark/frameworks/tvm/tvm_parameters_parser.py | 5 +++++ src/benchmark/frameworks/tvm/tvm_wrapper.py | 0 5 files changed, 9 insertions(+) create mode 100644 src/benchmark/frameworks/tvm/__init__.py create mode 100644 src/benchmark/frameworks/tvm/tvm_parameters_parser.py create mode 100644 src/benchmark/frameworks/tvm/tvm_wrapper.py diff --git a/src/benchmark/config_parser_factory.py b/src/benchmark/config_parser_factory.py index 5470b7165..85024f9fc 100644 --- a/src/benchmark/config_parser_factory.py +++ b/src/benchmark/config_parser_factory.py @@ -8,6 +8,7 @@ from frameworks.pytorch.pytorch_parameters_parser import PyTorchParametersParser from frameworks.onnx_runtime_python.onnx_runtime_python_parameters_parser import ONNXRuntimePythonParametersParser from frameworks.config_parser.dependent_parameters_parser_cpp import CppParametersParser +from frameworks.tvm.tvm_parameters_parser import TVMParametersParser def get_parameters_parser(framework): @@ -35,4 +36,6 @@ def get_parameters_parser(framework): return PyTorchParametersParser() if framework == KnownFrameworks.pytorch_cpp: return CppParametersParser() + if framework == KnownFrameworks.tvm: + return TVMParametersParser() raise NotImplementedError(f'Unknown framework {framework}') diff --git a/src/benchmark/frameworks/known_frameworks.py b/src/benchmark/frameworks/known_frameworks.py index 2f39898a0..c8ca0ca80 100644 --- a/src/benchmark/frameworks/known_frameworks.py +++ b/src/benchmark/frameworks/known_frameworks.py @@ -11,3 +11,4 @@ class KnownFrameworks: opencv_dnn_cpp = 'OpenCV DNN Cpp' pytorch = 'PyTorch' pytorch_cpp = 'PyTorch Cpp' + tvm = 'TVM' diff --git a/src/benchmark/frameworks/tvm/__init__.py b/src/benchmark/frameworks/tvm/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/benchmark/frameworks/tvm/tvm_parameters_parser.py b/src/benchmark/frameworks/tvm/tvm_parameters_parser.py new file mode 100644 index 000000000..fa075a6d1 --- /dev/null +++ b/src/benchmark/frameworks/tvm/tvm_parameters_parser.py @@ -0,0 +1,5 @@ +from ..config_parser.dependent_parameters_parser import DependentParametersParser +from ..config_parser.framework_parameters_parser import FrameworkParameters + +class TVMParametersParser(DependentParametersParser): + pass \ No newline at end of file diff --git a/src/benchmark/frameworks/tvm/tvm_wrapper.py b/src/benchmark/frameworks/tvm/tvm_wrapper.py new file mode 100644 index 000000000..e69de29bb From 9d48c08a8b15701e80fdc56c726b7b6c96bbff9b Mon Sep 17 00:00:00 2001 From: ismukhin Date: Thu, 19 Oct 2023 22:18:04 +0300 Subject: [PATCH 04/18] some features --- .../frameworks/tvm/tvm_parameters_parser.py | 63 ++++++++++++++++++- 1 file changed, 62 insertions(+), 1 deletion(-) diff --git a/src/benchmark/frameworks/tvm/tvm_parameters_parser.py b/src/benchmark/frameworks/tvm/tvm_parameters_parser.py index fa075a6d1..7814abd45 100644 --- a/src/benchmark/frameworks/tvm/tvm_parameters_parser.py +++ b/src/benchmark/frameworks/tvm/tvm_parameters_parser.py @@ -2,4 +2,65 @@ from ..config_parser.framework_parameters_parser import FrameworkParameters class TVMParametersParser(DependentParametersParser): - pass \ No newline at end of file + def parse_parameters(self, curr_test): + CONFIG_FRAMEWORK_DEPENDENT_TAG = 'FrameworkDependent' + CONFIG_FRAMEWORK_DEPENDENT_NAME_OF_FRAMEWORK_TAG = 'Framework' + CONFIG_FRAMEWORK_DEPENDENT_INPUT_NAME_TAG = 'InputName' + CONFIG_FRAMEWORK_DEPENDENT_INPUT_SHAPE_TAG = 'InputShape' + CONFIG_FRAMEWORK_DEPENDENT_NORMALIZE_TAG = 'Normalize' + CONFIG_FRAMEWORK_DEPENDENT_MEAN_TAG = 'Mean' + CONFIG_FRAMEWORK_DEPENDENT_STD_TAG = 'Std' + CONFIG_FRAMEWORK_DEPENDENT_CHANNEL_SWAP_TAG = 'ChannelSwap' + + dep_parameters_tag = curr_test.getElementsByTagName(CONFIG_FRAMEWORK_DEPENDENT_TAG)[0] + + _framework = dep_parameters_tag.getElementsByTagName( + CONFIG_FRAMEWORK_DEPENDENT_NAME_OF_FRAMEWORK_TAG)[0].firstChild + _input_name = dep_parameters_tag.getElementsByTagName( + CONFIG_FRAMEWORK_DEPENDENT_INPUT_NAME_TAG)[0].firstChild + _input_shape = dep_parameters_tag.getElementsByTagName( + CONFIG_FRAMEWORK_DEPENDENT_INPUT_SHAPE_TAG)[0].firstChild + _normalize = dep_parameters_tag.getElementsByTagName( + CONFIG_FRAMEWORK_DEPENDENT_NORMALIZE_TAG)[0].firstChild + _mean = dep_parameters_tag.getElementsByTagName( + CONFIG_FRAMEWORK_DEPENDENT_MEAN_TAG)[0].firstChild + _std = dep_parameters_tag.getElementsByTagName( + CONFIG_FRAMEWORK_DEPENDENT_STD_TAG)[0].firstChild + _channel_swap = dep_parameters_tag.getElementsByTagName( + CONFIG_FRAMEWORK_DEPENDENT_CHANNEL_SWAP_TAG)[0].firstChild + + +class TVMParameters(FrameworkParameters): + def __init__(self, framework, input_name, input_shape, + normalize, mean, std, channel_swap): + self.framework = None + self.input_name = None + self.input_shape = None + self.normalize = None + self.mean = None + self.std = None + self.channel_swap = None + + if self._framework_is_correct(framework): + self.framework = framework + if self._parameter_is_not_none(input_name): + self.input_name = input_name + if self._parameter_is_not_none(input_shape): + self.input_shape = input_shape + if self._parameter_is_not_none(normalize): + self.normalize = normalize + if self._parameter_is_not_none(mean): + self.mean = mean + if self._parameter_is_not_none(std): + self.std = std + if self._parameter_is_not_none(channel_swap): + self.channel_swap = channel_swap + + @staticmethod + def _framework_is_correct(framework): + correct_frameworks = ['mxnet', 'onnx', 'tvm', + 'tf', 'tflite', 'pytorch'] + if framework.lower() in correct_frameworks: + return True + else: + raise ValueError(f'Framework is required parameter. TVM support: {", ".join(correct_frameworks)}') \ No newline at end of file From 2006dd9cec37719527dbc21f498e4d14eb159c37 Mon Sep 17 00:00:00 2001 From: ismukhin Date: Fri, 20 Oct 2023 15:03:33 +0300 Subject: [PATCH 05/18] benchmark support --- requirements_frameworks.txt | 3 +- .../frameworks/framework_wrapper_registry.py | 2 + .../frameworks/tvm/tvm_parameters_parser.py | 10 +++ src/benchmark/frameworks/tvm/tvm_process.py | 90 +++++++++++++++++++ src/benchmark/frameworks/tvm/tvm_test.py | 29 ++++++ src/benchmark/frameworks/tvm/tvm_wrapper.py | 15 ++++ src/benchmark/tests/test_processes.py | 2 + test/smoke_test/smoke_config.xml | 30 +++++++ 8 files changed, 180 insertions(+), 1 deletion(-) create mode 100644 src/benchmark/frameworks/tvm/tvm_process.py create mode 100644 src/benchmark/frameworks/tvm/tvm_test.py diff --git a/requirements_frameworks.txt b/requirements_frameworks.txt index d0949dbd0..f69e87abe 100644 --- a/requirements_frameworks.txt +++ b/requirements_frameworks.txt @@ -1,4 +1,5 @@ openvino-dev[caffe,mxnet,tensorflow2,pytorch,onnx]==2022.3.0 gluoncv torchvision -onnxruntime \ No newline at end of file +onnxruntime +apache-tvm==0.14.dev170 \ No newline at end of file diff --git a/src/benchmark/frameworks/framework_wrapper_registry.py b/src/benchmark/frameworks/framework_wrapper_registry.py index cbbe11103..d41b912d4 100644 --- a/src/benchmark/frameworks/framework_wrapper_registry.py +++ b/src/benchmark/frameworks/framework_wrapper_registry.py @@ -10,6 +10,7 @@ from .tensorflow_lite_cpp.tensorflow_lite_cpp_wrapper import TensorFlowLiteCppWrapper from .opencv_dnn_python.opencv_dnn_python_wrapper import OpenCVDNNPythonWrapper from .mxnet.mxnet_wrapper import MXNetWrapper +from .tvm.tvm_wrapper import TVMWrapper from .opencv_dnn_cpp.opencv_dnn_cpp_wrapper import OpenCVDNNCppWrapper from .pytorch.pytorch_wrapper import PyTorchWrapper from .pytorch_cpp.pytorch_cpp_wrapper import PyTorchCppWrapper @@ -46,3 +47,4 @@ def _get_wrappers(self): self._framework_wrappers[OpenCVDNNCppWrapper.framework_name] = OpenCVDNNCppWrapper() self._framework_wrappers[PyTorchWrapper.framework_name] = PyTorchWrapper() self._framework_wrappers[PyTorchCppWrapper.framework_name] = PyTorchCppWrapper() + self._framework_wrappers[TVMWrapper.framework_name] = TVMWrapper() diff --git a/src/benchmark/frameworks/tvm/tvm_parameters_parser.py b/src/benchmark/frameworks/tvm/tvm_parameters_parser.py index 7814abd45..cc427fbb3 100644 --- a/src/benchmark/frameworks/tvm/tvm_parameters_parser.py +++ b/src/benchmark/frameworks/tvm/tvm_parameters_parser.py @@ -28,6 +28,16 @@ def parse_parameters(self, curr_test): CONFIG_FRAMEWORK_DEPENDENT_STD_TAG)[0].firstChild _channel_swap = dep_parameters_tag.getElementsByTagName( CONFIG_FRAMEWORK_DEPENDENT_CHANNEL_SWAP_TAG)[0].firstChild + + return TVMParameters( + framework=_framework.data if _framework else None, + input_name=_input_name.data if _input_name else None, + input_shape=_input_shape.data if _input_shape else None, + normalize=_normalize.data if _normalize else None, + mean=_mean.data if _mean else None, + std=_std.data if _std else None, + channel_swap=_channel_swap.data if _channel_swap else None, + ) class TVMParameters(FrameworkParameters): diff --git a/src/benchmark/frameworks/tvm/tvm_process.py b/src/benchmark/frameworks/tvm/tvm_process.py new file mode 100644 index 000000000..9e12d49bc --- /dev/null +++ b/src/benchmark/frameworks/tvm/tvm_process.py @@ -0,0 +1,90 @@ +from pathlib import Path + +from ..processes import ProcessHandler + + +class TVMProcess(ProcessHandler): + benchmark_app_name = 'tvm_python_benchmark' + launcher_latency_units = 'seconds' + + def __init__(self, test, executor, log): + super().__init__(test, executor, log) + + + @staticmethod + def create_process(test, executor, log): + framework = test.dep_parameters.framework.lower() + if framework == 'mxnet': + return MXNet_TVMProcess(test, executor, log) + else: + raise AssertionError(f'Unknown framework {framework}') + + def get_performance_metrics(self): + return self.get_performance_metrics_from_json_report() + + def _fill_command_line(self): + name = self._test.model.name + model_json = self._test.model.model + model_params = self._test.model.weight + dataset = self._test.dataset.path + input_shape = self._test.dep_parameters.input_shape + batch_size = self._test.indep_parameters.batch_size + iteration = self._test.indep_parameters.iteration + + if ((name is not None) + and (model_json is None or model_json == '') + and (model_params is None or model_params == '')): + common_params = (f'-mn {name} -i {dataset} -is {input_shape} ' + f'-b {batch_size} -ni {iteration} --report_path {self.report_path}') + elif (model_json is not None) and (model_params is not None): + common_params = (f'-m {model_json} -w {model_params} -i {dataset} ' + f'-is {input_shape} -b {batch_size} -ni {iteration} ' + f'--report_path {self.report_path}') + else: + raise Exception('Incorrect model parameters. Set model name or file names.') + + input_name = self._test.dep_parameters.input_name + common_params = TVMProcess._add_optional_argument_to_cmd_line( + common_params, '--input_name', input_name) + + normalize = self._test.dep_parameters.normalize + if normalize == 'True': + common_params = TVMProcess._add_flag_to_cmd_line( + common_params, '--norm') + + mean = self._test.dep_parameters.mean + common_params = TVMProcess._add_optional_argument_to_cmd_line( + common_params, '--mean', mean) + + std = self._test.dep_parameters.std + common_params = TVMProcess._add_optional_argument_to_cmd_line( + common_params, '--std', std) + + channel_swap = self._test.dep_parameters.channel_swap + common_params = TVMProcess._add_optional_argument_to_cmd_line( + common_params, '--channel_swap', channel_swap) + + device = self._test.indep_parameters.device + common_params = TVMProcess._add_optional_argument_to_cmd_line( + common_params, '--device', device) + + return f'{common_params}' + + +class MXNet_TVMProcess(TVMProcess): + def __init__(self, test, executor, log): + super().__init__(test, executor, log) + + def get_performance_metrics(self): + return self.get_performance_metrics_from_json_report() + + def _fill_command_line(self): + path_to_sync_script = Path.joinpath(self.inference_script_root, + 'inference_tvm_mxnet.py') + python = ProcessHandler.get_cmd_python_version() + time_limit = self._test.indep_parameters.test_time_limit + common_params = super()._fill_command_line() + common_params += f' --time {time_limit}' + command_line = f'{python} {path_to_sync_script} {common_params}' + + return command_line \ No newline at end of file diff --git a/src/benchmark/frameworks/tvm/tvm_test.py b/src/benchmark/frameworks/tvm/tvm_test.py new file mode 100644 index 000000000..ee2bf2bec --- /dev/null +++ b/src/benchmark/frameworks/tvm/tvm_test.py @@ -0,0 +1,29 @@ +from collections import OrderedDict + +from ..config_parser.test_reporter import Test + + +class TVMTest(Test): + def __init__(self, model, dataset, indep_parameters, dep_parameters): + super().__init__(model, dataset, indep_parameters, dep_parameters) + + def get_report(self, process): + parameters = OrderedDict() + parameters.update({'Device': self.indep_parameters.device}) + parameters.update({'Iteration count': self.indep_parameters.iteration}) + parameters.update({'Framework': self.dep_parameters.framework}) + other_param = self._get_optional_parameters_string(parameters) + + report_res = { + 'task': self.model.task, + 'model': self.model.name, + 'dataset': self.dataset.name, + 'source_framework': self.model.source_framework, + 'inference_framework': self.indep_parameters.inference_framework, + 'precision': self.model.precision, + 'batch_size': self.indep_parameters.batch_size, + 'mode': 'Sync', + 'framework_params': other_param, + } + + return report_res \ No newline at end of file diff --git a/src/benchmark/frameworks/tvm/tvm_wrapper.py b/src/benchmark/frameworks/tvm/tvm_wrapper.py index e69de29bb..e0eeed4db 100644 --- a/src/benchmark/frameworks/tvm/tvm_wrapper.py +++ b/src/benchmark/frameworks/tvm/tvm_wrapper.py @@ -0,0 +1,15 @@ +from .tvm_process import TVMProcess +from .tvm_test import TVMTest +from ..framework_wrapper import FrameworkWrapper +from ..known_frameworks import KnownFrameworks + +class TVMWrapper(FrameworkWrapper): + framework_name = KnownFrameworks.tvm + + @staticmethod + def create_process(test, executor, log, **kwargs): + return TVMProcess.create_process(test, executor, log) + + @staticmethod + def create_test(model, dataset, indep_parameters, dep_parameters): + return TVMTest(model, dataset, indep_parameters, dep_parameters) \ No newline at end of file diff --git a/src/benchmark/tests/test_processes.py b/src/benchmark/tests/test_processes.py index 7babf876f..ed806e272 100644 --- a/src/benchmark/tests/test_processes.py +++ b/src/benchmark/tests/test_processes.py @@ -7,6 +7,7 @@ from src.benchmark.frameworks.intel_caffe.intel_caffe_process import IntelCaffeProcess from src.benchmark.frameworks.known_frameworks import KnownFrameworks from src.benchmark.frameworks.mxnet.mxnet_process import MXNetProcess +from src.benchmark.frameworks.tvm.tvm_process import TVMProcess from src.benchmark.frameworks.onnx_runtime.onnx_runtime_process import OnnxRuntimeProcess from src.benchmark.frameworks.onnx_runtime_python.onnx_runtime_python_process import ONNXRuntimePythonProcess from src.benchmark.frameworks.opencv_dnn_cpp.opencv_dnn_cpp_process import OpenCVDNNCppProcess @@ -75,6 +76,7 @@ def test_python_version(os, mocker): ['MXNet', MXNetProcess], ['OpenCV DNN Python', OpenCVDNNPythonProcess], ['ONNX Runtime Python', ONNXRuntimePythonProcess], + ['TVM', TVMProcess], ]) @pytest.mark.parametrize('complex_test', [['sync', 'handwritten', None, SyncOpenVINOProcess], ['async', 'handwritten', None, AsyncOpenVINOProcess], diff --git a/test/smoke_test/smoke_config.xml b/test/smoke_test/smoke_config.xml index 3c3b86279..f15724baf 100644 --- a/test/smoke_test/smoke_config.xml +++ b/test/smoke_test/smoke_config.xml @@ -341,6 +341,36 @@ + + + classification + AlexNet + FP32 + TVM + + + + + Data + ./black_square.jpg + + + TVM + 1 + CPU + 5 + 1 + + + data + MXNet + 1 3 224 224 + True + 0.485 0.456 0.406 + 0.229 0.224 0.225 + + + classification From 0ba6146a032e38b30d701eeddf94685c4368c906 Mon Sep 17 00:00:00 2001 From: ismukhin Date: Fri, 20 Oct 2023 15:25:05 +0300 Subject: [PATCH 06/18] examination of None type for framework --- src/benchmark/frameworks/tvm/tvm_process.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/benchmark/frameworks/tvm/tvm_process.py b/src/benchmark/frameworks/tvm/tvm_process.py index 9e12d49bc..8e2a5c42b 100644 --- a/src/benchmark/frameworks/tvm/tvm_process.py +++ b/src/benchmark/frameworks/tvm/tvm_process.py @@ -13,11 +13,14 @@ def __init__(self, test, executor, log): @staticmethod def create_process(test, executor, log): - framework = test.dep_parameters.framework.lower() - if framework == 'mxnet': - return MXNet_TVMProcess(test, executor, log) + if framework is None: + framework = 'TVM' else: - raise AssertionError(f'Unknown framework {framework}') + framework = test.dep_parameters.framework.lower() + if framework == 'mxnet': + return MXNet_TVMProcess(test, executor, log) + else: + raise AssertionError(f'Unknown framework {framework}') def get_performance_metrics(self): return self.get_performance_metrics_from_json_report() From 72938be9853ac1be8fd3085720687fda8b4c3381 Mon Sep 17 00:00:00 2001 From: ismukhin Date: Fri, 20 Oct 2023 15:28:22 +0300 Subject: [PATCH 07/18] fix --- src/benchmark/frameworks/tvm/tvm_process.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/benchmark/frameworks/tvm/tvm_process.py b/src/benchmark/frameworks/tvm/tvm_process.py index 8e2a5c42b..e08c7dc3f 100644 --- a/src/benchmark/frameworks/tvm/tvm_process.py +++ b/src/benchmark/frameworks/tvm/tvm_process.py @@ -13,6 +13,7 @@ def __init__(self, test, executor, log): @staticmethod def create_process(test, executor, log): + framework = test.dep_parameters.framework if framework is None: framework = 'TVM' else: From d5afb7c5872583a517349274e138f2ae55d11aae Mon Sep 17 00:00:00 2001 From: ismukhin Date: Fri, 20 Oct 2023 15:32:49 +0300 Subject: [PATCH 08/18] fix1 --- src/benchmark/frameworks/tvm/tvm_process.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/benchmark/frameworks/tvm/tvm_process.py b/src/benchmark/frameworks/tvm/tvm_process.py index e08c7dc3f..a66efd6d2 100644 --- a/src/benchmark/frameworks/tvm/tvm_process.py +++ b/src/benchmark/frameworks/tvm/tvm_process.py @@ -16,6 +16,7 @@ def create_process(test, executor, log): framework = test.dep_parameters.framework if framework is None: framework = 'TVM' + return MXNet_TVMProcess(test, executor, log) else: framework = test.dep_parameters.framework.lower() if framework == 'mxnet': From 9fa26d0e81eb8fddc403134172d96c67d66e9c2e Mon Sep 17 00:00:00 2001 From: ismukhin Date: Wed, 25 Oct 2023 17:34:13 +0300 Subject: [PATCH 09/18] support of onnx and pytorch(torchvision) --- src/benchmark/frameworks/tvm/tvm_process.py | 102 ++++++--- src/inference/inference_tvm_mxnet.py | 90 ++++++-- src/inference/inference_tvm_onnx.py | 207 +++++++++++++++++++ src/inference/inference_tvm_pytorch.py | 217 ++++++++++++++++++++ src/inference/tvm_auxiliary.py | 78 +++---- test/smoke_test/smoke_config.xml | 60 ++++++ 6 files changed, 660 insertions(+), 94 deletions(-) create mode 100644 src/inference/inference_tvm_onnx.py create mode 100644 src/inference/inference_tvm_pytorch.py diff --git a/src/benchmark/frameworks/tvm/tvm_process.py b/src/benchmark/frameworks/tvm/tvm_process.py index a66efd6d2..83df7e7ab 100644 --- a/src/benchmark/frameworks/tvm/tvm_process.py +++ b/src/benchmark/frameworks/tvm/tvm_process.py @@ -8,46 +8,38 @@ class TVMProcess(ProcessHandler): launcher_latency_units = 'seconds' def __init__(self, test, executor, log): - super().__init__(test, executor, log) - + super().__init__(test, executor, log) @staticmethod def create_process(test, executor, log): framework = test.dep_parameters.framework if framework is None: framework = 'TVM' - return MXNet_TVMProcess(test, executor, log) + return TVMProcessMXNetFormat(test, executor, log) else: framework = test.dep_parameters.framework.lower() if framework == 'mxnet': - return MXNet_TVMProcess(test, executor, log) + return TVMProcessMXNetFormat(test, executor, log) + elif framework == 'pytorch': + return TVMProcessPyTorchFormat(test, executor, log) + elif framework == 'onnx': + return TVMProcessONNXFormat(test, executor, log) else: raise AssertionError(f'Unknown framework {framework}') def get_performance_metrics(self): return self.get_performance_metrics_from_json_report() - + def _fill_command_line(self): - name = self._test.model.name - model_json = self._test.model.model - model_params = self._test.model.weight dataset = self._test.dataset.path input_shape = self._test.dep_parameters.input_shape batch_size = self._test.indep_parameters.batch_size iteration = self._test.indep_parameters.iteration - if ((name is not None) - and (model_json is None or model_json == '') - and (model_params is None or model_params == '')): - common_params = (f'-mn {name} -i {dataset} -is {input_shape} ' - f'-b {batch_size} -ni {iteration} --report_path {self.report_path}') - elif (model_json is not None) and (model_params is not None): - common_params = (f'-m {model_json} -w {model_params} -i {dataset} ' - f'-is {input_shape} -b {batch_size} -ni {iteration} ' - f'--report_path {self.report_path}') - else: - raise Exception('Incorrect model parameters. Set model name or file names.') - + common_params = (f'-i {dataset} ' + f'-is {input_shape} -b {batch_size} -ni {iteration} ' + f'--report_path {self.report_path}') + input_name = self._test.dep_parameters.input_name common_params = TVMProcess._add_optional_argument_to_cmd_line( common_params, '--input_name', input_name) @@ -72,11 +64,11 @@ def _fill_command_line(self): device = self._test.indep_parameters.device common_params = TVMProcess._add_optional_argument_to_cmd_line( common_params, '--device', device) - + return f'{common_params}' -class MXNet_TVMProcess(TVMProcess): +class TVMProcessMXNetFormat(TVMProcess): def __init__(self, test, executor, log): super().__init__(test, executor, log) @@ -84,12 +76,74 @@ def get_performance_metrics(self): return self.get_performance_metrics_from_json_report() def _fill_command_line(self): + name = self._test.model.name + model_json = self._test.model.model + model_params = self._test.model.weight + if ((name is not None) + and (model_json is None or model_json == '') + and (model_params is None or model_params == '')): + common_params = (f'-mn {name} ') + elif (model_json is not None) and (model_params is not None): + common_params = (f'-m {model_json} -w {model_params} ') + else: + raise Exception('Incorrect model parameters. Set model name or file names.') path_to_sync_script = Path.joinpath(self.inference_script_root, 'inference_tvm_mxnet.py') python = ProcessHandler.get_cmd_python_version() time_limit = self._test.indep_parameters.test_time_limit - common_params = super()._fill_command_line() + common_params += super()._fill_command_line() + common_params += f' --time {time_limit}' + command_line = f'{python} {path_to_sync_script} {common_params}' + + return command_line + + +class TVMProcessPyTorchFormat(TVMProcess): + def __init__(self, test, executor, log): + super().__init__(test, executor, log) + + def get_performance_metrics(self): + return self.get_performance_metrics_from_json_report() + + def _fill_command_line(self): + name = self._test.model.name + model_json = self._test.model.model + model_params = self._test.model.weight + if ((name is not None) + and (model_json is None or model_json == '') + and (model_params is None or model_params == '')): + common_params = (f'-mn {name} ') + elif (model_json is not None) and (model_params is not None): + common_params = (f'-m {model_json} -w {model_params} ') + else: + raise Exception('Incorrect model parameters. Set model name or file names.') + path_to_sync_script = Path.joinpath(self.inference_script_root, + 'inference_tvm_pytorch.py') + python = ProcessHandler.get_cmd_python_version() + time_limit = self._test.indep_parameters.test_time_limit + common_params += super()._fill_command_line() + common_params += f' --time {time_limit}' + command_line = f'{python} {path_to_sync_script} {common_params}' + + return command_line + + +class TVMProcessONNXFormat(TVMProcess): + def __init__(self, test, executor, log): + super().__init__(test, executor, log) + + def get_performance_metrics(self): + return self.get_performance_metrics_from_json_report() + + def _fill_command_line(self): + model = self._test.model.model + common_params = f'-m {model} ' + path_to_sync_script = Path.joinpath(self.inference_script_root, + 'inference_tvm_onnx.py') + python = ProcessHandler.get_cmd_python_version() + time_limit = self._test.indep_parameters.test_time_limit + common_params += super()._fill_command_line() common_params += f' --time {time_limit}' command_line = f'{python} {path_to_sync_script} {common_params}' - return command_line \ No newline at end of file + return command_line diff --git a/src/inference/inference_tvm_mxnet.py b/src/inference/inference_tvm_mxnet.py index 1aaef87e6..b7e0e2fb4 100644 --- a/src/inference/inference_tvm_mxnet.py +++ b/src/inference/inference_tvm_mxnet.py @@ -1,11 +1,13 @@ import argparse import json import logging as log -import os import sys import traceback import warnings import mxnet +import tvm +import gluoncv + from pathlib import Path from time import time @@ -126,6 +128,53 @@ def cli_argument_parser(): return args +class MXNetToTVMConverter(TVMConverter): + def __init__(self, args): + super().__init__(args) + + def _get_device_for_framework(self): + device = self.args['device'] + if device == 'CPU': + return mxnet.cpu() + elif device == 'NVIDIA_GPU': + return mxnet.gpu() + else: + log.info(f'The device {device} is not supported') + raise ValueError('The device is not supported') + + def _get_mxnet_network(self): + model_name = self.args['model_name'] + model_path = self.args['model_path'] + weights = self.args['model_params'] + context = self._get_device_for_framework() + + if (model_name is not None and + (model_path is None) and (weights is None)): + log.info(f'Loading network \"{model_name}\" from GluonCV model zoo') + net = gluoncv.model_zoo.get_model(model_name, pretrained=True, ctx=context) + return net + + elif ((model_path is not None) and (weights is not None)): + log.info(f'Deserializing network from file ({model_path}, {weights})') + with warnings.catch_warnings(): + warnings.simplefilter('ignore') + net = mxnet.gluon.nn.SymbolBlock.imports( + model_path, [self.args['input_name']], weights, ctx=context) + return net + + else: + raise ValueError('Incorrect arguments.') + + def _convert_model_from_framework(self, target, dev): + net = self._get_mxnet_network() + shape_dict = {self.args['input_name']: self.args['input_shape']} + model, params = tvm.relay.frontend.from_mxnet(net, shape_dict) + with tvm.transform.PassContext(opt_level=3): + lib = tvm.relay.build(model, target=target, params=params) + module = tvm.contrib.graph_executor.GraphModule(lib["default"](dev)) + return module + + def inference_tvm(module, num_of_iterations, input_name, get_slice, test_duration): result = None time_infer = [] @@ -164,22 +213,29 @@ def main(): ) args = cli_argument_parser() report_writer = ReportWriter() - wrapper = TVMIOModelWrapper(create_dict_for_modelwrapper(args)) - transformer = TVMTransformer(create_dict_for_transformer(args)) - io = IOAdapter.get_io_adapter(args, wrapper, transformer) - converter = TVMConverter(create_dict_for_converter_mxnet(args)) - graph_module = converter.convert_model_from_framework('mxnet') - io.prepare_input(graph_module, args.input) - result, infer_time = inference_tvm(graph_module, args.number_iter, args.input_name, io.get_slice_input, args.time) - if args.number_iter == 1: - io.process_output(prepare_output(result, args.task, args.output_names), log) - inference_result = pp.calculate_performance_metrics_sync_mode(args.batch_size, infer_time) - report_writer.update_execution_results(**inference_result) - report_writer.write_report(args.report_path) - log.info('Performance results') - log.info(f'Performance results:\n{json.dumps(inference_result, indent=4)}') - + report_writer.update_framework_info(name='TVM', version=tvm.__version__) + report_writer.update_configuration_setup(batch_size=args.batch_size, + iterations_num=args.number_iter, + target_device=args.device) + try: + wrapper = TVMIOModelWrapper(create_dict_for_modelwrapper(args)) + transformer = TVMTransformer(create_dict_for_transformer(args)) + io = IOAdapter.get_io_adapter(args, wrapper, transformer) + converter = MXNetToTVMConverter(create_dict_for_converter_mxnet(args)) + graph_module = converter.get_graph_module() + io.prepare_input(graph_module, args.input) + result, infer_time = inference_tvm(graph_module, args.number_iter, args.input_name, io.get_slice_input, args.time) + if args.number_iter == 1: + io.process_output(prepare_output(result, args.task, args.output_names), log) + inference_result = pp.calculate_performance_metrics_sync_mode(args.batch_size, infer_time) + report_writer.update_execution_results(**inference_result) + report_writer.write_report(args.report_path) + log.info('Performance results') + log.info(f'Performance results:\n{json.dumps(inference_result, indent=4)}') + except Exception: + log.error(traceback.format_exc()) + sys.exit(1) if __name__ == '__main__': - sys.exit(main() or 0) \ No newline at end of file + sys.exit(main() or 0) diff --git a/src/inference/inference_tvm_onnx.py b/src/inference/inference_tvm_onnx.py new file mode 100644 index 000000000..738a02158 --- /dev/null +++ b/src/inference/inference_tvm_onnx.py @@ -0,0 +1,207 @@ +import argparse +import json +import logging as log +import sys +import traceback +import onnx +import tvm + + +from pathlib import Path +from time import time + + +import postprocessing_data as pp +from inference_tools.loop_tools import loop_inference, get_exec_time +from io_adapter import IOAdapter +from io_model_wrapper import TVMIOModelWrapper +from transformer import TVMTransformer +from reporter.report_writer import ReportWriter +from tvm_auxiliary import (TVMConverter, create_dict_for_converter_onnx, + prepare_output, create_dict_for_modelwrapper, + create_dict_for_transformer) + + +def cli_argument_parser(): + parser = argparse.ArgumentParser() + parser.add_argument('-mn', '--model_name', + help='Name of model.', + type=str, + dest='model_name') + parser.add_argument('-m', '--model', + help='Path to an .onnx file with a trained model.', + type=str, + dest='model_path') + parser.add_argument('-d', '--device', + help='Specify the target device to infer on CPU or ' + 'NVIDIA_GPU (CPU by default)', + default='CPU', + type=str, + dest='device') + parser.add_argument('-ni', '--number_iter', + help='Number of inference iterations.', + default=1, + type=int, + dest='number_iter') + parser.add_argument('--output_names', + help='Name of the output tensors.', + default='output0', + type=str, + nargs='+', + dest='output_names') + parser.add_argument('-t', '--task', + help='Task type determines the type of output processing ' + 'method. Available values: feedforward - without' + 'postprocessing (by default), classification - output' + 'is a vector of probabilities.', + choices=['feedforward', 'classification'], + default='feedforward', + type=str, + dest='task') + parser.add_argument('-i', '--input', + help='Path to data.', + required=True, + type=str, + nargs='+', + dest='input') + parser.add_argument('-in', '--input_name', + help='Input name.', + default='data', + type=str, + dest='input_name') + parser.add_argument('--time', required=False, default=0, type=int, + dest='time', + help='Optional. Time in seconds to execute topology.') + parser.add_argument('-is', '--input_shape', + help='Input shape BxWxHxC, B is a batch size,' + 'W is an input tensor width,' + 'H is an input tensor height,' + 'C is an input tensor number of channels.', + required=True, + type=int, + nargs=4, + dest='input_shape') + parser.add_argument('--norm', + help='Flag to normalize input images' + '(use --mean and --std arguments to set' + 'required normalization parameters).', + action='store_true', + dest='norm') + parser.add_argument('--mean', + help='Mean values.', + default=[0, 0, 0], + type=float, + nargs=3, + dest='mean') + parser.add_argument('--std', + help='Standard deviation values.', + default=[1., 1., 1.], + type=float, + nargs=3, + dest='std') + parser.add_argument('-nt', '--number_top', + help='Number of top results to print', + default=5, + type=int, + dest='number_top') + parser.add_argument('-b', '--batch_size', + help='Batch size.', + default=1, + type=int, + dest='batch_size') + parser.add_argument('--channel_swap', + help='Parameter of channel swap (WxHxC to CxWxH by default).', + default=[2, 0, 1], + type=int, + nargs=3, + dest='channel_swap') + parser.add_argument('--report_path', + type=Path, + default=Path(__file__).parent / 'tvm_inference_report.json', + dest='report_path') + args = parser.parse_args() + return args + + +class ONNXToTVMConverter(TVMConverter): + def __init__(self, args): + super().__init__(args) + + def _get_device_for_framework(self): + return super()._get_device_for_framework() + + def _convert_model_from_framework(self, target, dev): + model_path = self.args['model_path'] + model_onnx = onnx.load(model_path) + shape_dict = {self.args['input_name']: self.args['input_shape']} + model, params = tvm.relay.frontend.from_onnx(model_onnx, shape_dict) + with tvm.transform.PassContext(opt_level=3): + lib = tvm.relay.build(model, target=target, params=params) + module = tvm.contrib.graph_executor.GraphModule(lib["default"](dev)) + return module + + +def inference_tvm(module, num_of_iterations, input_name, get_slice, test_duration): + result = None + time_infer = [] + if num_of_iterations == 1: + slice_input = get_slice() + t0 = time() + module.set_input(input_name, slice_input[input_name]) + module.run() + result = module.get_output(0) + t1 = time() + time_infer.append(t1 - t0) + else: + time_infer = loop_inference(num_of_iterations, test_duration)(inference_iteration)(get_slice, input_name, module) + return result, time_infer + + +def inference_iteration(get_slice, input_name, module): + slice_input = get_slice() + _, exec_time = infer_slice(input_name, module, slice_input) + return exec_time + + +@get_exec_time() +def infer_slice(input_name, module, slice_input): + module.set_input(input_name, slice_input[input_name]) + module.run() + res = module.get_output(0) + return res + + +def main(): + log.basicConfig( + format='[ %(levelname)s ] %(message)s', + level=log.INFO, + stream=sys.stdout, + ) + args = cli_argument_parser() + report_writer = ReportWriter() + report_writer.update_framework_info(name='TVM', version=tvm.__version__) + report_writer.update_configuration_setup(batch_size=args.batch_size, + iterations_num=args.number_iter, + target_device=args.device) + try: + wrapper = TVMIOModelWrapper(create_dict_for_modelwrapper(args)) + transformer = TVMTransformer(create_dict_for_transformer(args)) + io = IOAdapter.get_io_adapter(args, wrapper, transformer) + converter = ONNXToTVMConverter(create_dict_for_converter_onnx(args)) + graph_module = converter.get_graph_module() + io.prepare_input(graph_module, args.input) + result, infer_time = inference_tvm(graph_module, args.number_iter, args.input_name, io.get_slice_input, args.time) + if args.number_iter == 1: + io.process_output(prepare_output(result, args.task, args.output_names), log) + inference_result = pp.calculate_performance_metrics_sync_mode(args.batch_size, infer_time) + report_writer.update_execution_results(**inference_result) + report_writer.write_report(args.report_path) + log.info('Performance results') + log.info(f'Performance results:\n{json.dumps(inference_result, indent=4)}') + except Exception: + log.error(traceback.format_exc()) + sys.exit(1) + + +if __name__ == '__main__': + sys.exit(main() or 0) diff --git a/src/inference/inference_tvm_pytorch.py b/src/inference/inference_tvm_pytorch.py new file mode 100644 index 000000000..243c2e45c --- /dev/null +++ b/src/inference/inference_tvm_pytorch.py @@ -0,0 +1,217 @@ +import argparse +import json +import logging as log +import sys +import traceback +import torch +import torchvision +import tvm + + +from pathlib import Path +from time import time + + +import postprocessing_data as pp +from inference_tools.loop_tools import loop_inference, get_exec_time +from io_adapter import IOAdapter +from io_model_wrapper import TVMIOModelWrapper +from transformer import TVMTransformer +from reporter.report_writer import ReportWriter +from tvm_auxiliary import (TVMConverter, create_dict_for_converter_mxnet, + prepare_output, create_dict_for_modelwrapper, + create_dict_for_transformer) + + +def cli_argument_parser(): + parser = argparse.ArgumentParser() + parser.add_argument('-mn', '--model_name', + help='Model name to download using packages from TorchVision.', + type=str, + dest='model_name') + parser.add_argument('-m', '--model', + help='Path to an .pt file with a trained model.', + type=str, + dest='model_path') + parser.add_argument('-w', '--weights', + help='Path to an .pth file with a trained weights.', + type=str, + dest='model_params') + parser.add_argument('-d', '--device', + help='Specify the target device to infer on CPU or ' + 'NVIDIA_GPU (CPU by default)', + default='CPU', + type=str, + dest='device') + parser.add_argument('-ni', '--number_iter', + help='Number of inference iterations.', + default=1, + type=int, + dest='number_iter') + parser.add_argument('--output_names', + help='Name of the output tensors.', + default='output0', + type=str, + nargs='+', + dest='output_names') + parser.add_argument('-t', '--task', + help='Task type determines the type of output processing ' + 'method. Available values: feedforward - without' + 'postprocessing (by default), classification - output' + 'is a vector of probabilities.', + choices=['feedforward', 'classification'], + default='feedforward', + type=str, + dest='task') + parser.add_argument('-i', '--input', + help='Path to data.', + required=True, + type=str, + nargs='+', + dest='input') + parser.add_argument('-in', '--input_name', + help='Input name.', + default='data', + type=str, + dest='input_name') + parser.add_argument('--time', required=False, default=0, type=int, + dest='time', + help='Optional. Time in seconds to execute topology.') + parser.add_argument('-is', '--input_shape', + help='Input shape BxWxHxC, B is a batch size,' + 'W is an input tensor width,' + 'H is an input tensor height,' + 'C is an input tensor number of channels.', + required=True, + type=int, + nargs=4, + dest='input_shape') + parser.add_argument('--norm', + help='Flag to normalize input images' + '(use --mean and --std arguments to set' + 'required normalization parameters).', + action='store_true', + dest='norm') + parser.add_argument('--mean', + help='Mean values.', + default=[0, 0, 0], + type=float, + nargs=3, + dest='mean') + parser.add_argument('--std', + help='Standard deviation values.', + default=[1., 1., 1.], + type=float, + nargs=3, + dest='std') + parser.add_argument('-nt', '--number_top', + help='Number of top results to print', + default=5, + type=int, + dest='number_top') + parser.add_argument('-b', '--batch_size', + help='Batch size.', + default=1, + type=int, + dest='batch_size') + parser.add_argument('--channel_swap', + help='Parameter of channel swap (WxHxC to CxWxH by default).', + default=[2, 0, 1], + type=int, + nargs=3, + dest='channel_swap') + parser.add_argument('--report_path', + type=Path, + default=Path(__file__).parent / 'tvm_inference_report.json', + dest='report_path') + args = parser.parse_args() + return args + + +class PyTorchToTVMConverter(TVMConverter): + def __init__(self, args): + super().__init__(args) + + def _get_device_for_framework(self): + return super()._get_device_for_framework() + + def _convert_model_from_framework(self, target, dev): + model_name = self.args['model_name'] + pt_model = getattr(torchvision.models, model_name)(pretrained=True) + pt_model = pt_model.eval() + input_shape = self.args['input_shape'] + input_data = torch.randn(input_shape) + scripted_model = torch.jit.trace(pt_model, input_data).eval() + input_name = self.args['input_name'] + shape_list = [(input_name, input_shape)] + model, params = tvm.relay.frontend.from_pytorch(scripted_model, shape_list) + with tvm.transform.PassContext(opt_level=3): + lib = tvm.relay.build(model, target=target, params=params) + module = tvm.contrib.graph_executor.GraphModule(lib["default"](dev)) + return module + + +def inference_tvm(module, num_of_iterations, input_name, get_slice, test_duration): + result = None + time_infer = [] + if num_of_iterations == 1: + slice_input = get_slice() + t0 = time() + module.set_input(input_name, slice_input[input_name]) + module.run() + result = module.get_output(0) + t1 = time() + time_infer.append(t1 - t0) + else: + time_infer = loop_inference(num_of_iterations, test_duration)(inference_iteration)(get_slice, input_name, module) + return result, time_infer + + +def inference_iteration(get_slice, input_name, module): + slice_input = get_slice() + _, exec_time = infer_slice(input_name, module, slice_input) + return exec_time + + +@get_exec_time() +def infer_slice(input_name, module, slice_input): + module.set_input(input_name, slice_input[input_name]) + module.run() + res = module.get_output(0) + return res + + +def main(): + log.basicConfig( + format='[ %(levelname)s ] %(message)s', + level=log.INFO, + stream=sys.stdout, + ) + args = cli_argument_parser() + report_writer = ReportWriter() + report_writer.update_framework_info(name='TVM', version=tvm.__version__) + report_writer.update_configuration_setup(batch_size=args.batch_size, + iterations_num=args.number_iter, + target_device=args.device) + try: + wrapper = TVMIOModelWrapper(create_dict_for_modelwrapper(args)) + transformer = TVMTransformer(create_dict_for_transformer(args)) + io = IOAdapter.get_io_adapter(args, wrapper, transformer) + converter = PyTorchToTVMConverter(create_dict_for_converter_mxnet(args)) + graph_module = converter.get_graph_module() + io.prepare_input(graph_module, args.input) + result, infer_time = inference_tvm(graph_module, args.number_iter, args.input_name, io.get_slice_input, args.time) + if args.number_iter == 1: + io.process_output(prepare_output(result, args.task, args.output_names), log) + inference_result = pp.calculate_performance_metrics_sync_mode(args.batch_size, infer_time) + report_writer.update_execution_results(**inference_result) + report_writer.write_report(args.report_path) + log.info('Performance results') + log.info(f'Performance results:\n{json.dumps(inference_result, indent=4)}') + except Exception: + log.error(traceback.format_exc()) + sys.exit(1) + + +if __name__ == '__main__': + sys.exit(main() or 0) diff --git a/src/inference/tvm_auxiliary.py b/src/inference/tvm_auxiliary.py index 0522e6b3d..2a0a8390c 100644 --- a/src/inference/tvm_auxiliary.py +++ b/src/inference/tvm_auxiliary.py @@ -1,67 +1,32 @@ import tvm import logging as log from scipy.special import softmax +import abc -class TVMConverter: +class TVMConverter(metaclass=abc.ABCMeta): def __init__(self, args): self.args = args self.net = None - + + @abc.abstractmethod + def _get_device_for_framework(self): + pass + + @abc.abstractmethod + def _convert_model_from_framework(self, target, dev): + pass + def _get_target_device(self): if self.args['device'] == 'CPU': target = tvm.target.Target('llvm') dev = tvm.cpu(0) return target, dev - def convert_model_from_framework(self, framework): - + def get_graph_module(self): target, dev = self._get_target_device() - - if framework == 'mxnet': - import mxnet, gluoncv - if self.args['device'] == 'CPU': - context = mxnet.cpu() - model_name = self.args['model_name'] - log.info(f'Loading network \"{model_name}\" from GluonCV model zoo') - net = gluoncv.model_zoo.get_model(model_name, pretrained=True, ctx=context) - shape_dict = {self.args['input_name']: self.args['input_shape']} - model, params = tvm.relay.frontend.from_mxnet(net, shape_dict) - with tvm.transform.PassContext(opt_level=3): - lib = tvm.relay.build(model, target=target, params=params) - module = tvm.contrib.graph_executor.GraphModule(lib["default"](dev)) - return module - - elif framework == 'pytorch': - import torch, torchvision - model_name = self.args['model_name'] - pt_model = getattr(torchvision.models, model_name)(pretrained=True) - pt_model = pt_model.eval() - input_shape = self.args['input_shape'] - input_data = torch.randn(input_shape) - scripted_model = torch.jit.trace(pt_model, input_data).eval() - input_name = self.args['input_name'] - shape_list = [(input_name, input_shape)] - model, params = tvm.relay.frontend.from_pytorch(scripted_model, shape_list) - with tvm.transform.PassContext(opt_level=3): - lib = tvm.relay.build(model, target=target, params=params) - module = tvm.contrib.graph_executor.GraphModule(lib["default"](dev)) - return module - - elif framework == 'onnx': - import onnx - model_path = self.args['model_name'] - model_onnx = onnx.load(model_path) - shape_dict = {self.args['input_name']: self.args['input_shape']} - model, params = tvm.relay.frontend.from_onnx(model_onnx, shape_dict) - with tvm.transform.PassContext(opt_level=3): - lib = tvm.relay.build(model, target=target, params=params) - module = tvm.contrib.graph_executor.GraphModule(lib["default"](dev)) - return module - - - - + module = self._convert_model_from_framework(target, dev) + return module def create_dict_for_converter_mxnet(args): @@ -76,6 +41,17 @@ def create_dict_for_converter_mxnet(args): return dictionary +def create_dict_for_converter_onnx(args): + dictionary = { + 'input_name': args.input_name, + 'input_shape': [args.batch_size] + args.input_shape[1:4], + 'model_name': args.model_name, + 'model_path': args.model_path, + 'device': args.device, + } + return dictionary + + def create_dict_for_transformer(args): dictionary = { 'channel_swap': args.channel_swap, @@ -102,7 +78,3 @@ def prepare_output(result, task, output_names): return {} if task == 'classification': return {output_names[0]: softmax(result.asnumpy())} - - - - \ No newline at end of file diff --git a/test/smoke_test/smoke_config.xml b/test/smoke_test/smoke_config.xml index abfbbf1d1..79cf2ccd9 100644 --- a/test/smoke_test/smoke_config.xml +++ b/test/smoke_test/smoke_config.xml @@ -437,6 +437,66 @@ + + + classification + AlexNet + FP32 + TVM + + + + + Data + ./black_square.jpg + + + TVM + 1 + CPU + 5 + 1 + + + data + PyTorch + 1 3 224 224 + True + 0.485 0.456 0.406 + 0.229 0.224 0.225 + + + + + + classification + efficientnet-b0 + FP32 + TVM + ./working_dir_smoke/public/efficientnet-b0-pytorch/efficientnet-b0.onnx + + + + Data + ./black_square.jpg + + + TVM + 1 + CPU + 5 + 1 + + + data + ONNX + 1 3 224 224 + True + 0.485 0.456 0.406 + 0.229 0.224 0.225 + + + classification From 92e97b1a23cd4b3aa268f7224fa13ad3717b4a03 Mon Sep 17 00:00:00 2001 From: ismukhin Date: Wed, 25 Oct 2023 18:00:09 +0300 Subject: [PATCH 10/18] pytorch and codestyle fixes --- .../frameworks/tvm/tvm_parameters_parser.py | 3 ++- src/benchmark/frameworks/tvm/tvm_process.py | 1 + src/benchmark/frameworks/tvm/tvm_test.py | 2 +- src/benchmark/frameworks/tvm/tvm_wrapper.py | 3 ++- src/inference/inference_tvm_mxnet.py | 12 ++++++++++-- src/inference/inference_tvm_onnx.py | 12 ++++++++++-- src/inference/inference_tvm_pytorch.py | 16 +++++++++++++--- 7 files changed, 39 insertions(+), 10 deletions(-) diff --git a/src/benchmark/frameworks/tvm/tvm_parameters_parser.py b/src/benchmark/frameworks/tvm/tvm_parameters_parser.py index cc427fbb3..28156e2df 100644 --- a/src/benchmark/frameworks/tvm/tvm_parameters_parser.py +++ b/src/benchmark/frameworks/tvm/tvm_parameters_parser.py @@ -1,6 +1,7 @@ from ..config_parser.dependent_parameters_parser import DependentParametersParser from ..config_parser.framework_parameters_parser import FrameworkParameters + class TVMParametersParser(DependentParametersParser): def parse_parameters(self, curr_test): CONFIG_FRAMEWORK_DEPENDENT_TAG = 'FrameworkDependent' @@ -73,4 +74,4 @@ def _framework_is_correct(framework): if framework.lower() in correct_frameworks: return True else: - raise ValueError(f'Framework is required parameter. TVM support: {", ".join(correct_frameworks)}') \ No newline at end of file + raise ValueError(f'Framework is required parameter. TVM support: {", ".join(correct_frameworks)}') diff --git a/src/benchmark/frameworks/tvm/tvm_process.py b/src/benchmark/frameworks/tvm/tvm_process.py index 83df7e7ab..331179c7f 100644 --- a/src/benchmark/frameworks/tvm/tvm_process.py +++ b/src/benchmark/frameworks/tvm/tvm_process.py @@ -147,3 +147,4 @@ def _fill_command_line(self): command_line = f'{python} {path_to_sync_script} {common_params}' return command_line + diff --git a/src/benchmark/frameworks/tvm/tvm_test.py b/src/benchmark/frameworks/tvm/tvm_test.py index ee2bf2bec..95818cbae 100644 --- a/src/benchmark/frameworks/tvm/tvm_test.py +++ b/src/benchmark/frameworks/tvm/tvm_test.py @@ -26,4 +26,4 @@ def get_report(self, process): 'framework_params': other_param, } - return report_res \ No newline at end of file + return report_res diff --git a/src/benchmark/frameworks/tvm/tvm_wrapper.py b/src/benchmark/frameworks/tvm/tvm_wrapper.py index e0eeed4db..34dc4cd72 100644 --- a/src/benchmark/frameworks/tvm/tvm_wrapper.py +++ b/src/benchmark/frameworks/tvm/tvm_wrapper.py @@ -3,6 +3,7 @@ from ..framework_wrapper import FrameworkWrapper from ..known_frameworks import KnownFrameworks + class TVMWrapper(FrameworkWrapper): framework_name = KnownFrameworks.tvm @@ -12,4 +13,4 @@ def create_process(test, executor, log, **kwargs): @staticmethod def create_test(model, dataset, indep_parameters, dep_parameters): - return TVMTest(model, dataset, indep_parameters, dep_parameters) \ No newline at end of file + return TVMTest(model, dataset, indep_parameters, dep_parameters) diff --git a/src/inference/inference_tvm_mxnet.py b/src/inference/inference_tvm_mxnet.py index b7e0e2fb4..68b38bed8 100644 --- a/src/inference/inference_tvm_mxnet.py +++ b/src/inference/inference_tvm_mxnet.py @@ -168,10 +168,11 @@ def _get_mxnet_network(self): def _convert_model_from_framework(self, target, dev): net = self._get_mxnet_network() shape_dict = {self.args['input_name']: self.args['input_shape']} + log.info('Creating graph module from MXNet model') model, params = tvm.relay.frontend.from_mxnet(net, shape_dict) with tvm.transform.PassContext(opt_level=3): lib = tvm.relay.build(model, target=target, params=params) - module = tvm.contrib.graph_executor.GraphModule(lib["default"](dev)) + module = tvm.contrib.graph_executor.GraphModule(lib['default'](dev)) return module @@ -221,12 +222,19 @@ def main(): wrapper = TVMIOModelWrapper(create_dict_for_modelwrapper(args)) transformer = TVMTransformer(create_dict_for_transformer(args)) io = IOAdapter.get_io_adapter(args, wrapper, transformer) + log.info(f'Shape for input layer {args.input_name}: {args.input_shape}') converter = MXNetToTVMConverter(create_dict_for_converter_mxnet(args)) graph_module = converter.get_graph_module() + log.info(f'Preparing input data: {args.input}') io.prepare_input(graph_module, args.input) + log.info(f'Starting inference ({args.number_iter} iterations) on {args.device}') result, infer_time = inference_tvm(graph_module, args.number_iter, args.input_name, io.get_slice_input, args.time) if args.number_iter == 1: - io.process_output(prepare_output(result, args.task, args.output_names), log) + log.info('Converting output tensor to print results') + res = prepare_output(result, args.task, args.output_names) + log.info('Inference results') + io.process_output(res, log) + log.info('Computing performance metrics') inference_result = pp.calculate_performance_metrics_sync_mode(args.batch_size, infer_time) report_writer.update_execution_results(**inference_result) report_writer.write_report(args.report_path) diff --git a/src/inference/inference_tvm_onnx.py b/src/inference/inference_tvm_onnx.py index 738a02158..bbeb56983 100644 --- a/src/inference/inference_tvm_onnx.py +++ b/src/inference/inference_tvm_onnx.py @@ -134,10 +134,11 @@ def _convert_model_from_framework(self, target, dev): model_path = self.args['model_path'] model_onnx = onnx.load(model_path) shape_dict = {self.args['input_name']: self.args['input_shape']} + log.info('Creating graph module from ONNX model') model, params = tvm.relay.frontend.from_onnx(model_onnx, shape_dict) with tvm.transform.PassContext(opt_level=3): lib = tvm.relay.build(model, target=target, params=params) - module = tvm.contrib.graph_executor.GraphModule(lib["default"](dev)) + module = tvm.contrib.graph_executor.GraphModule(lib['default'](dev)) return module @@ -187,12 +188,19 @@ def main(): wrapper = TVMIOModelWrapper(create_dict_for_modelwrapper(args)) transformer = TVMTransformer(create_dict_for_transformer(args)) io = IOAdapter.get_io_adapter(args, wrapper, transformer) + log.info(f'Shape for input layer {args.input_name}: {args.input_shape}') converter = ONNXToTVMConverter(create_dict_for_converter_onnx(args)) graph_module = converter.get_graph_module() + log.info(f'Preparing input data: {args.input}') io.prepare_input(graph_module, args.input) + log.info(f'Starting inference ({args.number_iter} iterations) on {args.device}') result, infer_time = inference_tvm(graph_module, args.number_iter, args.input_name, io.get_slice_input, args.time) if args.number_iter == 1: - io.process_output(prepare_output(result, args.task, args.output_names), log) + log.info('Converting output tensor to print results') + res = prepare_output(result, args.task, args.output_names) + log.info('Inference results') + io.process_output(res, log) + log.info('Computing performance metrics') inference_result = pp.calculate_performance_metrics_sync_mode(args.batch_size, infer_time) report_writer.update_execution_results(**inference_result) report_writer.write_report(args.report_path) diff --git a/src/inference/inference_tvm_pytorch.py b/src/inference/inference_tvm_pytorch.py index 243c2e45c..e0c24e6b9 100644 --- a/src/inference/inference_tvm_pytorch.py +++ b/src/inference/inference_tvm_pytorch.py @@ -137,17 +137,20 @@ def _get_device_for_framework(self): def _convert_model_from_framework(self, target, dev): model_name = self.args['model_name'] - pt_model = getattr(torchvision.models, model_name)(pretrained=True) + log.info('Get model from TorchVision') + pt_model = torchvision.models.__getattribute__(model_name) + pt_model = pt_model(weights=True) pt_model = pt_model.eval() input_shape = self.args['input_shape'] input_data = torch.randn(input_shape) scripted_model = torch.jit.trace(pt_model, input_data).eval() input_name = self.args['input_name'] shape_list = [(input_name, input_shape)] + log.info('Creating graph module from PyTorch model') model, params = tvm.relay.frontend.from_pytorch(scripted_model, shape_list) with tvm.transform.PassContext(opt_level=3): lib = tvm.relay.build(model, target=target, params=params) - module = tvm.contrib.graph_executor.GraphModule(lib["default"](dev)) + module = tvm.contrib.graph_executor.GraphModule(lib['default'](dev)) return module @@ -197,12 +200,19 @@ def main(): wrapper = TVMIOModelWrapper(create_dict_for_modelwrapper(args)) transformer = TVMTransformer(create_dict_for_transformer(args)) io = IOAdapter.get_io_adapter(args, wrapper, transformer) + log.info(f'Shape for input layer {args.input_name}: {args.input_shape}') converter = PyTorchToTVMConverter(create_dict_for_converter_mxnet(args)) graph_module = converter.get_graph_module() + log.info(f'Preparing input data: {args.input}') io.prepare_input(graph_module, args.input) + log.info(f'Starting inference ({args.number_iter} iterations) on {args.device}') result, infer_time = inference_tvm(graph_module, args.number_iter, args.input_name, io.get_slice_input, args.time) if args.number_iter == 1: - io.process_output(prepare_output(result, args.task, args.output_names), log) + log.info('Converting output tensor to print results') + res = prepare_output(result, args.task, args.output_names) + log.info('Inference results') + io.process_output(res, log) + log.info('Computing performance metrics') inference_result = pp.calculate_performance_metrics_sync_mode(args.batch_size, infer_time) report_writer.update_execution_results(**inference_result) report_writer.write_report(args.report_path) From 56894133fa15f8ab5b393c8fcbe674713e77da4c Mon Sep 17 00:00:00 2001 From: ismukhin Date: Wed, 25 Oct 2023 18:20:56 +0300 Subject: [PATCH 11/18] fixes --- .../frameworks/tvm/tvm_parameters_parser.py | 2 +- src/benchmark/frameworks/tvm/tvm_process.py | 3 +-- src/benchmark/frameworks/tvm/tvm_test.py | 2 +- src/inference/inference_tvm_mxnet.py | 16 +++++++++++----- src/inference/inference_tvm_onnx.py | 12 +++++++++--- src/inference/inference_tvm_pytorch.py | 14 ++++++++++---- src/inference/transformer.py | 8 ++++---- src/inference/tvm_auxiliary.py | 4 +++- 8 files changed, 40 insertions(+), 21 deletions(-) diff --git a/src/benchmark/frameworks/tvm/tvm_parameters_parser.py b/src/benchmark/frameworks/tvm/tvm_parameters_parser.py index 28156e2df..0a42821f7 100644 --- a/src/benchmark/frameworks/tvm/tvm_parameters_parser.py +++ b/src/benchmark/frameworks/tvm/tvm_parameters_parser.py @@ -29,7 +29,7 @@ def parse_parameters(self, curr_test): CONFIG_FRAMEWORK_DEPENDENT_STD_TAG)[0].firstChild _channel_swap = dep_parameters_tag.getElementsByTagName( CONFIG_FRAMEWORK_DEPENDENT_CHANNEL_SWAP_TAG)[0].firstChild - + return TVMParameters( framework=_framework.data if _framework else None, input_name=_input_name.data if _input_name else None, diff --git a/src/benchmark/frameworks/tvm/tvm_process.py b/src/benchmark/frameworks/tvm/tvm_process.py index 331179c7f..f3dadcafb 100644 --- a/src/benchmark/frameworks/tvm/tvm_process.py +++ b/src/benchmark/frameworks/tvm/tvm_process.py @@ -8,7 +8,7 @@ class TVMProcess(ProcessHandler): launcher_latency_units = 'seconds' def __init__(self, test, executor, log): - super().__init__(test, executor, log) + super().__init__(test, executor, log) @staticmethod def create_process(test, executor, log): @@ -147,4 +147,3 @@ def _fill_command_line(self): command_line = f'{python} {path_to_sync_script} {common_params}' return command_line - diff --git a/src/benchmark/frameworks/tvm/tvm_test.py b/src/benchmark/frameworks/tvm/tvm_test.py index 95818cbae..5f567c09b 100644 --- a/src/benchmark/frameworks/tvm/tvm_test.py +++ b/src/benchmark/frameworks/tvm/tvm_test.py @@ -6,7 +6,7 @@ class TVMTest(Test): def __init__(self, model, dataset, indep_parameters, dep_parameters): super().__init__(model, dataset, indep_parameters, dep_parameters) - + def get_report(self, process): parameters = OrderedDict() parameters.update({'Device': self.indep_parameters.device}) diff --git a/src/inference/inference_tvm_mxnet.py b/src/inference/inference_tvm_mxnet.py index 68b38bed8..0114852f0 100644 --- a/src/inference/inference_tvm_mxnet.py +++ b/src/inference/inference_tvm_mxnet.py @@ -148,8 +148,8 @@ def _get_mxnet_network(self): weights = self.args['model_params'] context = self._get_device_for_framework() - if (model_name is not None and - (model_path is None) and (weights is None)): + if (model_name is not None + and (model_path is None) and (weights is None)): log.info(f'Loading network \"{model_name}\" from GluonCV model zoo') net = gluoncv.model_zoo.get_model(model_name, pretrained=True, ctx=context) return net @@ -181,14 +181,16 @@ def inference_tvm(module, num_of_iterations, input_name, get_slice, test_duratio time_infer = [] if num_of_iterations == 1: slice_input = get_slice() - t0 = time() + t0 = time() module.set_input(input_name, slice_input[input_name]) module.run() result = module.get_output(0) t1 = time() time_infer.append(t1 - t0) else: - time_infer = loop_inference(num_of_iterations, test_duration)(inference_iteration)(get_slice, input_name, module) + time_infer = loop_inference(num_of_iterations, test_duration)(inference_iteration)(get_slice, + input_name, + module) return result, time_infer @@ -228,7 +230,11 @@ def main(): log.info(f'Preparing input data: {args.input}') io.prepare_input(graph_module, args.input) log.info(f'Starting inference ({args.number_iter} iterations) on {args.device}') - result, infer_time = inference_tvm(graph_module, args.number_iter, args.input_name, io.get_slice_input, args.time) + result, infer_time = inference_tvm(graph_module, + args.number_iter, + args.input_name, + io.get_slice_input, + args.time) if args.number_iter == 1: log.info('Converting output tensor to print results') res = prepare_output(result, args.task, args.output_names) diff --git a/src/inference/inference_tvm_onnx.py b/src/inference/inference_tvm_onnx.py index bbeb56983..4b9472690 100644 --- a/src/inference/inference_tvm_onnx.py +++ b/src/inference/inference_tvm_onnx.py @@ -147,14 +147,16 @@ def inference_tvm(module, num_of_iterations, input_name, get_slice, test_duratio time_infer = [] if num_of_iterations == 1: slice_input = get_slice() - t0 = time() + t0 = time() module.set_input(input_name, slice_input[input_name]) module.run() result = module.get_output(0) t1 = time() time_infer.append(t1 - t0) else: - time_infer = loop_inference(num_of_iterations, test_duration)(inference_iteration)(get_slice, input_name, module) + time_infer = loop_inference(num_of_iterations, test_duration)(inference_iteration)(get_slice, + input_name, + module) return result, time_infer @@ -194,7 +196,11 @@ def main(): log.info(f'Preparing input data: {args.input}') io.prepare_input(graph_module, args.input) log.info(f'Starting inference ({args.number_iter} iterations) on {args.device}') - result, infer_time = inference_tvm(graph_module, args.number_iter, args.input_name, io.get_slice_input, args.time) + result, infer_time = inference_tvm(graph_module, + args.number_iter, + args.input_name, + io.get_slice_input, + args.time) if args.number_iter == 1: log.info('Converting output tensor to print results') res = prepare_output(result, args.task, args.output_names) diff --git a/src/inference/inference_tvm_pytorch.py b/src/inference/inference_tvm_pytorch.py index e0c24e6b9..859eaeddf 100644 --- a/src/inference/inference_tvm_pytorch.py +++ b/src/inference/inference_tvm_pytorch.py @@ -139,7 +139,7 @@ def _convert_model_from_framework(self, target, dev): model_name = self.args['model_name'] log.info('Get model from TorchVision') pt_model = torchvision.models.__getattribute__(model_name) - pt_model = pt_model(weights=True) + pt_model = pt_model(True) pt_model = pt_model.eval() input_shape = self.args['input_shape'] input_data = torch.randn(input_shape) @@ -159,14 +159,16 @@ def inference_tvm(module, num_of_iterations, input_name, get_slice, test_duratio time_infer = [] if num_of_iterations == 1: slice_input = get_slice() - t0 = time() + t0 = time() module.set_input(input_name, slice_input[input_name]) module.run() result = module.get_output(0) t1 = time() time_infer.append(t1 - t0) else: - time_infer = loop_inference(num_of_iterations, test_duration)(inference_iteration)(get_slice, input_name, module) + time_infer = loop_inference(num_of_iterations, test_duration)(inference_iteration)(get_slice, + input_name, + module) return result, time_infer @@ -206,7 +208,11 @@ def main(): log.info(f'Preparing input data: {args.input}') io.prepare_input(graph_module, args.input) log.info(f'Starting inference ({args.number_iter} iterations) on {args.device}') - result, infer_time = inference_tvm(graph_module, args.number_iter, args.input_name, io.get_slice_input, args.time) + result, infer_time = inference_tvm(graph_module, + args.number_iter, + args.input_name, + io.get_slice_input, + args.time) if args.number_iter == 1: log.info('Converting output tensor to print results') res = prepare_output(result, args.task, args.output_names) diff --git a/src/inference/transformer.py b/src/inference/transformer.py index c8601e8a9..dc022ccb3 100644 --- a/src/inference/transformer.py +++ b/src/inference/transformer.py @@ -265,7 +265,7 @@ class ONNXRuntimeTransformer(TensorFlowLiteTransformer): class TVMTransformer(Transformer): def __init__(self, converting): self._converting = converting - + def __set_norm(self, image): std = np.array([self._converting['std'][0], self._converting['std'][1], @@ -278,7 +278,7 @@ def __set_norm(self, image): image[:, :, i] -= mean[i] image[:, :, i] /= std[i] return image - + def __set_channel_swap(self, image): if self._converting['channel_swap'] is not None: transposing_form = (self._converting['channel_swap'][0], @@ -287,13 +287,13 @@ def __set_channel_swap(self, image): transposed_image = image.transpose(transposing_form) return transposed_image return image - + def _transform(self, image, element_type): transformed_image = np.copy(image).astype(element_type) normalized_image = self.__set_norm(transformed_image) transposed_image = self.__set_channel_swap(normalized_image) return transposed_image - + def transform_images(self, images, shape, element_type, input_name): dataset_size = images.shape[0] new_shape = [dataset_size] + shape[1:] diff --git a/src/inference/tvm_auxiliary.py b/src/inference/tvm_auxiliary.py index 2a0a8390c..b73eae5df 100644 --- a/src/inference/tvm_auxiliary.py +++ b/src/inference/tvm_auxiliary.py @@ -18,7 +18,9 @@ def _convert_model_from_framework(self, target, dev): pass def _get_target_device(self): - if self.args['device'] == 'CPU': + device = self.args['device'] + if device == 'CPU': + log.info(f'Inference will be executed on {device}') target = tvm.target.Target('llvm') dev = tvm.cpu(0) return target, dev From 02f6f212f24af0f3568e200423b3d6c0f4a90218 Mon Sep 17 00:00:00 2001 From: ismukhin Date: Wed, 25 Oct 2023 18:26:19 +0300 Subject: [PATCH 12/18] fixes1 --- src/benchmark/frameworks/tvm/tvm_test.py | 2 +- src/inference/inference_tvm_mxnet.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/benchmark/frameworks/tvm/tvm_test.py b/src/benchmark/frameworks/tvm/tvm_test.py index 5f567c09b..346888db5 100644 --- a/src/benchmark/frameworks/tvm/tvm_test.py +++ b/src/benchmark/frameworks/tvm/tvm_test.py @@ -6,7 +6,7 @@ class TVMTest(Test): def __init__(self, model, dataset, indep_parameters, dep_parameters): super().__init__(model, dataset, indep_parameters, dep_parameters) - + def get_report(self, process): parameters = OrderedDict() parameters.update({'Device': self.indep_parameters.device}) diff --git a/src/inference/inference_tvm_mxnet.py b/src/inference/inference_tvm_mxnet.py index 0114852f0..a17bb8ce9 100644 --- a/src/inference/inference_tvm_mxnet.py +++ b/src/inference/inference_tvm_mxnet.py @@ -148,8 +148,9 @@ def _get_mxnet_network(self): weights = self.args['model_params'] context = self._get_device_for_framework() - if (model_name is not None - and (model_path is None) and (weights is None)): + if ((model_name is not None) + and (model_path is None) + and (weights is None)): log.info(f'Loading network \"{model_name}\" from GluonCV model zoo') net = gluoncv.model_zoo.get_model(model_name, pretrained=True, ctx=context) return net From 1ade07142835671dd2a3232eb8fd1bbf3e9e42ad Mon Sep 17 00:00:00 2001 From: ismukhin Date: Wed, 25 Oct 2023 18:33:22 +0300 Subject: [PATCH 13/18] fixes2 --- src/inference/inference_tvm_pytorch.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/inference/inference_tvm_pytorch.py b/src/inference/inference_tvm_pytorch.py index 859eaeddf..4f85d737d 100644 --- a/src/inference/inference_tvm_pytorch.py +++ b/src/inference/inference_tvm_pytorch.py @@ -2,6 +2,7 @@ import json import logging as log import sys +import importlib import traceback import torch import torchvision @@ -137,9 +138,10 @@ def _get_device_for_framework(self): def _convert_model_from_framework(self, target, dev): model_name = self.args['model_name'] + module = 'torchvision.models' log.info('Get model from TorchVision') - pt_model = torchvision.models.__getattribute__(model_name) - pt_model = pt_model(True) + pt_model = importlib.import_module(module).__getattribute__(model_name) + pt_model = pt_model(weights=True) pt_model = pt_model.eval() input_shape = self.args['input_shape'] input_data = torch.randn(input_shape) From 429b730d2c0dda746bc96109c08f6f13a3743227 Mon Sep 17 00:00:00 2001 From: ismukhin Date: Wed, 25 Oct 2023 18:33:59 +0300 Subject: [PATCH 14/18] fixes3 --- src/inference/inference_tvm_pytorch.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/inference/inference_tvm_pytorch.py b/src/inference/inference_tvm_pytorch.py index 4f85d737d..536d21dc2 100644 --- a/src/inference/inference_tvm_pytorch.py +++ b/src/inference/inference_tvm_pytorch.py @@ -5,7 +5,6 @@ import importlib import traceback import torch -import torchvision import tvm From 581199ad66a4a96c6a143d608843e2e2163ad9ec Mon Sep 17 00:00:00 2001 From: ismukhin Date: Wed, 25 Oct 2023 20:33:38 +0300 Subject: [PATCH 15/18] fixes4 --- src/inference/inference_tvm_pytorch.py | 4 ++-- test/smoke_test/smoke_config.xml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/inference/inference_tvm_pytorch.py b/src/inference/inference_tvm_pytorch.py index 536d21dc2..6ec6e2a1a 100644 --- a/src/inference/inference_tvm_pytorch.py +++ b/src/inference/inference_tvm_pytorch.py @@ -139,8 +139,8 @@ def _convert_model_from_framework(self, target, dev): model_name = self.args['model_name'] module = 'torchvision.models' log.info('Get model from TorchVision') - pt_model = importlib.import_module(module).__getattribute__(model_name) - pt_model = pt_model(weights=True) + model = importlib.import_module(module).__getattribute__(model_name) + pt_model = model(weights=True) pt_model = pt_model.eval() input_shape = self.args['input_shape'] input_data = torch.randn(input_shape) diff --git a/test/smoke_test/smoke_config.xml b/test/smoke_test/smoke_config.xml index 79cf2ccd9..b848265bb 100644 --- a/test/smoke_test/smoke_config.xml +++ b/test/smoke_test/smoke_config.xml @@ -440,7 +440,7 @@ classification - AlexNet + alexnet FP32 TVM From d44ba4894c555c28ec0bae4536bd38c879f07426 Mon Sep 17 00:00:00 2001 From: ismukhin Date: Sun, 29 Oct 2023 02:19:15 +0300 Subject: [PATCH 16/18] fixes --- src/benchmark/README.md | 2 + .../frameworks/tvm/tvm_parameters_parser.py | 9 +- src/benchmark/frameworks/tvm/tvm_process.py | 21 +-- src/benchmark/frameworks/tvm/tvm_test.py | 1 + src/configs/README.md | 16 +++ .../benchmark_configuration_file_template.xml | 31 +++++ src/inference/README.md | 130 ++++++++++++++++++ src/inference/inference_tvm_mxnet.py | 66 ++++----- src/inference/inference_tvm_onnx.py | 65 ++++----- src/inference/inference_tvm_pytorch.py | 65 ++++----- src/inference/transformer.py | 25 ++-- src/inference/tvm_auxiliary.py | 44 +++++- test/smoke_test/smoke_config.xml | 3 + 13 files changed, 331 insertions(+), 147 deletions(-) diff --git a/src/benchmark/README.md b/src/benchmark/README.md index c34292575..2d6e868bd 100644 --- a/src/benchmark/README.md +++ b/src/benchmark/README.md @@ -17,6 +17,7 @@ - [OpenCV][opencv]. - [MXNet][mxnet]. - [PyTorch][pytorch]. +- [TVM][tvm]. ### Алгоритм работы скрипта @@ -264,3 +265,4 @@ pip install openvino_dev[mxnet,caffe,caffe2,onnx,pytorch,tensorflow2]== + + + + + + TVM + + + + + + + + + TVM + + + + + + + + + + + + + + + + diff --git a/src/inference/README.md b/src/inference/README.md index 9f6bd34d5..51a0cabb8 100644 --- a/src/inference/README.md +++ b/src/inference/README.md @@ -827,6 +827,136 @@ python3 inference_onnx_runtime.py \ -m /.onnx \ -i / ``` +## Вывод глубоких моделей с использованием Apache TVM + +#### Аргументы командной строки + +Название скриптов: + +```bash +inference_tvm_mxnet.py +inference_tvm_pytorch.py +inference_tvm_onnx.py +``` + +Обязательные аргументы: + +- `-i / --input` - путь до изображения или директории с изображениями + (расширения файлов `.jpg`, `.png`, `.bmp` и т.д.). +- `-is / --input_shape` - размеры входного тензора сети в формате + BxCxWxH, B - размер пачки, C - количество каналов изображений, + W - ширина изображений, H - высота изображений. + +Опциональные аргументы: + +- `-t / --task` - название задачи. Текущая реализация поддерживает + решение задачи классификации. По умолчанию принимает значение + `classification`. +- `-b / --batch_size` - количество изображений, которые будут обработаны + за один проход сети. По умолчанию равно `1`. +- `-in / --input_name` - название входа модели. По умолчанию модель + имеет один вход с названием `data`. Текущая реализация вывода + предусматривает наличие только одного входа. +- `--norm` - флаг необходимости нормировки изображений. + Среднее и среднеквадратическое отклонение, которые принимаются + на вход указываются в следующих двух аргументах. +- `--mean` - среднее значение интенсивности, которое вычитается + из изображений в процессе нормировки. По умолчанию + данный параметр принимает значение `0 0 0`. +- `--std` - среднеквадратическое отклонение интенсивности, на которое + делится значение интенсивности каждого пикселя входного изображения + в процессе нормировки. По умолчанию данный параметр принимает значение `1 1 1`. +- `--channel_swap` - порядок перестановки цветовых каналов изображения. + Загрузка изображений осуществляется в формате BGR (порядок + соответствует `(0, 1, 2)`), а большинство нейронных сетей принимают + на вход изображения в формате RGB, поэтому по умолчанию порядок + `(2, 1, 0)`. +- `-d / --device` - оборудование, на котором выполняется вывод сети. + Поддерживается вывод на CPU (значение параметра `CPU`). + По умолчанию принимает значение `CPU`. +- `-l / --labels`- путь до файла в формате JSON с перечнем меток + при решении задачи. По умолчанию принимает значение + `image_net_labels.json`, что соответствует меткам набора данных + ImageNet. +- `-ni / --number_iter` - количество прямых проходов по сети. + По умолчанию выполняется один проход по сети. +- `-ol / --opt_level` - параметр, определяющий уровень оптимизации + графа вычислений нейронной сети для ускорения инференса. По умолчанию + оптимизации не применяются. +- `--raw_output` - работа скрипта без логов. По умолчанию не установлен. + +Аргументы, необходимые для инференса моделей MXNet с использованием Apache TVM: + +- `-m / --model` - путь до описания архитектуры модели + в формате `.json`, если модель загружается из файла. + Модель должна быть обучена с использованием Gluon API + и экспортирована с помощью метода `export`. +- `-w / --weights` - путь до весов обученной модели, + которые хранятся в файле расширением `.params`, + если модель загружается из файла. Модель должна быть + обучена с использованием Gluon API и экспортирована + с помощью метода `export`. +- `-mn / --model_name` - название модели, если модель + загружается из [Gluon Model Zoo][gluon_modelzoo]. + При таком варианте запуска модель загружается из сети Интернет. + +Аргументы, необходимые для инференса моделей PyTorch с использованием Apache TVM: + +- `-m / --model` - путь до описания архитектуры модели + в формате `.pt`. +- `-w / --weights` - путь до файла с весами в формате `.pth`. +- `-mm / --module` - путь до Python модуля или относительный путь + до Python файла с архитектурой модели. По умолчанию, данный параметр + принимает значение `torchvision.models`, модуль с [моделями][torchvision_models], которые решают + задачу классификации. +- `-mn / --model_name` - имя модели. + +Аргументы, необходимые для инференса моделей ONNX Runtime с использованием Apache TVM: + +- `-m / --model` - путь до описания архитектуры модели + в формате `.onnx`. +- `-mn / --model_name` - имя модели. + +#### Примеры запуска + +##### Запуск для MXNet + +```bash +python3 inference_tvm_mxnet.py \ + -mn \ + -i / \ + -ol \ + --input_shape \ + --mean \ + --std \ + --norm +``` + +##### Запуск для PyTorch + +```bash +python3 inference_tvm_pytorch.py \ + -mn \ + -i / \ + -ol \ + --input_shape \ + --mean \ + --std \ + --norm +``` + +##### Запуск для ONNX Runtime + +```bash +python3 inference_tvm_pytorch.py \ + -m /.onnx \ + -i / \ + -ol \ + --input_shape \ + --mean \ + --std \ + --norm +``` ## Квантизация глубоких моделей с использованием MXNet diff --git a/src/inference/inference_tvm_mxnet.py b/src/inference/inference_tvm_mxnet.py index a17bb8ce9..a411349fd 100644 --- a/src/inference/inference_tvm_mxnet.py +++ b/src/inference/inference_tvm_mxnet.py @@ -10,17 +10,16 @@ from pathlib import Path -from time import time + import postprocessing_data as pp -from inference_tools.loop_tools import loop_inference, get_exec_time from io_adapter import IOAdapter from io_model_wrapper import TVMIOModelWrapper from transformer import TVMTransformer from reporter.report_writer import ReportWriter from tvm_auxiliary import (TVMConverter, create_dict_for_converter_mxnet, prepare_output, create_dict_for_modelwrapper, - create_dict_for_transformer) + create_dict_for_transformer, inference_tvm) def cli_argument_parser(): @@ -114,12 +113,22 @@ def cli_argument_parser(): default=1, type=int, dest='batch_size') + parser.add_argument('-ol', '--opt_level', + help='TVM optimization level for graph module.', + default=0, + type=int, + dest='opt_level') parser.add_argument('--channel_swap', help='Parameter of channel swap (WxHxC to CxWxH by default).', default=[2, 0, 1], type=int, nargs=3, dest='channel_swap') + parser.add_argument('--raw_output', + help='Raw output without logs.', + default=False, + type=bool, + dest='raw_output') parser.add_argument('--report_path', type=Path, default=Path(__file__).parent / 'tvm_inference_report.json', @@ -168,47 +177,16 @@ def _get_mxnet_network(self): def _convert_model_from_framework(self, target, dev): net = self._get_mxnet_network() + op_lev = self.args['opt_level'] shape_dict = {self.args['input_name']: self.args['input_shape']} log.info('Creating graph module from MXNet model') model, params = tvm.relay.frontend.from_mxnet(net, shape_dict) - with tvm.transform.PassContext(opt_level=3): + with tvm.transform.PassContext(opt_level=op_lev): lib = tvm.relay.build(model, target=target, params=params) module = tvm.contrib.graph_executor.GraphModule(lib['default'](dev)) return module -def inference_tvm(module, num_of_iterations, input_name, get_slice, test_duration): - result = None - time_infer = [] - if num_of_iterations == 1: - slice_input = get_slice() - t0 = time() - module.set_input(input_name, slice_input[input_name]) - module.run() - result = module.get_output(0) - t1 = time() - time_infer.append(t1 - t0) - else: - time_infer = loop_inference(num_of_iterations, test_duration)(inference_iteration)(get_slice, - input_name, - module) - return result, time_infer - - -def inference_iteration(get_slice, input_name, module): - slice_input = get_slice() - _, exec_time = infer_slice(input_name, module, slice_input) - return exec_time - - -@get_exec_time() -def infer_slice(input_name, module, slice_input): - module.set_input(input_name, slice_input[input_name]) - module.run() - res = module.get_output(0) - return res - - def main(): log.basicConfig( format='[ %(levelname)s ] %(message)s', @@ -236,11 +214,17 @@ def main(): args.input_name, io.get_slice_input, args.time) - if args.number_iter == 1: - log.info('Converting output tensor to print results') - res = prepare_output(result, args.task, args.output_names) - log.info('Inference results') - io.process_output(res, log) + + if not args.raw_output: + if args.number_iter == 1: + try: + log.info('Converting output tensor to print results') + res = prepare_output(result, args.task, args.output_names) + log.info('Inference results') + io.process_output(res, log) + except Exception as ex: + log.warning('Error when printing inference results. {0}'.format(str(ex))) + log.info('Computing performance metrics') inference_result = pp.calculate_performance_metrics_sync_mode(args.batch_size, infer_time) report_writer.update_execution_results(**inference_result) diff --git a/src/inference/inference_tvm_onnx.py b/src/inference/inference_tvm_onnx.py index 4b9472690..cc00eec00 100644 --- a/src/inference/inference_tvm_onnx.py +++ b/src/inference/inference_tvm_onnx.py @@ -8,18 +8,16 @@ from pathlib import Path -from time import time import postprocessing_data as pp -from inference_tools.loop_tools import loop_inference, get_exec_time from io_adapter import IOAdapter from io_model_wrapper import TVMIOModelWrapper from transformer import TVMTransformer from reporter.report_writer import ReportWriter from tvm_auxiliary import (TVMConverter, create_dict_for_converter_onnx, prepare_output, create_dict_for_modelwrapper, - create_dict_for_transformer) + create_dict_for_transformer, inference_tvm) def cli_argument_parser(): @@ -109,6 +107,16 @@ def cli_argument_parser(): default=1, type=int, dest='batch_size') + parser.add_argument('-ol', '--opt_level', + help='TVM optimization level for graph module.', + default=0, + type=int, + dest='opt_level') + parser.add_argument('--raw_output', + help='Raw output without logs.', + default=False, + type=bool, + dest='raw_output') parser.add_argument('--channel_swap', help='Parameter of channel swap (WxHxC to CxWxH by default).', default=[2, 0, 1], @@ -132,48 +140,17 @@ def _get_device_for_framework(self): def _convert_model_from_framework(self, target, dev): model_path = self.args['model_path'] + opt_lev = self.args['opt_level'] model_onnx = onnx.load(model_path) shape_dict = {self.args['input_name']: self.args['input_shape']} log.info('Creating graph module from ONNX model') model, params = tvm.relay.frontend.from_onnx(model_onnx, shape_dict) - with tvm.transform.PassContext(opt_level=3): + with tvm.transform.PassContext(opt_level=opt_lev): lib = tvm.relay.build(model, target=target, params=params) module = tvm.contrib.graph_executor.GraphModule(lib['default'](dev)) return module -def inference_tvm(module, num_of_iterations, input_name, get_slice, test_duration): - result = None - time_infer = [] - if num_of_iterations == 1: - slice_input = get_slice() - t0 = time() - module.set_input(input_name, slice_input[input_name]) - module.run() - result = module.get_output(0) - t1 = time() - time_infer.append(t1 - t0) - else: - time_infer = loop_inference(num_of_iterations, test_duration)(inference_iteration)(get_slice, - input_name, - module) - return result, time_infer - - -def inference_iteration(get_slice, input_name, module): - slice_input = get_slice() - _, exec_time = infer_slice(input_name, module, slice_input) - return exec_time - - -@get_exec_time() -def infer_slice(input_name, module, slice_input): - module.set_input(input_name, slice_input[input_name]) - module.run() - res = module.get_output(0) - return res - - def main(): log.basicConfig( format='[ %(levelname)s ] %(message)s', @@ -201,11 +178,17 @@ def main(): args.input_name, io.get_slice_input, args.time) - if args.number_iter == 1: - log.info('Converting output tensor to print results') - res = prepare_output(result, args.task, args.output_names) - log.info('Inference results') - io.process_output(res, log) + + if not args.raw_output: + if args.number_iter == 1: + try: + log.info('Converting output tensor to print results') + res = prepare_output(result, args.task, args.output_names) + log.info('Inference results') + io.process_output(res, log) + except Exception as ex: + log.warning('Error when printing inference results. {0}'.format(str(ex))) + log.info('Computing performance metrics') inference_result = pp.calculate_performance_metrics_sync_mode(args.batch_size, infer_time) report_writer.update_execution_results(**inference_result) diff --git a/src/inference/inference_tvm_pytorch.py b/src/inference/inference_tvm_pytorch.py index 6ec6e2a1a..fa8d6cd8a 100644 --- a/src/inference/inference_tvm_pytorch.py +++ b/src/inference/inference_tvm_pytorch.py @@ -9,18 +9,16 @@ from pathlib import Path -from time import time import postprocessing_data as pp -from inference_tools.loop_tools import loop_inference, get_exec_time from io_adapter import IOAdapter from io_model_wrapper import TVMIOModelWrapper from transformer import TVMTransformer from reporter.report_writer import ReportWriter from tvm_auxiliary import (TVMConverter, create_dict_for_converter_mxnet, prepare_output, create_dict_for_modelwrapper, - create_dict_for_transformer) + create_dict_for_transformer, inference_tvm) def cli_argument_parser(): @@ -114,6 +112,16 @@ def cli_argument_parser(): default=1, type=int, dest='batch_size') + parser.add_argument('-ol', '--opt_level', + help='TVM optimization level for graph module.', + default=0, + type=int, + dest='opt_level') + parser.add_argument('--raw_output', + help='Raw output without logs.', + default=False, + type=bool, + dest='raw_output') parser.add_argument('--channel_swap', help='Parameter of channel swap (WxHxC to CxWxH by default).', default=[2, 0, 1], @@ -137,6 +145,7 @@ def _get_device_for_framework(self): def _convert_model_from_framework(self, target, dev): model_name = self.args['model_name'] + opt_lev = self.args['opt_level'] module = 'torchvision.models' log.info('Get model from TorchVision') model = importlib.import_module(module).__getattribute__(model_name) @@ -149,44 +158,12 @@ def _convert_model_from_framework(self, target, dev): shape_list = [(input_name, input_shape)] log.info('Creating graph module from PyTorch model') model, params = tvm.relay.frontend.from_pytorch(scripted_model, shape_list) - with tvm.transform.PassContext(opt_level=3): + with tvm.transform.PassContext(opt_level=opt_lev): lib = tvm.relay.build(model, target=target, params=params) module = tvm.contrib.graph_executor.GraphModule(lib['default'](dev)) return module -def inference_tvm(module, num_of_iterations, input_name, get_slice, test_duration): - result = None - time_infer = [] - if num_of_iterations == 1: - slice_input = get_slice() - t0 = time() - module.set_input(input_name, slice_input[input_name]) - module.run() - result = module.get_output(0) - t1 = time() - time_infer.append(t1 - t0) - else: - time_infer = loop_inference(num_of_iterations, test_duration)(inference_iteration)(get_slice, - input_name, - module) - return result, time_infer - - -def inference_iteration(get_slice, input_name, module): - slice_input = get_slice() - _, exec_time = infer_slice(input_name, module, slice_input) - return exec_time - - -@get_exec_time() -def infer_slice(input_name, module, slice_input): - module.set_input(input_name, slice_input[input_name]) - module.run() - res = module.get_output(0) - return res - - def main(): log.basicConfig( format='[ %(levelname)s ] %(message)s', @@ -214,11 +191,17 @@ def main(): args.input_name, io.get_slice_input, args.time) - if args.number_iter == 1: - log.info('Converting output tensor to print results') - res = prepare_output(result, args.task, args.output_names) - log.info('Inference results') - io.process_output(res, log) + + if not args.raw_output: + if args.number_iter == 1: + try: + log.info('Converting output tensor to print results') + res = prepare_output(result, args.task, args.output_names) + log.info('Inference results') + io.process_output(res, log) + except Exception as ex: + log.warning('Error when printing inference results. {0}'.format(str(ex))) + log.info('Computing performance metrics') inference_result = pp.calculate_performance_metrics_sync_mode(args.batch_size, infer_time) report_writer.update_execution_results(**inference_result) diff --git a/src/inference/transformer.py b/src/inference/transformer.py index dc022ccb3..ee1e33f06 100644 --- a/src/inference/transformer.py +++ b/src/inference/transformer.py @@ -267,17 +267,20 @@ def __init__(self, converting): self._converting = converting def __set_norm(self, image): - std = np.array([self._converting['std'][0], - self._converting['std'][1], - self._converting['std'][2]]) - mean = np.array([self._converting['mean'][0], - self._converting['mean'][1], - self._converting['mean'][2]]) - for i in range(image.shape[2]): - image[:, :, i] /= 255 - image[:, :, i] -= mean[i] - image[:, :, i] /= std[i] - return image + if self._converting['norm']: + std = np.array([self._converting['std'][0], + self._converting['std'][1], + self._converting['std'][2]]) + mean = np.array([self._converting['mean'][0], + self._converting['mean'][1], + self._converting['mean'][2]]) + for i in range(image.shape[2]): + image[:, :, i] /= 255 + image[:, :, i] -= mean[i] + image[:, :, i] /= std[i] + return image + else: + return image def __set_channel_swap(self, image): if self._converting['channel_swap'] is not None: diff --git a/src/inference/tvm_auxiliary.py b/src/inference/tvm_auxiliary.py index b73eae5df..efb7045c9 100644 --- a/src/inference/tvm_auxiliary.py +++ b/src/inference/tvm_auxiliary.py @@ -2,12 +2,16 @@ import logging as log from scipy.special import softmax import abc +from time import time + +import postprocessing_data as pp +from inference_tools.loop_tools import loop_inference, get_exec_time class TVMConverter(metaclass=abc.ABCMeta): def __init__(self, args): self.args = args - self.net = None + self.graph = None @abc.abstractmethod def _get_device_for_framework(self): @@ -27,8 +31,8 @@ def _get_target_device(self): def get_graph_module(self): target, dev = self._get_target_device() - module = self._convert_model_from_framework(target, dev) - return module + self.graph = self._convert_model_from_framework(target, dev) + return self.graph def create_dict_for_converter_mxnet(args): @@ -39,6 +43,7 @@ def create_dict_for_converter_mxnet(args): 'model_path': args.model_path, 'model_params': args.model_params, 'device': args.device, + 'opt_level': args.opt_level, } return dictionary @@ -50,6 +55,7 @@ def create_dict_for_converter_onnx(args): 'model_name': args.model_name, 'model_path': args.model_path, 'device': args.device, + 'opt_level': args.opt_level, } return dictionary @@ -75,6 +81,38 @@ def create_dict_for_modelwrapper(args): return dictionary +def inference_tvm(module, num_of_iterations, input_name, get_slice, test_duration): + result = None + time_infer = [] + if num_of_iterations == 1: + slice_input = get_slice() + t0 = time() + module.set_input(input_name, slice_input[input_name]) + module.run() + result = module.get_output(0) + t1 = time() + time_infer.append(t1 - t0) + else: + time_infer = loop_inference(num_of_iterations, test_duration)(inference_iteration)(get_slice, + input_name, + module) + return result, time_infer + + +def inference_iteration(get_slice, input_name, module): + slice_input = get_slice() + _, exec_time = infer_slice(input_name, module, slice_input) + return exec_time + + +@get_exec_time() +def infer_slice(input_name, module, slice_input): + module.set_input(input_name, slice_input[input_name]) + module.run() + res = module.get_output(0) + return res + + def prepare_output(result, task, output_names): if task == 'feedforward': return {} diff --git a/test/smoke_test/smoke_config.xml b/test/smoke_test/smoke_config.xml index b848265bb..0986aa67f 100644 --- a/test/smoke_test/smoke_config.xml +++ b/test/smoke_test/smoke_config.xml @@ -435,6 +435,7 @@ 0.485 0.456 0.406 0.229 0.224 0.225 + @@ -465,6 +466,7 @@ 0.485 0.456 0.406 0.229 0.224 0.225 + 2 @@ -495,6 +497,7 @@ 0.485 0.456 0.406 0.229 0.224 0.225 + 3 From 463ec43095cc3d3e7aa294ad41f10ac2b95d9289 Mon Sep 17 00:00:00 2001 From: ismukhin Date: Sun, 29 Oct 2023 02:24:18 +0300 Subject: [PATCH 17/18] fixes1 --- src/benchmark/frameworks/tvm/tvm_process.py | 9 +++------ src/inference/tvm_auxiliary.py | 1 - 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/src/benchmark/frameworks/tvm/tvm_process.py b/src/benchmark/frameworks/tvm/tvm_process.py index 4ef96df63..dc2151488 100644 --- a/src/benchmark/frameworks/tvm/tvm_process.py +++ b/src/benchmark/frameworks/tvm/tvm_process.py @@ -90,8 +90,7 @@ def _fill_command_line(self): common_params = (f'-m {model_json} -w {model_params} ') else: raise Exception('Incorrect model parameters. Set model name or file names.') - path_to_script = Path.joinpath(self.inference_script_root, - 'inference_tvm_mxnet.py') + path_to_script = Path.joinpath(self.inference_script_root, 'inference_tvm_mxnet.py') python = ProcessHandler.get_cmd_python_version() time_limit = self._test.indep_parameters.test_time_limit common_params += super()._fill_command_line() @@ -120,8 +119,7 @@ def _fill_command_line(self): common_params = (f'-m {model_json} -w {model_params} ') else: raise Exception('Incorrect model parameters. Set model name or file names.') - path_to_script = Path.joinpath(self.inference_script_root, - 'inference_tvm_pytorch.py') + path_to_script = Path.joinpath(self.inference_script_root, 'inference_tvm_pytorch.py') python = ProcessHandler.get_cmd_python_version() time_limit = self._test.indep_parameters.test_time_limit common_params += super()._fill_command_line() @@ -141,8 +139,7 @@ def get_performance_metrics(self): def _fill_command_line(self): model = self._test.model.model common_params = f'-m {model} ' - path_to_script = Path.joinpath(self.inference_script_root, - 'inference_tvm_onnx.py') + path_to_script = Path.joinpath(self.inference_script_root, 'inference_tvm_onnx.py') python = ProcessHandler.get_cmd_python_version() time_limit = self._test.indep_parameters.test_time_limit common_params += super()._fill_command_line() diff --git a/src/inference/tvm_auxiliary.py b/src/inference/tvm_auxiliary.py index efb7045c9..18c624404 100644 --- a/src/inference/tvm_auxiliary.py +++ b/src/inference/tvm_auxiliary.py @@ -4,7 +4,6 @@ import abc from time import time -import postprocessing_data as pp from inference_tools.loop_tools import loop_inference, get_exec_time From d68d22bfbb18d5fd07157ea9e38c5635882d6475 Mon Sep 17 00:00:00 2001 From: ismukhin Date: Sun, 29 Oct 2023 15:54:27 +0300 Subject: [PATCH 18/18] readme update --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index b2320e467..f68330f0e 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,7 @@ DLI supports inference using the following frameworks: - [MXNet][mxnet] (Python Gluon API). - [OpenCV DNN][opencv-dnn] (C++ and Python APIs). - [PyTorch][pytorch] (C++ and Python APIs). +- [TVM][tvm] (Python API) More information about DLI is available on the web-site ([here][dli-ru-web-page] (in Russian) @@ -211,6 +212,7 @@ Report questions, issues and suggestions, using: [mxnet]: https://mxnet.apache.org [opencv-dnn]: https://docs.opencv.org/4.7.0/d2/d58/tutorial_table_of_content_dnn.html [pytorch]: https://pytorch.org +[tvm]: https://tvm.apache.org [benchmark-app]: https://github.com/openvinotoolkit/openvino/tree/master/samples/cpp/benchmark_app [dli-ru-web-page]: http://hpc-education.unn.ru/dli-ru [dli-web-page]: http://hpc-education.unn.ru/dli