[TVM] inference script

requirements_frameworks.txt

@@ -1,4 +1,5 @@
 openvino-dev[caffe,mxnet,tensorflow2,pytorch,onnx]==2022.3.0
 gluoncv
 torchvision
-onnxruntime
+onnxruntime
+apache-tvm==0.14.dev170

src/benchmark/config_parser_factory.py

@@ -8,6 +8,7 @@
 from frameworks.pytorch.pytorch_parameters_parser import PyTorchParametersParser
 from frameworks.onnx_runtime_python.onnx_runtime_python_parameters_parser import ONNXRuntimePythonParametersParser
 from frameworks.config_parser.dependent_parameters_parser_cpp import CppParametersParser
+from frameworks.tvm.tvm_parameters_parser import TVMParametersParser
 
 
 def get_parameters_parser(framework):
@@ -35,4 +36,6 @@ def get_parameters_parser(framework):
         return PyTorchParametersParser()
     if framework == KnownFrameworks.pytorch_cpp:
         return CppParametersParser()
+    if framework == KnownFrameworks.tvm:
+        return TVMParametersParser()
     raise NotImplementedError(f'Unknown framework {framework}')

src/benchmark/frameworks/framework_wrapper_registry.py

@@ -10,6 +10,7 @@
 from .tensorflow_lite_cpp.tensorflow_lite_cpp_wrapper import TensorFlowLiteCppWrapper
 from .opencv_dnn_python.opencv_dnn_python_wrapper import OpenCVDNNPythonWrapper
 from .mxnet.mxnet_wrapper import MXNetWrapper
+from .tvm.tvm_wrapper import TVMWrapper
 from .opencv_dnn_cpp.opencv_dnn_cpp_wrapper import OpenCVDNNCppWrapper
 from .pytorch.pytorch_wrapper import PyTorchWrapper
 from .pytorch_cpp.pytorch_cpp_wrapper import PyTorchCppWrapper
@@ -46,3 +47,4 @@ def _get_wrappers(self):
         self._framework_wrappers[OpenCVDNNCppWrapper.framework_name] = OpenCVDNNCppWrapper()
         self._framework_wrappers[PyTorchWrapper.framework_name] = PyTorchWrapper()
         self._framework_wrappers[PyTorchCppWrapper.framework_name] = PyTorchCppWrapper()
+        self._framework_wrappers[TVMWrapper.framework_name] = TVMWrapper()

src/benchmark/frameworks/known_frameworks.py

@@ -11,3 +11,4 @@ class KnownFrameworks:
     opencv_dnn_cpp = 'OpenCV DNN Cpp'
     pytorch = 'PyTorch'
     pytorch_cpp = 'PyTorch Cpp'
+    tvm = 'TVM'

src/benchmark/frameworks/tvm/tvm_parameters_parser.py

@@ -0,0 +1,77 @@
+from ..config_parser.dependent_parameters_parser import DependentParametersParser
+from ..config_parser.framework_parameters_parser import FrameworkParameters
+
+
+class TVMParametersParser(DependentParametersParser):
+    def parse_parameters(self, curr_test):
+        CONFIG_FRAMEWORK_DEPENDENT_TAG = 'FrameworkDependent'
+        CONFIG_FRAMEWORK_DEPENDENT_NAME_OF_FRAMEWORK_TAG = 'Framework'
+        CONFIG_FRAMEWORK_DEPENDENT_INPUT_NAME_TAG = 'InputName'
+        CONFIG_FRAMEWORK_DEPENDENT_INPUT_SHAPE_TAG = 'InputShape'
+        CONFIG_FRAMEWORK_DEPENDENT_NORMALIZE_TAG = 'Normalize'
+        CONFIG_FRAMEWORK_DEPENDENT_MEAN_TAG = 'Mean'
+        CONFIG_FRAMEWORK_DEPENDENT_STD_TAG = 'Std'
+        CONFIG_FRAMEWORK_DEPENDENT_CHANNEL_SWAP_TAG = 'ChannelSwap'
+
+        dep_parameters_tag = curr_test.getElementsByTagName(CONFIG_FRAMEWORK_DEPENDENT_TAG)[0]
+
+        _framework = dep_parameters_tag.getElementsByTagName(
+            CONFIG_FRAMEWORK_DEPENDENT_NAME_OF_FRAMEWORK_TAG)[0].firstChild
+        _input_name = dep_parameters_tag.getElementsByTagName(
+            CONFIG_FRAMEWORK_DEPENDENT_INPUT_NAME_TAG)[0].firstChild
+        _input_shape = dep_parameters_tag.getElementsByTagName(
+            CONFIG_FRAMEWORK_DEPENDENT_INPUT_SHAPE_TAG)[0].firstChild
+        _normalize = dep_parameters_tag.getElementsByTagName(
+            CONFIG_FRAMEWORK_DEPENDENT_NORMALIZE_TAG)[0].firstChild
+        _mean = dep_parameters_tag.getElementsByTagName(
+            CONFIG_FRAMEWORK_DEPENDENT_MEAN_TAG)[0].firstChild
+        _std = dep_parameters_tag.getElementsByTagName(
+            CONFIG_FRAMEWORK_DEPENDENT_STD_TAG)[0].firstChild
+        _channel_swap = dep_parameters_tag.getElementsByTagName(
+            CONFIG_FRAMEWORK_DEPENDENT_CHANNEL_SWAP_TAG)[0].firstChild
+
+        return TVMParameters(
+            framework=_framework.data if _framework else None,
+            input_name=_input_name.data if _input_name else None,
+            input_shape=_input_shape.data if _input_shape else None,
+            normalize=_normalize.data if _normalize else None,
+            mean=_mean.data if _mean else None,
+            std=_std.data if _std else None,
+            channel_swap=_channel_swap.data if _channel_swap else None,
+        )
+
+
+class TVMParameters(FrameworkParameters):
+    def __init__(self, framework, input_name, input_shape,
+                 normalize, mean, std, channel_swap):
+        self.framework = None
+        self.input_name = None
+        self.input_shape = None
+        self.normalize = None
+        self.mean = None
+        self.std = None
+        self.channel_swap = None
+
+        if self._framework_is_correct(framework):
+            self.framework = framework
+        if self._parameter_is_not_none(input_name):
+            self.input_name = input_name
+        if self._parameter_is_not_none(input_shape):
+            self.input_shape = input_shape
+        if self._parameter_is_not_none(normalize):
+            self.normalize = normalize
+        if self._parameter_is_not_none(mean):
+            self.mean = mean
+        if self._parameter_is_not_none(std):
+            self.std = std
+        if self._parameter_is_not_none(channel_swap):
+            self.channel_swap = channel_swap
+
+    @staticmethod
+    def _framework_is_correct(framework):
+        correct_frameworks = ['mxnet', 'onnx', 'tvm',
+                              'tf', 'tflite', 'pytorch']
+        if framework.lower() in correct_frameworks:
+            return True
+        else:
+            raise ValueError(f'Framework is required parameter. TVM support: {", ".join(correct_frameworks)}')

src/benchmark/frameworks/tvm/tvm_process.py

@@ -0,0 +1,149 @@
+from pathlib import Path
+
+from ..processes import ProcessHandler
+
+
+class TVMProcess(ProcessHandler):
+    benchmark_app_name = 'tvm_python_benchmark'
+    launcher_latency_units = 'seconds'
+
+    def __init__(self, test, executor, log):
+        super().__init__(test, executor, log)
+
+    @staticmethod
+    def create_process(test, executor, log):
+        framework = test.dep_parameters.framework
+        if framework is None:
+            framework = 'TVM'
+            return TVMProcessMXNetFormat(test, executor, log)
+        else:
+            framework = test.dep_parameters.framework.lower()
+            if framework == 'mxnet':
+                return TVMProcessMXNetFormat(test, executor, log)
+            elif framework == 'pytorch':
+                return TVMProcessPyTorchFormat(test, executor, log)
+            elif framework == 'onnx':
+                return TVMProcessONNXFormat(test, executor, log)
+            else:
+                raise AssertionError(f'Unknown framework {framework}')
+
+    def get_performance_metrics(self):
+        return self.get_performance_metrics_from_json_report()
+
+    def _fill_command_line(self):
+        dataset = self._test.dataset.path
+        input_shape = self._test.dep_parameters.input_shape
+        batch_size = self._test.indep_parameters.batch_size
+        iteration = self._test.indep_parameters.iteration
+
+        common_params = (f'-i {dataset} '
+                         f'-is {input_shape} -b {batch_size} -ni {iteration} '
+                         f'--report_path {self.report_path}')
+
+        input_name = self._test.dep_parameters.input_name
+        common_params = TVMProcess._add_optional_argument_to_cmd_line(
+            common_params, '--input_name', input_name)
+
+        normalize = self._test.dep_parameters.normalize
+        if normalize == 'True':
+            common_params = TVMProcess._add_flag_to_cmd_line(
+                common_params, '--norm')
+
+        mean = self._test.dep_parameters.mean
+        common_params = TVMProcess._add_optional_argument_to_cmd_line(
+            common_params, '--mean', mean)
+
+        std = self._test.dep_parameters.std
+        common_params = TVMProcess._add_optional_argument_to_cmd_line(
+            common_params, '--std', std)
+
+        channel_swap = self._test.dep_parameters.channel_swap
+        common_params = TVMProcess._add_optional_argument_to_cmd_line(
+            common_params, '--channel_swap', channel_swap)
+
+        device = self._test.indep_parameters.device
+        common_params = TVMProcess._add_optional_argument_to_cmd_line(
+            common_params, '--device', device)
+
+        return f'{common_params}'
+
+
+class TVMProcessMXNetFormat(TVMProcess):
+    def __init__(self, test, executor, log):
+        super().__init__(test, executor, log)
+
+    def get_performance_metrics(self):
+        return self.get_performance_metrics_from_json_report()
+
+    def _fill_command_line(self):
+        name = self._test.model.name
+        model_json = self._test.model.model
+        model_params = self._test.model.weight
+        if ((name is not None)
+                and (model_json is None or model_json == '')
+                and (model_params is None or model_params == '')):
+            common_params = (f'-mn {name} ')
+        elif (model_json is not None) and (model_params is not None):
+            common_params = (f'-m {model_json} -w {model_params} ')
+        else:
+            raise Exception('Incorrect model parameters. Set model name or file names.')
+        path_to_sync_script = Path.joinpath(self.inference_script_root,
+                                            'inference_tvm_mxnet.py')
+        python = ProcessHandler.get_cmd_python_version()
+        time_limit = self._test.indep_parameters.test_time_limit
+        common_params += super()._fill_command_line()
+        common_params += f' --time {time_limit}'
+        command_line = f'{python} {path_to_sync_script} {common_params}'
+
+        return command_line
+
+
+class TVMProcessPyTorchFormat(TVMProcess):
+    def __init__(self, test, executor, log):
+        super().__init__(test, executor, log)
+
+    def get_performance_metrics(self):
+        return self.get_performance_metrics_from_json_report()
+
+    def _fill_command_line(self):
+        name = self._test.model.name
+        model_json = self._test.model.model
+        model_params = self._test.model.weight
+        if ((name is not None)
+                and (model_json is None or model_json == '')
+                and (model_params is None or model_params == '')):
+            common_params = (f'-mn {name} ')
+        elif (model_json is not None) and (model_params is not None):
+            common_params = (f'-m {model_json} -w {model_params} ')
+        else:
+            raise Exception('Incorrect model parameters. Set model name or file names.')
+        path_to_sync_script = Path.joinpath(self.inference_script_root,
+                                            'inference_tvm_pytorch.py')
+        python = ProcessHandler.get_cmd_python_version()
+        time_limit = self._test.indep_parameters.test_time_limit
+        common_params += super()._fill_command_line()
+        common_params += f' --time {time_limit}'
+        command_line = f'{python} {path_to_sync_script} {common_params}'
+
+        return command_line
+
+
+class TVMProcessONNXFormat(TVMProcess):
+    def __init__(self, test, executor, log):
+        super().__init__(test, executor, log)
+
+    def get_performance_metrics(self):
+        return self.get_performance_metrics_from_json_report()
+
+    def _fill_command_line(self):
+        model = self._test.model.model
+        common_params = f'-m {model} '
+        path_to_sync_script = Path.joinpath(self.inference_script_root,
+                                            'inference_tvm_onnx.py')
+        python = ProcessHandler.get_cmd_python_version()
+        time_limit = self._test.indep_parameters.test_time_limit
+        common_params += super()._fill_command_line()
+        common_params += f' --time {time_limit}'
+        command_line = f'{python} {path_to_sync_script} {common_params}'
+
+        return command_line

src/benchmark/frameworks/tvm/tvm_test.py

@@ -0,0 +1,29 @@
+from collections import OrderedDict
+
+from ..config_parser.test_reporter import Test
+
+
+class TVMTest(Test):
+    def __init__(self, model, dataset, indep_parameters, dep_parameters):
+        super().__init__(model, dataset, indep_parameters, dep_parameters)
+
+    def get_report(self, process):
+        parameters = OrderedDict()
+        parameters.update({'Device': self.indep_parameters.device})
+        parameters.update({'Iteration count': self.indep_parameters.iteration})
+        parameters.update({'Framework': self.dep_parameters.framework})
+        other_param = self._get_optional_parameters_string(parameters)
+
+        report_res = {
+            'task': self.model.task,
+            'model': self.model.name,
+            'dataset': self.dataset.name,
+            'source_framework': self.model.source_framework,
+            'inference_framework': self.indep_parameters.inference_framework,
+            'precision': self.model.precision,
+            'batch_size': self.indep_parameters.batch_size,
+            'mode': 'Sync',
+            'framework_params': other_param,
+        }
+
+        return report_res

src/benchmark/frameworks/tvm/tvm_wrapper.py

@@ -0,0 +1,16 @@
+from .tvm_process import TVMProcess
+from .tvm_test import TVMTest
+from ..framework_wrapper import FrameworkWrapper
+from ..known_frameworks import KnownFrameworks
+
+
+class TVMWrapper(FrameworkWrapper):
+    framework_name = KnownFrameworks.tvm
+
+    @staticmethod
+    def create_process(test, executor, log, **kwargs):
+        return TVMProcess.create_process(test, executor, log)
+
+    @staticmethod
+    def create_test(model, dataset, indep_parameters, dep_parameters):
+        return TVMTest(model, dataset, indep_parameters, dep_parameters)

src/benchmark/tests/test_processes.py

@@ -7,6 +7,7 @@
 from src.benchmark.frameworks.intel_caffe.intel_caffe_process import IntelCaffeProcess
 from src.benchmark.frameworks.known_frameworks import KnownFrameworks
 from src.benchmark.frameworks.mxnet.mxnet_process import MXNetProcess
+from src.benchmark.frameworks.tvm.tvm_process import TVMProcess
 from src.benchmark.frameworks.onnx_runtime.onnx_runtime_process import OnnxRuntimeProcess
 from src.benchmark.frameworks.onnx_runtime_python.onnx_runtime_python_process import ONNXRuntimePythonProcess
 from src.benchmark.frameworks.opencv_dnn_cpp.opencv_dnn_cpp_process import OpenCVDNNCppProcess
@@ -75,6 +76,7 @@ def test_python_version(os, mocker):
                                                  ['MXNet', MXNetProcess],
                                                  ['OpenCV DNN Python', OpenCVDNNPythonProcess],
                                                  ['ONNX Runtime Python', ONNXRuntimePythonProcess],
+                                                 ['TVM', TVMProcess],
                                                  ])
 @pytest.mark.parametrize('complex_test', [['sync', 'handwritten', None, SyncOpenVINOProcess],
                                           ['async', 'handwritten', None, AsyncOpenVINOProcess],