Merge branch 'master' into update_table

itlab-vision · Dec 15, 2024 · ca17877 · ca17877
2 parents 372b9f3 + 651c17e
commit ca17877
Show file tree

Hide file tree

Showing 72 changed files with 1,929 additions and 188 deletions.
diff --git a/README.md b/README.md
@@ -12,7 +12,8 @@ as well as regularly publishing the obtained measurements.
 The main advantage of DLI from the existing benchmarks
 is the availability of performance results for a large number
 of deep models inferred on Intel-platforms (Intel CPUs, Intel
-Processor Graphics, Intel Movidius Neural Compute Stick).
+Processor Graphics). In addition, there are benchmarking results
+for several models inferred on RISC-V platform.
 
 DLI supports inference using the following frameworks:
 
@@ -30,6 +31,7 @@ DLI supports inference using the following frameworks:
 - [Spektral][spektral] (Python API).
 - [RKNN][rknn] (C++ API).
 - [ncnn][ncnn] (Python API).
+- [PaddlePaddle][PaddlePaddle] (Python API).
 
 More information about DLI is available on the web-site
 ([here][dli-ru-web-page] (in Russian)
@@ -44,8 +46,7 @@ This project is licensed under the terms of the [Apache 2.0 license](LICENSE).
 Please consider citing the following papers.
 
 1. Kustikova V., Vasilyev E., Khvatov A., Kumbrasiev P., Rybkin R.,
-   Kogteva N. DLI: Dee
-p Learning Inference Benchmark //
+   Kogteva N. DLI: Deep Learning Inference Benchmark //
    Communications in Computer and Information Science.
    V.1129. 2019. P. 542-553.
 
@@ -72,6 +73,10 @@ p Learning Inference Benchmark //
    and Programming. - 2024. - Vol. 25(2). - P. 127-141. -
    [https://num-meth.ru/index.php/journal/article/view/1332/1264][nummeth2023].
    (In Russian)
+1. Mukhin I., Rodimkov Y., Vasiliev E., Volokitin V., Sidorova A.,
+   Kozinov E., Meyerov I., Kustikova V. Benchmarking Deep Learning
+   Inference on RISC-V CPUs // Springer Lecture Notes in Computer
+   Science. – 2024. – Accepted.
 
 ## Repo structure
 
@@ -86,6 +91,7 @@ p Learning Inference Benchmark //
   - `Dockerfile` is the main Dockerfile.
   - `Caffe` is a directory of Dockerfiles for Intel® Optimization
     for Caffe.
+  - `DGL` is a directory of Dockerfiles for Deep Graph Library (DGL).
   - `MXNet` is a directory of Dockerfiles for MXNet.
   - `ONNXRuntime` is a directory of Dockerfiles for ONNX Runtime.
   - `OpenCV` is a directory of Dockerfiles for OpenCV.
@@ -95,6 +101,8 @@ p Learning Inference Benchmark //
   - `TVM` is a directory of Dockerfiles for Apache TVM.
   - `TensorFlow` is a directory of Dockerfiles for Intel® Optimizations
     for TensorFlow.
+  - `TensorFlowLite` is a directory of Dockerfiles for TensorFlow Lite.
+  - `PaddlePaddle` is a directory of Dockerfiles for PaddlePaddle.
 
 - `docs` directory contains auxiliary documentation. Please, find
   complete documentation at the [Wiki page][dli-wiki].
@@ -262,6 +270,7 @@ Report questions, issues and suggestions, using:
 [spektral]: https://graphneural.network
 [rknn]: https://github.com/rockchip-linux/rknn-toolkit2
 [ncnn]: https://github.com/Tencent/ncnn
+[PaddlePaddle]: https://www.paddlepaddle.org.cn/en
 [benchmark-app]: https://github.com/openvinotoolkit/openvino/tree/master/samples/cpp/benchmark_app
 [dli-ru-web-page]: http://hpc-education.unn.ru/dli-ru
 [dli-web-page]: http://hpc-education.unn.ru/dli

diff --git a/demo/OpenVINO_DLDT/README.md b/demo/OpenVINO_DLDT/README.md
diff --git a/demo/README.md b/demo/README.md
@@ -0,0 +1,57 @@
+# Demo
+
+## Description
+
+### Basic information
+
+The script allows you to run the full cycle of the DLI
+Benchmark system, demonstrating the operability of the entire
+system, from system deployment to conversion of tables with
+results. At the moment, a version for Linux systems with
+the deployment of the DLI Benchmark using Docker containers
+has been implemented.
+
+### Algorithm
+
+The application creates two directories on the local machine,
+`server` and `client`, which represent the environments
+for the FTP server and client, respectively. The output
+of the application is CSV, HTML, and XLSX tables with
+the results of the experiments measuring the performance
+and assessing the quality of the deep `SampleNet` model,
+which is a test model for the [AccuracyChecker][accuracy-checker]
+tool in [Intel® Distribution of OpenVINO™ Toolkit][openvino-toolkit].
+
+### Application results
+
+The output of the application is a set of CSV, HTML and XLSX
+tables with the results of the experiments on measuring performance
+and assessing the quality of the test model.
+
+## Usage
+
+```bash
+chmod a+x linux.sh
+./linux.sh -l <login> -p <password> -d <benchmark_datasets>
+```
+
+Command line arguments:
+
+- `-l` corresponds to the login to connect to the current machine.
+- `-p` corresponds to the password to connect to the current machine.
+- `-d` corresponds to the GitHub repository that contains
+  the benchmarking datasets. It is required that the repository
+  contains the directory `Datasets/ImageNET/`, in which at least
+  one image is stored. It is necessary for system deployment. 
+
+Example:
+
+```bash
+chmod a+x linux.sh
+./linux.sh -l admin -p admin -d "https://github.com/account/repo.git"
+```
+
+
+<!-- LINKS -->
+[openvino-toolkit]: https://software.intel.com/en-us/openvino-toolkit
+[accuracy-checker]: https://docs.openvino.ai/latest/omz_tools_accuracy_checker.html
diff --git a/demo/accuracy_checker_configs/MXNet.xml b/demo/accuracy_checker_configs/MXNet.xml
@@ -0,0 +1,17 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Tests>
+    <Test>
+        <Model>
+            <Task>classification</Task>
+            <Name>samplenet-0000</Name>
+            <Precision>FP32</Precision>
+            <SourceFramework>MXNet</SourceFramework>
+            <Directory>/media/models</Directory>
+        </Model>
+        <Parameters>
+            <InferenceFramework>MXNet</InferenceFramework>
+            <Device>CPU</Device>
+            <Config>{CONFIG_PATH}</Config>
+        </Parameters>
+    </Test>
+</Tests>
diff --git a/demo/accuracy_checker_configs/MXNet.yml b/demo/accuracy_checker_configs/MXNet.yml
@@ -0,0 +1,33 @@
+models:
+  - name: SampLeNet_example
+    launchers:
+      - framework: mxnet
+        device: CPU
+        model:   samplenet-0000.params
+        adapter: classification
+        inputs:
+        - name: 'data'
+          type: INPUT
+          shape: 3, 32, 32
+
+    datasets:
+      - name: sample_dataset
+        data_source: sample_dataset/test
+        annotation_conversion:
+          converter: cifar
+          data_batch_file: cifar-10-batches-py/test_batch
+          convert_images: True
+          converted_images_dir: sample_dataset/test
+          num_classes: 10
+
+        preprocessing:
+          - type: resize
+            size: 32
+          - type: bgr_to_rgb
+          - type: normalization
+            mean: (125.307, 122.961, 113.8575)
+            std: (51.5865, 50.847, 51.255)
+
+        metrics:
+          - type: accuracy
+            top_k: 1
diff --git a/demo/accuracy_checker_configs/ONNX_Runtime_Python.xml b/demo/accuracy_checker_configs/ONNX_Runtime_Python.xml
@@ -0,0 +1,17 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Tests>
+    <Test>
+        <Model>
+            <Task>classification</Task>
+            <Name>SampleNet</Name>
+            <Precision>FP32</Precision>
+            <SourceFramework>ONNXRuntime</SourceFramework>
+            <Directory>/media/models</Directory>
+        </Model>
+        <Parameters>
+            <InferenceFramework>ONNX Runtime Python</InferenceFramework>
+            <Device>CPU</Device>
+            <Config>{CONFIG_PATH}</Config>
+        </Parameters>
+    </Test>
+</Tests>
diff --git a/demo/accuracy_checker_configs/ONNX_Runtime_Python.yml b/demo/accuracy_checker_configs/ONNX_Runtime_Python.yml
@@ -0,0 +1,34 @@
+models:
+  - name: SampLeNet_example
+    launchers:
+      - framework: onnx_runtime
+        device: CPU
+        model:   samplenet.onnx
+        adapter: classification
+        batch: 1
+        inputs:
+          - type: INPUT
+            name: data
+            layout: NCHW
+
+    datasets:
+      - name: sample_dataset
+        data_source: sample_dataset/test
+        annotation_conversion:
+          converter: cifar
+          data_batch_file: cifar-10-batches-py/test_batch
+          convert_images: True
+          converted_images_dir: sample_dataset/test
+          num_classes: 10
+
+        preprocessing:
+          - type: resize
+            size: 32
+          - type: bgr_to_rgb
+          - type: normalization
+            mean: (125.307, 122.961, 113.8575)
+            std: (51.5865, 50.847, 51.255)
+
+        metrics:
+          - type: accuracy
+            top_k: 1
diff --git a/demo/accuracy_checker_configs/OpenCV_DNN_Python.xml b/demo/accuracy_checker_configs/OpenCV_DNN_Python.xml
@@ -0,0 +1,17 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Tests>
+    <Test>
+        <Model>
+            <Task>classification</Task>
+            <Name>SampleNet</Name>
+            <Precision>FP32</Precision>
+            <SourceFramework>Caffe</SourceFramework>
+            <Directory>/media/models</Directory>
+        </Model>
+        <Parameters>
+            <InferenceFramework>OpenCV DNN Python</InferenceFramework>
+            <Device>CPU</Device>
+            <Config>{CONFIG_PATH}</Config>
+        </Parameters>
+    </Test>
+</Tests>
diff --git a/demo/accuracy_checker_configs/OpenCV_DNN_Python.yml b/demo/accuracy_checker_configs/OpenCV_DNN_Python.yml
@@ -0,0 +1,37 @@
+models:
+  - name: SampLeNet_example
+    launchers:
+      - framework: opencv
+        device: CPU
+        tags:
+          - FP32
+        backend: OCV
+        inputs:
+          - name: 'input'
+            type: INPUT
+            shape: 3, 32, 32
+        model:   SampLeNet.prototxt
+        weights: SampLeNet.caffemodel
+        adapter: classification
+
+    datasets:
+      - name: sample_dataset
+        data_source: sample_dataset/test
+        annotation_conversion:
+          converter: cifar
+          data_batch_file: cifar-10-batches-py/test_batch
+          convert_images: True
+          converted_images_dir: sample_dataset/test
+          num_classes: 10
+
+        preprocessing:
+          - type: resize
+            size: 32
+          - type: bgr_to_rgb
+          - type: normalization
+            mean: (125.307, 122.961, 113.8575)
+            std: (51.5865, 50.847, 51.255)
+
+        metrics:
+          - type: accuracy
+            top_k: 1
diff --git a/demo/accuracy_checker_configs/OpenVINO_DLDT.xml b/demo/accuracy_checker_configs/OpenVINO_DLDT.xml
@@ -0,0 +1,17 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Tests>
+    <Test>
+        <Model>
+            <Task>classification</Task>
+            <Name>SampleNet</Name>
+            <Precision>FP32</Precision>
+            <SourceFramework>Caffe</SourceFramework>
+            <Directory>/media/models</Directory>
+        </Model>
+        <Parameters>
+            <InferenceFramework>OpenVINO DLDT</InferenceFramework>
+            <Device>CPU</Device>
+            <Config>{CONFIG_PATH}</Config>
+        </Parameters>
+    </Test>
+</Tests>