[DO NOT MERGE] [CI] OV tests in GitHub Actions #217
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
name: Tests on Windows | |
on: | |
workflow_dispatch: | |
pull_request: | |
push: | |
branches: | |
- master | |
concurrency: | |
group: ${{ github.workflow }}-${{ github.ref }} | |
cancel-in-progress: true | |
env: | |
CMAKE_BUILD_TYPE: 'Release' | |
# CCACHE_MAXSIZE: '10G' | |
CMAKE_GENERATOR: 'Ninja' | |
CMAKE_CXX_COMPILER_LAUNCHER: sccache | |
CMAKE_C_COMPILER_LAUNCHER: sccache | |
OPENVINO_REPO: "${{ github.workspace }}\\openvino" | |
OPENVINO_CONTRIB_REPO: "${{ github.workspace }}\\openvino_contrib" | |
INSTALL_DIR: "${{ github.workspace }}\\install" | |
INSTALL_TEST_DIR: "${{ github.workspace }}\\install\\tests" | |
SAMPLES_INSTALL_DIR: "${{ github.workspace }}\\install\\samples" | |
LAYER_TESTS_INSTALL_DIR: "${{ github.workspace }}\\install\\tests\\layer_tests" | |
BUILD_DIR: "${{ github.workspace }}\\build" | |
DATA_PATH: "${{ github.workspace }}\\testdata" | |
MODELS_PATH: "${{ github.workspace }}\\testdata" | |
OV_TEMP: "${{ github.workspace }}\\openvino_temp" | |
# CCACHE_DIR: /sources_cache/build_cache | |
PYTHON_STATIC_ARGS: -m "not dynamic_library and not template_plugin" | |
VCVARSPATH: "C:\\Program Files\\Microsoft Visual Studio\\2022\\Enterprise\\VC\\Auxiliary\\Build\\vcvarsall.bat" | |
jobs: | |
Build: | |
defaults: | |
run: | |
shell: pwsh | |
runs-on: windows-latest-8-cores | |
steps: | |
# TODO: do not provide 'ref' and 'repository' when scheduled on actual PRs | |
- name: Clone OpenVINO | |
uses: actions/checkout@v3 | |
with: | |
repository: 'akashchi/openvino' | |
ref: 'ci/test-gha' | |
path: 'openvino' | |
submodules: 'recursive' | |
- name: Clone OpenVINO Contrib | |
uses: actions/checkout@v3 | |
with: | |
repository: 'openvinotoolkit/openvino_contrib' | |
path: 'openvino_contrib' | |
submodules: 'recursive' | |
- name: Clone testdata for C API tests | |
uses: actions/checkout@v3 | |
with: | |
repository: 'openvinotoolkit/testdata' | |
path: 'testdata' | |
submodules: 'recursive' | |
lfs: 'true' | |
# | |
# Dependencies | |
# | |
- uses: actions/setup-python@v4 | |
with: | |
python-version: '3.11' | |
cache: 'pip' | |
cache-dependency-path: '**/requirements*.txt' | |
- name: Install python dependencies | |
run: | | |
# For Python API | |
python3 -m pip install Scons | |
python3 -m pip install -r ${{ env.OPENVINO_REPO }}/src/bindings/python/wheel/requirements-dev.txt | |
python3 -m pip install -r ${{ env.OPENVINO_REPO }}/src/bindings/python/requirements.txt | |
python3 -m pip install -r ${{ env.OPENVINO_REPO }}/src/bindings/python/requirements_test.txt | |
# For running Python API tests | |
python3 -m pip install -r ${{ env.OPENVINO_REPO }}/src/bindings/python/src/compatibility/openvino/requirements-dev.txt | |
# For running ONNX frontend unit tests | |
python3 -m pip install -r ${{ env.OPENVINO_REPO }}/src/frontends/onnx/tests/requirements.txt | |
# For running TensorFlow frontend unit tests | |
python3 -m pip install -r ${{ env.OPENVINO_REPO }}/src/frontends/tensorflow/tests/requirements.txt | |
# For running Paddle frontend unit tests | |
python3 -m pip install -r ${{ env.OPENVINO_REPO }}/src/frontends/paddle/tests/requirements.txt | |
- name: Install MO dependencies | |
run: | | |
python3 -m pip install -r ${{ env.OPENVINO_REPO }}/tools/mo/requirements_mxnet.txt | |
python3 -m pip install -r ${{ env.OPENVINO_REPO }}/tools/mo/requirements_caffe.txt | |
python3 -m pip install -r ${{ env.OPENVINO_REPO }}/tools/mo/requirements_kaldi.txt | |
python3 -m pip install -r ${{ env.OPENVINO_REPO }}/tools/mo/requirements_onnx.txt | |
python3 -m pip install -r ${{ env.OPENVINO_REPO }}/tools/mo/requirements_tf2.txt | |
python3 -m pip install -r ${{ env.OPENVINO_REPO }}/tools/mo/requirements_dev.txt | |
- name: Install build dependencies | |
run: | | |
choco install --no-progress ninja | |
choco install --no-progress shellcheck | |
- name: Get tools versions | |
run: | | |
python3 --version | |
cmake --version | |
# | |
# Build | |
# | |
- name: Get number of CPU cores | |
uses: SimenB/github-actions-cpu-cores@v1 | |
id: cpu-cores | |
- uses: ilammy/msvc-dev-cmd@v1 | |
- name: Setup sccache | |
uses: hendrikmuhs/ccache-action@v1.2 | |
with: | |
variant: sccache | |
max-size: "2000M" | |
# Should save cache only if run in the master branch of the base repo | |
# github.ref_name is 'ref/PR_#' in case of the PR, and 'branch_name' when executed on push | |
# save: ${{ github.ref_name == 'master' && 'true' || 'false' }} | |
key: ${{ github.job }}-windows | |
restore-keys: | | |
${{ github.job }}-windows | |
- name: CMake configure | |
run: | | |
& {{ env.VCVARSPATH }} x64 && cmake -G "Ninja Multi-Config" ` | |
-DENABLE_CPPLINT=OFF ` | |
-DENABLE_ONEDNN_FOR_GPU=OFF ` | |
-DBUILD_SHARED_LIBS=OFF ` | |
-DENABLE_TESTS=ON ` | |
-DCMAKE_COMPILE_WARNING_AS_ERROR=OFF ` | |
-DENABLE_STRICT_DEPENDENCIES=OFF ` | |
-DENABLE_PYTHON=ON ` | |
-DBUILD_nvidia_plugin=OFF ` | |
-DCMAKE_DISABLE_FIND_PACKAGE_PkgConfig=ON ` | |
-DCUSTOM_OPERATIONS="calculate_grid;complex_mul;fft;grid_sample;sparse_conv;sparse_conv_transpose" ` | |
-DOPENVINO_EXTRA_MODULES=${{ env.OPENVINO_CONTRIB_REPO }}\modules ` | |
-DCMAKE_BUILD_TYPE=Release ` | |
-S ${{ env.OPENVINO_REPO }} ` | |
-B ${{ env.BUILD_DIR }} | |
- name: Build | |
run: | | |
& {{ env.VCVARSPATH }} x64 && cmake --build ${{ env.BUILD_DIR }} --parallel ${{ steps.cpu-cores.outputs.count }} --config Release | |
- name: Install | |
run: cmake -DCMAKE_INSTALL_PREFIX=${{ env.INSTALL_DIR }} -P ${{ env.BUILD_DIR }}/cmake_install.cmake | |
- name: Install Wheels | |
run: python3 -m pip install openvino-dev --find-links=${{ env.INSTALL_DIR }}\tools | |
- name: CMake Samples Tests | |
run: | | |
& {{ env.VCVARSPATH }} x64 && cmake -S ${{ env.OPENVINO_REPO }}/tests/samples_tests -B ${{ env.BUILD_DIR }}/samples_tests | |
- name: Install Samples Tests | |
run: cmake -DCOMPONENT=tests -DCMAKE_INSTALL_PREFIX=${{ env.INSTALL_DIR }} -P ${{ env.BUILD_DIR }}/samples_tests/cmake_install.cmake | |
- name: Install Tests | |
run: cmake -DCMAKE_INSTALL_PREFIX=${{ env.INSTALL_DIR }} -DCOMPONENT=tests -P ${{ env.BUILD_DIR }}\cmake_install.cmake | |
- name: Cmake Layer Tests | |
run: | | |
& {{ env.VCVARSPATH }} x64 && cmake -S ${{ env.OPENVINO_REPO }}/tests/layer_tests -B ${{ env.BUILD_DIR }}/layer_tests | |
- name: Build Layer Tests | |
run: cmake --build ${{ env.BUILD_DIR }}/layer_tests --parallel --config Release | |
- name: Install Layer Tests | |
run: cmake -DCOMPONENT=tests -DCMAKE_INSTALL_PREFIX=${{ env.INSTALL_DIR }} -P ${{ env.BUILD_DIR }}/layer_tests/cmake_install.cmake | |
- name: Pack Artifacts | |
run: | | |
$file=Get-ChildItem -Path "${{ env.INSTALL_DIR }}" -Exclude "tests" | |
$compress = @{ | |
Path = $file | |
CompressionLevel = "Optimal" | |
DestinationPath = "${{ env.BUILD_DIR }}/openvino_package.zip" | |
} | |
Compress-Archive @compress | |
$file=Get-ChildItem -Path "${{ env.INSTALL_DIR }}\tests" | |
$compress = @{ | |
Path = $file | |
CompressionLevel = "Optimal" | |
DestinationPath = "${{ env.BUILD_DIR }}/openvino_tests.zip" | |
} | |
Compress-Archive @compress | |
- name: Build cpp samples | |
run: | | |
& {{ env.VCVARSPATH }} x64 | |
& ${{ env.SAMPLES_INSTALL_DIR }}/cpp/build_samples_msvc.bat -i ${{ env.INSTALL_DIR }} | |
- name: Build c samples | |
run: | | |
& {{ env.VCVARSPATH }} x64 | |
& ${{ env.SAMPLES_INSTALL_DIR }}/c/build_samples_msvc.bat -i ${{ env.INSTALL_DIR }} | |
- name: Samples tests | |
shell: cmd | |
run: | | |
python3 -m pip install --ignore-installed PyYAML -r ${{ env.INSTALL_TEST_DIR }}/smoke_tests/requirements.txt | |
call "${{ env.INSTALL_DIR }}\\setupvars.bat" && python3 -m pytest -sv ${{ env.INSTALL_TEST_DIR }}/smoke_tests --env_conf ${{ env.INSTALL_TEST_DIR }}/smoke_tests/env_config.yml --junitxml=${{ env.INSTALL_TEST_DIR }}/TEST-SamplesSmokeTests.xml | |
env: | |
IE_APP_PATH: ${{ env.INSTALL_DIR }}/samples_bin | |
IE_APP_PYTHON_PATH: ${{ env.INSTALL_DIR }}/samples/python | |
SHARE: ${{ env.INSTALL_TEST_DIR }}/smoke_tests/samples_smoke_tests_data | |
WORKSPACE: ${{ env.INSTALL_DIR }} | |
# Present in the "Build" job due to the fact that these tests require build directory | |
- name: ONNX frontend tests | |
shell: cmd | |
run: | | |
call "${{ env.INSTALL_DIR }}\\setupvars.bat" && ${{ env.INSTALL_TEST_DIR }}/ov_onnx_frontend_tests --gtest_print_time=1 --gtest_filter=-*IE_GPU* --gtest_output=xml:${{ env.INSTALL_TEST_DIR }}/TEST-ONNXFrontend.xml | |
- name: List installed files | |
if: ${{ always() }} | |
run: | | |
Get-ChildItem -Recurse -Directory ${{ env.INSTALL_DIR }} | |
- name: Upload openvino package | |
uses: actions/upload-artifact@v3 | |
with: | |
name: openvino_package | |
path: ${{ env.BUILD_DIR }}/openvino_package.zip | |
if-no-files-found: 'error' | |
- name: Upload openvino tests package | |
uses: actions/upload-artifact@v3 | |
with: | |
name: openvino_tests | |
path: ${{ env.BUILD_DIR }}/openvino_tests.zip | |
if-no-files-found: 'error' | |
# MO_UT_PyTorch_LT: | |
# needs: Build | |
# defaults: | |
# run: | |
# shell: pwsh | |
# runs-on: 'windows-latest' | |
# env: | |
# OPENVINO_REPO: "${{ github.workspace }}\\openvino" | |
# OPENVINO_CONTRIB_REPO: "${{ github.workspace }}\\openvino_contrib" | |
# INSTALL_DIR: "${{ github.workspace }}\\install" | |
# INSTALL_TEST_DIR: "${{ github.workspace }}\\install\\tests" | |
# SAMPLES_INSTALL_DIR: "${{ github.workspace }}\\install\\samples" | |
# LAYER_TESTS_INSTALL_DIR: "${{ github.workspace }}\\install\\tests\\layer_tests" | |
# BUILD_DIR: "${{ github.workspace }}\\build" | |
# DATA_PATH: "${{ github.workspace }}\\testdata" | |
# MODELS_PATH: "${{ github.workspace }}\\testdata" | |
# PYTHON_STATIC_ARGS: -m "not dynamic_library and not template_plugin" | |
# | |
# steps: | |
# - name: Create Directories | |
# run: | | |
# mkdir ${{ env.INSTALL_DIR }} | |
# mkdir ${{ env.INSTALL_TEST_DIR }} | |
# | |
# - name: Download OpenVINO package | |
# uses: actions/download-artifact@v3 | |
# with: | |
# name: openvino_package | |
# path: ${{ env.INSTALL_DIR }} | |
# | |
# - name: Download OpenVINO tests package | |
# uses: actions/download-artifact@v3 | |
# with: | |
# name: openvino_tests | |
# path: ${{ env.INSTALL_TEST_DIR }} | |
# | |
# - name: Extract OpenVINO packages | |
# run: | | |
# pushd ${{ env.INSTALL_DIR }} | |
# Expand-Archive openvino_package.zip -DestinationPath "${{ env.INSTALL_DIR }}" | |
# popd | |
# pushd ${{ env.INSTALL_TEST_DIR }} | |
# Expand-Archive openvino_tests.zip -DestinationPath "${{ env.INSTALL_TEST_DIR }}" | |
# popd | |
# | |
# - name: Check extraction | |
# run: | | |
# ls "${{ github.workspace }}" | |
# ls "${{ env.INSTALL_DIR }}" | |
# ls "${{ env.INSTALL_TEST_DIR }}" | |
# | |
# # TODO: do not provide 'ref' and 'repository' when scheduled on actual PRs | |
# - name: Clone OpenVINO | |
# uses: actions/checkout@v3 | |
# with: | |
# repository: 'akashchi/openvino' | |
# ref: 'ci/test-gha' | |
# path: 'openvino' | |
# submodules: 'recursive' | |
# | |
# - name: Clone OpenVINO Contrib | |
# uses: actions/checkout@v3 | |
# with: | |
# repository: 'openvinotoolkit/openvino_contrib' | |
# path: 'openvino_contrib' | |
# submodules: 'recursive' | |
# | |
# # Have to use py3.7 due to https://stackoverflow.com/questions/59507552/from-mxnet-import-nd-results-in-syntaxerror-yield-inside-list-comprehension & ticket 95984 | |
# - uses: actions/setup-python@v4 | |
# with: | |
# python-version: '3.7' | |
# | |
# - name: Install python dependencies | |
# run: | | |
# # For Python API | |
# python3 -m pip install -r ${{ env.OPENVINO_REPO }}/src/bindings/python/wheel/requirements-dev.txt | |
# python3 -m pip install -r ${{ env.OPENVINO_REPO }}/src/bindings/python/requirements.txt | |
# | |
# # For running Python API tests | |
# python3 -m pip install -r ${{ env.OPENVINO_REPO }}/src/bindings/python/src/compatibility/openvino/requirements-dev.txt | |
# | |
# # For running ONNX frontend unit tests | |
# python3 -m pip install --force-reinstall -r ${{ env.OPENVINO_REPO }}/src/frontends/onnx/tests/requirements.txt | |
# | |
# # For running TensorFlow frontend unit tests | |
# python3 -m pip install -r ${{ env.OPENVINO_REPO }}/src/frontends/tensorflow/tests/requirements.txt | |
# | |
# # For running Paddle frontend unit tests | |
# python3 -m pip install -r ${{ env.OPENVINO_REPO }}/src/frontends/paddle/tests/requirements.txt | |
# | |
# - name: Install MO dependencies | |
# run: | | |
# python3 -m pip install -r ${{ env.OPENVINO_REPO }}/tools/mo/requirements_mxnet.txt | |
# python3 -m pip install -r ${{ env.OPENVINO_REPO }}/tools/mo/requirements_caffe.txt | |
# python3 -m pip install -r ${{ env.OPENVINO_REPO }}/tools/mo/requirements_kaldi.txt | |
# python3 -m pip install -r ${{ env.OPENVINO_REPO }}/tools/mo/requirements_onnx.txt | |
# python3 -m pip install -r ${{ env.OPENVINO_REPO }}/tools/mo/requirements_tf2.txt | |
# python3 -m pip install -r ${{ env.OPENVINO_REPO }}/tools/mo/requirements_dev.txt | |
# | |
# - name: Install Python wheels | |
# run: | | |
# python3 -m pip install openvino-dev --force-reinstall --find-links=${{ env.INSTALL_DIR }}\tools | |
# | |
# - name: Model Optimizer UT | |
# shell: cmd | |
# run: | | |
# set PYTHONPATH=${{ env.OPENVINO_REPO }}\tools\mo;${{ env.LAYER_TESTS_INSTALL_DIR }};${{ env.INSTALL_TEST_DIR }};${{ env.INSTALL_DIR }}\python\python3.7;%PYTHONPATH% | |
# | |
# call "${{ env.INSTALL_DIR }}\\setupvars.bat" -pyver 3.7 && python -m pytest -s ${{ env.INSTALL_TEST_DIR }}/mo/unit_tests --junitxml=${{ env.INSTALL_TEST_DIR }}/TEST-ModelOptimizer.xml | |
# | |
# - name: PyTorch Layer Tests | |
# shell: cmd | |
# run: | | |
# python3 -m pip install -r ${{ env.LAYER_TESTS_INSTALL_DIR }}/requirements.txt | |
# | |
# set PYTHONPATH=${{ env.OPENVINO_REPO }}\tools\mo;${{ env.LAYER_TESTS_INSTALL_DIR }};%PYTHONPATH% | |
# | |
# call "${{ env.INSTALL_DIR }}\\setupvars.bat" -pyver 3.7 && python3 -m pytest ${{ env.LAYER_TESTS_INSTALL_DIR }}/pytorch_tests -m precommit --junitxml=${{ env.INSTALL_TEST_DIR }}/TEST-pytorch.xml | |
# env: | |
# TEST_DEVICE: CPU | |
# | |
# - name: Upload Test Results | |
# uses: actions/upload-artifact@v3 | |
# if: ${{ always() }} | |
# with: | |
# name: test-results-python | |
# path: ${{ env.INSTALL_TEST_DIR }}/TEST*.xml | |
# if-no-files-found: 'error' | |
Python_Unit_Tests: | |
needs: Build | |
defaults: | |
run: | |
shell: pwsh | |
runs-on: 'windows-latest' | |
env: | |
OPENVINO_REPO: "${{ github.workspace }}\\openvino" | |
OPENVINO_CONTRIB_REPO: "${{ github.workspace }}\\openvino_contrib" | |
INSTALL_DIR: "${{ github.workspace }}\\install" | |
INSTALL_TEST_DIR: "${{ github.workspace }}\\install\\tests" | |
SAMPLES_INSTALL_DIR: "${{ github.workspace }}\\install\\samples" | |
LAYER_TESTS_INSTALL_DIR: "${{ github.workspace }}\\install\\tests\\layer_tests" | |
BUILD_DIR: "${{ github.workspace }}\\build" | |
DATA_PATH: "${{ github.workspace }}\\testdata" | |
MODELS_PATH: "${{ github.workspace }}\\testdata" | |
PYTHON_STATIC_ARGS: -m "not dynamic_library and not template_plugin" | |
steps: | |
- name: Create Directories | |
run: | | |
mkdir ${{ env.INSTALL_DIR }} | |
mkdir ${{ env.INSTALL_TEST_DIR }} | |
- name: Download OpenVINO package | |
uses: actions/download-artifact@v3 | |
with: | |
name: openvino_package | |
path: ${{ env.INSTALL_DIR }} | |
- name: Download OpenVINO tests package | |
uses: actions/download-artifact@v3 | |
with: | |
name: openvino_tests | |
path: ${{ env.INSTALL_TEST_DIR }} | |
- name: Extract OpenVINO packages | |
run: | | |
pushd ${{ env.INSTALL_DIR }} | |
Expand-Archive openvino_package.zip -DestinationPath "${{ env.INSTALL_DIR }}" | |
popd | |
pushd ${{ env.INSTALL_TEST_DIR }} | |
Expand-Archive openvino_tests.zip -DestinationPath "${{ env.INSTALL_TEST_DIR }}" | |
popd | |
- name: Check extraction | |
run: | | |
ls "${{ github.workspace }}" | |
ls "${{ env.INSTALL_DIR }}" | |
ls "${{ env.INSTALL_TEST_DIR }}" | |
# TODO: do not provide 'ref' and 'repository' when scheduled on actual PRs | |
- name: Clone OpenVINO | |
uses: actions/checkout@v3 | |
with: | |
repository: 'akashchi/openvino' | |
ref: 'ci/test-gha' | |
path: 'openvino' | |
submodules: 'recursive' | |
- name: Clone OpenVINO Contrib | |
uses: actions/checkout@v3 | |
with: | |
repository: 'openvinotoolkit/openvino_contrib' | |
path: 'openvino_contrib' | |
submodules: 'recursive' | |
- uses: actions/setup-python@v4 | |
with: | |
python-version: '3.11' | |
cache: 'pip' | |
cache-dependency-path: '**/requirements*.txt' | |
- name: Install python dependencies | |
run: | | |
# For Python API | |
python3 -m pip install -r ${{ env.OPENVINO_REPO }}/src/bindings/python/wheel/requirements-dev.txt | |
python3 -m pip install -r ${{ env.OPENVINO_REPO }}/src/bindings/python/requirements.txt | |
# For running Python API tests | |
python3 -m pip install -r ${{ env.OPENVINO_REPO }}/src/bindings/python/src/compatibility/openvino/requirements-dev.txt | |
# For running ONNX frontend unit tests | |
python3 -m pip install --force-reinstall -r ${{ env.OPENVINO_REPO }}/src/frontends/onnx/tests/requirements.txt | |
# For running TensorFlow frontend unit tests | |
python3 -m pip install -r ${{ env.OPENVINO_REPO }}/src/frontends/tensorflow/tests/requirements.txt | |
# For running Paddle frontend unit tests | |
python3 -m pip install -r ${{ env.OPENVINO_REPO }}/src/frontends/paddle/tests/requirements.txt | |
- name: Install MO dependencies | |
run: | | |
python3 -m pip install -r ${{ env.OPENVINO_REPO }}/tools/mo/requirements_mxnet.txt | |
python3 -m pip install -r ${{ env.OPENVINO_REPO }}/tools/mo/requirements_caffe.txt | |
python3 -m pip install -r ${{ env.OPENVINO_REPO }}/tools/mo/requirements_kaldi.txt | |
python3 -m pip install -r ${{ env.OPENVINO_REPO }}/tools/mo/requirements_onnx.txt | |
python3 -m pip install -r ${{ env.OPENVINO_REPO }}/tools/mo/requirements_tf2.txt | |
python3 -m pip install -r ${{ env.OPENVINO_REPO }}/tools/mo/requirements_dev.txt | |
- name: Install Python wheels | |
run: | | |
python3 -m pip install openvino-dev --force-reinstall --find-links=${{ env.INSTALL_DIR }}\tools | |
- name: nGraph and IE Python Bindings Tests | |
if: ${{ always() }} | |
shell: cmd | |
run: | | |
set PYTHONPATH=${{ env.OPENVINO_REPO }}\tools\mo;${{ env.LAYER_TESTS_INSTALL_DIR }};%PYTHONPATH% | |
call "${{ env.INSTALL_DIR }}\\setupvars.bat" && python3 -m pytest -s ${{ env.INSTALL_TEST_DIR }}/pyngraph ${{ env.PYTHON_STATIC_ARGS }} --junitxml=${{ env.INSTALL_TEST_DIR }}/TEST-Pyngraph.xml --ignore=${{ env.INSTALL_TEST_DIR }}/pyngraph/tests/test_onnx/test_zoo_models.py --ignore=${{ env.INSTALL_TEST_DIR }}/pyngraph/tests/test_onnx/test_backend.py | |
- name: Python API 2.0 Tests | |
if: ${{ always() }} | |
shell: cmd | |
run: | | |
set PYTHONPATH=${{ env.OPENVINO_REPO }}\tools\mo;${{ env.LAYER_TESTS_INSTALL_DIR }};%PYTHONPATH% | |
call "${{ env.INSTALL_DIR }}\\setupvars.bat" && python3 -m pytest -sv ${{ env.INSTALL_TEST_DIR }}/pyopenvino ${{ env.PYTHON_STATIC_ARGS }} --junitxml=${{ env.INSTALL_TEST_DIR }}/TEST-Pyngraph.xml --ignore=${{ env.INSTALL_TEST_DIR }}/pyopenvino/tests/test_utils/test_utils.py --ignore=${{ env.INSTALL_TEST_DIR }}/pyopenvino/tests/test_onnx/test_zoo_models.py --ignore=${{ env.INSTALL_TEST_DIR }}/pyopenvino/tests/test_onnx/test_backend.py | |
- name: Model Optimizer UT | |
if: ${{ always() }} | |
shell: cmd | |
run: | | |
set PYTHONPATH=${{ env.OPENVINO_REPO }}\tools\mo;${{ env.LAYER_TESTS_INSTALL_DIR }};${{ env.INSTALL_TEST_DIR }};${{ env.INSTALL_DIR }}\python\python3.11;%PYTHONPATH% | |
:: TODO: figure out why they need to be reinstalled | |
python3 -m pip install -r ${{ env.OPENVINO_REPO }}/tools/mo/requirements_mxnet.txt | |
python3 -m pip install -r ${{ env.OPENVINO_REPO }}/tools/mo/requirements_caffe.txt | |
python3 -m pip install -r ${{ env.OPENVINO_REPO }}/tools/mo/requirements_kaldi.txt | |
python3 -m pip install -r ${{ env.OPENVINO_REPO }}/tools/mo/requirements_onnx.txt | |
python3 -m pip install -r ${{ env.OPENVINO_REPO }}/tools/mo/requirements_tf2.txt | |
python3 -m pip install -r ${{ env.OPENVINO_REPO }}/tools/mo/requirements_dev.txt | |
call "${{ env.INSTALL_DIR }}\\setupvars.bat" && python3 -m pytest -s ${{ env.INSTALL_TEST_DIR }}/mo/unit_tests --junitxml=${{ env.INSTALL_TEST_DIR }}/TEST-ModelOptimizer.xml | |
- name: PyTorch Layer Tests | |
if: ${{ always() }} | |
shell: cmd | |
run: | | |
python3 -m pip install -r ${{ env.LAYER_TESTS_INSTALL_DIR }}/requirements.txt | |
set PYTHONPATH=${{ env.OPENVINO_REPO }}\tools\mo;${{ env.LAYER_TESTS_INSTALL_DIR }};%PYTHONPATH% | |
call "${{ env.INSTALL_DIR }}\\setupvars.bat" && python3 -m pytest ${{ env.LAYER_TESTS_INSTALL_DIR }}/pytorch_tests -m precommit --junitxml=${{ env.INSTALL_TEST_DIR }}/TEST-pytorch.xml | |
env: | |
TEST_DEVICE: CPU | |
- name: TensorFlow 1 Layer Tests - TF FE | |
if: ${{ always() }} | |
shell: cmd | |
run: | | |
python3 -m pip install -r ${{ env.LAYER_TESTS_INSTALL_DIR }}/requirements.txt | |
set PYTHONPATH=${{ env.OPENVINO_REPO }}\tools\mo;${{ env.LAYER_TESTS_INSTALL_DIR }};%PYTHONPATH% | |
call "${{ env.INSTALL_DIR }}\\setupvars.bat" && python3 -m pytest ${{ env.LAYER_TESTS_INSTALL_DIR }}/tensorflow_tests/ --use_new_frontend -m precommit_tf_fe --junitxml=${{ env.INSTALL_TEST_DIR }}/TEST-tf_fe.xml | |
env: | |
TEST_DEVICE: CPU | |
- name: TensorFlow 2 Layer Tests - TF FE | |
if: ${{ always() }} | |
shell: cmd | |
run: | | |
python3 -m pip install -r ${{ env.LAYER_TESTS_INSTALL_DIR }}/requirements.txt | |
set PYTHONPATH=${{ env.OPENVINO_REPO }}\tools\mo;${{ env.LAYER_TESTS_INSTALL_DIR }};%PYTHONPATH% | |
call "${{ env.INSTALL_DIR }}\\setupvars.bat" && python3 -m pytest ${{ env.LAYER_TESTS_INSTALL_DIR }}/tensorflow2_keras_tests/ --use_new_frontend -m precommit_tf_fe --junitxml=${{ env.INSTALL_TEST_DIR }}/TEST-tf2_fe.xml | |
env: | |
TEST_DEVICE: CPU | |
- name: TensorFlow 1 Layer Tests - Legacy FE | |
if: ${{ always() }} | |
shell: cmd | |
run: | | |
python3 -m pip install -r ${{ env.LAYER_TESTS_INSTALL_DIR }}/requirements.txt | |
set PYTHONPATH=${{ env.OPENVINO_REPO }}\tools\mo;${{ env.LAYER_TESTS_INSTALL_DIR }};%PYTHONPATH% | |
call "${{ env.INSTALL_DIR }}\\setupvars.bat" && python3 -m pytest ${{ env.LAYER_TESTS_INSTALL_DIR }}/tensorflow_tests/test_tf_Roll.py --ir_version=10 --junitxml=${{ env.INSTALL_TEST_DIR }}/TEST-tf_Roll.xml | |
- name: TensorFlow 2 Layer Tests - Legacy FE | |
if: ${{ always() }} | |
shell: cmd | |
run: | | |
python3 -m pip install -r ${{ env.LAYER_TESTS_INSTALL_DIR }}/requirements.txt | |
set PYTHONPATH=${{ env.OPENVINO_REPO }}\tools\mo;${{ env.LAYER_TESTS_INSTALL_DIR }};%PYTHONPATH% | |
call "${{ env.INSTALL_DIR }}\\setupvars.bat" && python3 -m pytest ${{ env.LAYER_TESTS_INSTALL_DIR }}/tensorflow2_keras_tests/test_tf2_keras_activation.py --ir_version=11 --junitxml=${{ env.INSTALL_TEST_DIR }}/TEST-tf2_Activation.xml -k "sigmoid" | |
env: | |
TEST_DEVICE: CPU | |
- name: TensorFlow Lite Layer Tests - TFL FE | |
if: ${{ always() }} | |
shell: cmd | |
run: | | |
python3 -m pip install -r ${{ env.LAYER_TESTS_INSTALL_DIR }}/requirements.txt | |
set PYTHONPATH=${{ env.OPENVINO_REPO }}\tools\mo;${{ env.LAYER_TESTS_INSTALL_DIR }};%PYTHONPATH% | |
call "${{ env.INSTALL_DIR }}\\setupvars.bat" && python3 -m pytest ${{ env.LAYER_TESTS_INSTALL_DIR }}/tensorflow_lite_tests/ --junitxml=${{ env.INSTALL_TEST_DIR }}/TEST-tfl_fe.xml | |
env: | |
TEST_DEVICE: CPU | |
- name: MO Python API Tests | |
if: ${{ always() }} | |
shell: cmd | |
run: | | |
python3 -m pip install -r ${{ env.LAYER_TESTS_INSTALL_DIR }}/requirements.txt | |
set PYTHONPATH=${{ env.OPENVINO_REPO }}\tools\mo;${{ env.LAYER_TESTS_INSTALL_DIR }};%PYTHONPATH% | |
call "${{ env.INSTALL_DIR }}\\setupvars.bat" && python3 -m pytest ${{ env.LAYER_TESTS_INSTALL_DIR }}/mo_python_api_tests --junitxml=${{ env.INSTALL_TEST_DIR }}/TEST-test_mo_convert.xml | |
env: | |
TEST_DEVICE: CPU | |
- name: Python Frontend tests | |
if: ${{ always() }} | |
shell: cmd | |
run: | | |
python3 -m pip install -r ${{ env.LAYER_TESTS_INSTALL_DIR }}/requirements.txt | |
set PYTHONPATH=${{ env.OPENVINO_REPO }}\tools\mo;${{ env.LAYER_TESTS_INSTALL_DIR }};%PYTHONPATH% | |
call "${{ env.INSTALL_DIR }}\\setupvars.bat" && python3 -m pytest ${{ env.LAYER_TESTS_INSTALL_DIR }}/py_frontend_tests --junitxml=${{ env.INSTALL_TEST_DIR }}/TEST-test_py_fontend.xml | |
- name: Upload Test Results | |
uses: actions/upload-artifact@v3 | |
if: ${{ always() }} | |
with: | |
name: test-results-python | |
path: ${{ env.INSTALL_TEST_DIR }}/TEST*.xml | |
if-no-files-found: 'error' | |
CXX_Unit_Tests: | |
needs: Build | |
defaults: | |
run: | |
shell: pwsh | |
runs-on: 'windows-latest' | |
env: | |
INSTALL_DIR: "${{ github.workspace }}\\install" | |
INSTALL_TEST_DIR: "${{ github.workspace }}\\install\\tests" | |
steps: | |
- name: Create Directories | |
run: | | |
mkdir ${{ env.INSTALL_DIR }} | |
mkdir ${{ env.INSTALL_TEST_DIR }} | |
- name: Download OpenVINO package | |
uses: actions/download-artifact@v3 | |
with: | |
name: openvino_package | |
path: ${{ env.INSTALL_DIR }} | |
- name: Download OpenVINO tests package | |
uses: actions/download-artifact@v3 | |
with: | |
name: openvino_tests | |
path: ${{ env.INSTALL_TEST_DIR }} | |
- name: Extract OpenVINO packages | |
run: | | |
pushd ${{ env.INSTALL_DIR }} | |
Expand-Archive openvino_package.zip -DestinationPath "${{ env.INSTALL_DIR }}" | |
popd | |
pushd ${{ env.INSTALL_TEST_DIR }} | |
Expand-Archive openvino_tests.zip -DestinationPath "${{ env.INSTALL_TEST_DIR }}" | |
popd | |
- name: Check extraction | |
run: | | |
ls "${{ github.workspace }}" | |
ls "${{ env.INSTALL_DIR }}" | |
ls "${{ env.INSTALL_TEST_DIR }}" | |
- name: OpenVINO Core unit tests | |
shell: cmd | |
run: | | |
call "${{ env.INSTALL_DIR }}\\setupvars.bat" && ${{ env.INSTALL_TEST_DIR }}/ov_core_unit_tests --gtest_print_time=1 --gtest_filter=-*IE_GPU* --gtest_output=xml:${{ env.INSTALL_TEST_DIR }}/TEST-NGraphUT.xml | |
- name: OpenVINO Inference functional tests | |
shell: cmd | |
run: | | |
call "${{ env.INSTALL_DIR }}\\setupvars.bat" && ${{ env.INSTALL_TEST_DIR }}/ov_inference_functional_tests --gtest_print_time=1 --gtest_output=xml:${{ env.INSTALL_TEST_DIR }}/TEST-InferenceFunc.xml | |
- name: OpenVINO Inference unit tests | |
shell: cmd | |
run: | | |
call "${{ env.INSTALL_DIR }}\\setupvars.bat" && ${{ env.INSTALL_TEST_DIR }}/ov_inference_unit_tests --gtest_print_time=1 --gtest_output=xml:${{ env.INSTALL_TEST_DIR }}/TEST-InferenceUnit.xml | |
- name: Low Precision Transformations Tests | |
shell: cmd | |
run: | | |
call "${{ env.INSTALL_DIR }}\\setupvars.bat" && ${{ env.INSTALL_TEST_DIR }}/ov_lp_transformations_tests --gtest_print_time=1 --gtest_output=xml:${{ env.INSTALL_TEST_DIR }}/TEST-LpTransformations.xml | |
- name: OpenVINO Conditional compilation tests | |
shell: cmd | |
run: | | |
call "${{ env.INSTALL_DIR }}\\setupvars.bat" && ${{ env.INSTALL_TEST_DIR }}/ov_conditional_compilation_tests --gtest_print_time=1 --gtest_output=xml:${{ env.INSTALL_TEST_DIR }}/TEST-ConditionalCompilation.xml | |
- name: IR frontend tests | |
shell: cmd | |
run: | | |
call "${{ env.INSTALL_DIR }}\\setupvars.bat" && ${{ env.INSTALL_TEST_DIR }}/ov_ir_frontend_tests --gtest_print_time=1 --gtest_output=xml:${{ env.INSTALL_TEST_DIR }}/TEST-IRFrontend.xml | |
# - name: PaddlePaddle frontend tests # Disabled in Azure: https://github.com/openvinotoolkit/openvino/blob/master/.ci/azure/linux.yml#L403 | |
# shell: cmd | |
# run: | | |
# call "${{ env.INSTALL_DIR }}\\setupvars.bat" && ${{ env.INSTALL_TEST_DIR }}/paddle_tests --gtest_print_time=1 --gtest_filter=*smoke* --gtest_output=xml:${{ env.INSTALL_TEST_DIR }}/TEST-PaddleTests.xml | |
# - name: ONNX frontend tests # Present in the "Build" job due to the fact that these tests require build directory | |
# shell: cmd | |
# run: | | |
# call "${{ env.INSTALL_DIR }}\\setupvars.bat" && ${{ env.INSTALL_TEST_DIR }}/ov_onnx_frontend_tests --gtest_print_time=1 --gtest_filter=-*IE_GPU* --gtest_output=xml:${{ env.INSTALL_TEST_DIR }}/TEST-ONNXFrontend.xml | |
- name: TensorFlow Common tests | |
shell: cmd | |
run: | | |
call "${{ env.INSTALL_DIR }}\\setupvars.bat" && ${{ env.INSTALL_TEST_DIR }}/ov_tensorflow_common_tests --gtest_print_time=1 --gtest_output=xml:${{ env.INSTALL_TEST_DIR }}/TEST-TensorFlowCommonFrontend.xml | |
- name: TensorFlow frontend tests | |
shell: cmd | |
run: | | |
call "${{ env.INSTALL_DIR }}\\setupvars.bat" && ${{ env.INSTALL_TEST_DIR }}/ov_tensorflow_frontend_tests --gtest_print_time=1 --gtest_output=xml:${{ env.INSTALL_TEST_DIR }}/TEST-TensorFlowFrontend.xml | |
- name: TensorFlow Lite frontend tests | |
shell: cmd | |
run: | | |
call "${{ env.INSTALL_DIR }}\\setupvars.bat" && ${{ env.INSTALL_TEST_DIR }}/ov_tensorflow_lite_frontend_tests --gtest_print_time=1 --gtest_output=xml:${{ env.INSTALL_TEST_DIR }}/TEST-TensorFlowLiteFrontend.xml | |
- name: Transformations Tests | |
shell: cmd | |
run: | | |
call "${{ env.INSTALL_DIR }}\\setupvars.bat" && ${{ env.INSTALL_TEST_DIR }}/ov_transformations_tests --gtest_print_time=1 --gtest_output=xml:${{ env.INSTALL_TEST_DIR }}/TEST-Transformations.xml | |
- name: Common test utils tests | |
shell: cmd | |
run: | | |
call "${{ env.INSTALL_DIR }}\\setupvars.bat" && ${{ env.INSTALL_TEST_DIR }}/commonUtilsTests --gtest_print_time=1 --gtest_output=xml:${{ env.INSTALL_TEST_DIR }}/TEST-commonUtilsTests.xml | |
- name: CPU plugin unit tests | |
shell: cmd | |
run: | | |
call "${{ env.INSTALL_DIR }}\\setupvars.bat" && ${{ env.INSTALL_TEST_DIR }}/ov_cpu_unit_tests --gtest_print_time=1 --gtest_output=xml:${{ env.INSTALL_TEST_DIR }}/TEST-CPUUnitTests.xml | |
# - name: GNA plugin unit tests # Disabled in Azure: https://github.com/openvinotoolkit/openvino/blob/master/.ci/azure/linux.yml#L434 | |
# shell: cmd | |
# run: | | |
# call "${{ env.INSTALL_DIR }}\\setupvars.bat" && ${{ env.INSTALL_TEST_DIR }}/ov_gna_unit_tests --gtest_print_time=1 --gtest_output=xml:${{ env.INSTALL_TEST_DIR }}/TEST-GNAUnitTests.xml | |
- name: AUTO UT | |
shell: cmd | |
run: | | |
call "${{ env.INSTALL_DIR }}\\setupvars.bat" && ${{ env.INSTALL_TEST_DIR }}/ov_auto_unit_tests --gtest_print_time=1 --gtest_output=xml:${{ env.INSTALL_TEST_DIR }}/TEST-ov_auto_unit_tests.xml | |
- name: Template plugin tests | |
shell: cmd | |
run: | | |
call "${{ env.INSTALL_DIR }}\\setupvars.bat" && ${{ env.INSTALL_TEST_DIR }}/ov_template_func_tests --gtest_print_time=1 --gtest_filter=*smoke* --gtest_output=xml:${{ env.INSTALL_TEST_DIR }}/TEST-TemplateFuncTests.xml | |
- name: Inference Engine C API tests | |
shell: cmd | |
run: | | |
call "${{ env.INSTALL_DIR }}\\setupvars.bat" && ${{ env.INSTALL_TEST_DIR }}/InferenceEngineCAPITests --gtest_print_time=1 --gtest_output=xml:${{ env.INSTALL_TEST_DIR }}/TEST-InferenceEngineCAPITests.xml | |
- name: OpenVINO C API tests | |
shell: cmd | |
run: | | |
call "${{ env.INSTALL_DIR }}\\setupvars.bat" && ${{ env.INSTALL_TEST_DIR }}/ov_capi_test --gtest_print_time=1 --gtest_output=xml:${{ env.INSTALL_TEST_DIR }}/TEST-OpenVINOCAPITests.xml | |
- name: AutoBatch FuncTests | |
shell: cmd | |
run: | | |
call "${{ env.INSTALL_DIR }}\\setupvars.bat" && ${{ env.INSTALL_TEST_DIR }}/ov_auto_batch_func_tests --gtest_output=xml:${{ env.INSTALL_TEST_DIR }}/TEST-ov_auto_batch_func_tests.xml | |
- name: Upload Test Results | |
uses: actions/upload-artifact@v3 | |
if: ${{ always() }} | |
with: | |
name: test-results-cpp | |
path: ${{ env.INSTALL_TEST_DIR }}/TEST*.xml | |
if-no-files-found: 'error' | |
CPU_Functional_Tests: | |
needs: Build | |
defaults: | |
run: | |
shell: pwsh | |
runs-on: 'windows-latest' | |
# runs-on: 'windows-latest-8-cores' | |
env: | |
INSTALL_DIR: "${{ github.workspace }}\\install" | |
INSTALL_TEST_DIR: "${{ github.workspace }}\\install\\tests" | |
steps: | |
- name: Create Directories | |
run: | | |
mkdir ${{ env.INSTALL_DIR }} | |
mkdir ${{ env.INSTALL_TEST_DIR }} | |
- name: Download OpenVINO package | |
uses: actions/download-artifact@v3 | |
with: | |
name: openvino_package | |
path: ${{ env.INSTALL_DIR }} | |
- name: Download OpenVINO tests package | |
uses: actions/download-artifact@v3 | |
with: | |
name: openvino_tests | |
path: ${{ env.INSTALL_TEST_DIR }} | |
- name: Extract OpenVINO packages | |
run: | | |
pushd ${{ env.INSTALL_DIR }} | |
Expand-Archive openvino_package.zip -DestinationPath "${{ env.INSTALL_DIR }}" | |
popd | |
pushd ${{ env.INSTALL_TEST_DIR }} | |
Expand-Archive openvino_tests.zip -DestinationPath "${{ env.INSTALL_TEST_DIR }}" | |
popd | |
- name: Check extraction | |
run: | | |
ls "${{ github.workspace }}" | |
ls "${{ env.INSTALL_DIR }}" | |
ls "${{ env.INSTALL_TEST_DIR }}" | |
- name: Intel CPU plugin func tests | |
shell: cmd | |
run: | | |
call "${{ env.INSTALL_DIR }}\\setupvars.bat" && ${{ env.INSTALL_TEST_DIR }}/ov_cpu_func_tests --gtest_print_time=1 --gtest_filter=*smoke* --gtest_output=xml:${{ env.INSTALL_TEST_DIR }}/TEST-CPUFuncTests.xml | |
- name: Upload Test Results | |
uses: actions/upload-artifact@v3 | |
if: ${{ always() }} | |
with: | |
name: test-results-functional-cpu | |
path: ${{ env.INSTALL_TEST_DIR }}/TEST*.xml | |
if-no-files-found: 'error' | |
# GPU_Functional_Tests: # fails due to absence of the GPU | |
# needs: Build | |
# defaults: | |
# run: | |
# shell: pwsh | |
# runs-on: 'windows-latest' | |
# env: | |
# INSTALL_DIR: "${{ github.workspace }}\\install" | |
# INSTALL_TEST_DIR: "${{ github.workspace }}\\install\\tests" | |
# GTEST_PARALLEL_DIR: "${{ github.workspace }}\\gtest_parallel" | |
# | |
# steps: | |
# - name: Create Directories | |
# run: | | |
# mkdir ${{ env.INSTALL_DIR }} | |
# mkdir ${{ env.INSTALL_TEST_DIR }} | |
# mkdir ${{ env.GTEST_PARALLEL_DIR }} | |
# | |
# - name: Download OpenVINO package | |
# uses: actions/download-artifact@v3 | |
# with: | |
# name: openvino_package | |
# path: ${{ env.INSTALL_DIR }} | |
# | |
# - name: Download OpenVINO tests package | |
# uses: actions/download-artifact@v3 | |
# with: | |
# name: openvino_tests | |
# path: ${{ env.INSTALL_TEST_DIR }} | |
# | |
# - name: Extract OpenVINO packages | |
# run: | | |
# pushd ${{ env.INSTALL_DIR }} | |
# Expand-Archive openvino_package.zip -DestinationPath "${{ env.INSTALL_DIR }}" | |
# popd | |
# | |
# pushd ${{ env.INSTALL_TEST_DIR }} | |
# Expand-Archive openvino_tests.zip -DestinationPath "${{ env.INSTALL_TEST_DIR }}" | |
# popd | |
# | |
# - name: Check extraction | |
# run: | | |
# ls "${{ github.workspace }}" | |
# ls "${{ env.INSTALL_DIR }}" | |
# ls "${{ env.INSTALL_TEST_DIR }}" | |
# | |
# - name: Clone gtest_parallel | |
# run: | | |
# git clone https://github.com/google/gtest-parallel.git ${{ env.GTEST_PARALLEL_DIR }} | |
# | |
# - uses: actions/setup-python@v4 | |
# with: | |
# python-version: '3.11' | |
# | |
# - name: Intel GPU plugin func tests | |
# shell: cmd | |
# run: | | |
# call "${{ env.INSTALL_DIR }}\\setupvars.bat" && python3 ${{ env.GTEST_PARALLEL_DIR }}/gtest_parallel.py ${{ env.INSTALL_TEST_DIR }}/ov_gpu_func_tests --gtest_filter=*smoke* | |
# # ${{ env.INSTALL_TEST_DIR }}/ov_gpu_func_tests --gtest_print_time=1 --gtest_filter=*smoke* --gtest_output=xml:"${{ env.INSTALL_TEST_DIR }}/TEST-GPUFuncTests.xml" |