From 26978c52f744d5eafc2fedfc3e22ba4456ea44ce Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Tue, 17 Sep 2024 12:47:44 +0200 Subject: [PATCH 01/72] enabled all builds --- .github/workflows/linux_sanitizers.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/linux_sanitizers.yml b/.github/workflows/linux_sanitizers.yml index a56bbe54f82b53..91f3cda878b36b 100644 --- a/.github/workflows/linux_sanitizers.yml +++ b/.github/workflows/linux_sanitizers.yml @@ -39,8 +39,8 @@ jobs: SANITIZER_CMAKE_OPTION: '-DENABLE_SANITIZER=ON' - SANITIZER: 'UndefinedBehavior' SANITIZER_CMAKE_OPTION: '-DENABLE_UB_SANITIZER=ON' -# - SANITIZER: 'Thread' # Problems with protobuf -# SANITIZER_CMAKE_OPTION: '-DENABLE_THREAD_SANITIZER=ON' + - SANITIZER: 'Thread' # Problems with protobuf + SANITIZER_CMAKE_OPTION: '-DENABLE_THREAD_SANITIZER=ON' env: DEBIAN_FRONTEND: noninteractive # to prevent apt-get from waiting user input CMAKE_BUILD_TYPE: 'RelWithDebInfo' From 0c09b0658b77f9d05bc499d206bea1c2079c214b Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Tue, 17 Sep 2024 14:05:09 +0200 Subject: [PATCH 02/72] enable pull events --- .github/workflows/linux_sanitizers.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/linux_sanitizers.yml b/.github/workflows/linux_sanitizers.yml index 91f3cda878b36b..820c237c6c36a8 100644 --- a/.github/workflows/linux_sanitizers.yml +++ b/.github/workflows/linux_sanitizers.yml @@ -4,7 +4,10 @@ on: # run daily at 00:00 - cron: '0 0 * * *' workflow_dispatch: - # pull_request: + pull_request: + branches: + - master + - 'releases/**' concurrency: # github.ref is not unique in post-commit From 0c4c277d5bb2defd9fa765673c500526e3c3e093 Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Wed, 18 Sep 2024 14:03:08 +0200 Subject: [PATCH 03/72] added tsan suppresions.txt --- .github/workflows/linux_sanitizers.yml | 16 ++++++++++------ .../compile_flags/sanitizer.cmake | 4 ++-- tests/{ => sanitizers}/asan/ignore.txt | 0 tests/{ => sanitizers}/asan/suppressions.supp | 0 tests/{ => sanitizers}/lsan/suppressions.txt | 0 tests/sanitizers/tsan/suppressions.txt | 1 + 6 files changed, 13 insertions(+), 8 deletions(-) rename tests/{ => sanitizers}/asan/ignore.txt (100%) rename tests/{ => sanitizers}/asan/suppressions.supp (100%) rename tests/{ => sanitizers}/lsan/suppressions.txt (100%) create mode 100644 tests/sanitizers/tsan/suppressions.txt diff --git a/.github/workflows/linux_sanitizers.yml b/.github/workflows/linux_sanitizers.yml index 820c237c6c36a8..79271bfd256410 100644 --- a/.github/workflows/linux_sanitizers.yml +++ b/.github/workflows/linux_sanitizers.yml @@ -54,8 +54,9 @@ jobs: INSTALL_DIR: /__w/openvino/openvino/openvino_install INSTALL_TEST_DIR: /__w/openvino/openvino/tests_install BUILD_DIR: /__w/openvino/openvino/openvino_build - LSAN_IGNORE: /__w/openvino/openvino/openvino/tests/lsan/suppressions.txt - ASAN_IGNORE: /__w/openvino/openvino/openvino/tests/asan/suppressions.supp + LSAN_IGNORE: /__w/openvino/openvino/openvino/tests/sanitizers/lsan/suppressions.txt + ASAN_IGNORE: /__w/openvino/openvino/openvino/tests/sanitizers/asan/suppressions.supp + TSAN_IGNORE: /__w/openvino/openvino/openvino/tests/sanitizers/lsan/suppressions.txt CXX: clang++ CC: clang @@ -132,6 +133,7 @@ jobs: run: | export ASAN_OPTIONS=halt_on_error=0:suppressions=${ASAN_IGNORE} export LSAN_OPTIONS=suppressions=${LSAN_IGNORE}:NEOReadDebugKeys=1:DisableDeepBind=1 + export TSAN_OPTIONS=suppressions=${TSAN_IGNORE} export CC=clang export CXX=clang++ cmake \ @@ -160,6 +162,7 @@ jobs: run: | export ASAN_OPTIONS=halt_on_error=0:suppressions=${ASAN_IGNORE} export LSAN_OPTIONS=suppressions=${LSAN_IGNORE}:NEOReadDebugKeys=1:DisableDeepBind=1 + export TSAN_OPTIONS=suppressions=${TSAN_IGNORE} cmake --build ${BUILD_DIR} --parallel --config ${{ env.CMAKE_BUILD_TYPE }} - name: Cmake install - OpenVINO @@ -254,8 +257,8 @@ jobs: echo "INSTALL_DIR=$GITHUB_WORKSPACE/install" >> "$GITHUB_ENV" echo "INSTALL_TEST_DIR=$GITHUB_WORKSPACE/install/tests" >> "$GITHUB_ENV" - echo "ASAN_OPTIONS=halt_on_error=0:suppressions=$GITHUB_WORKSPACE/openvino/tests/asan/suppressions.supp" >> "$GITHUB_ENV" - echo "LSAN_OPTIONS=suppressions=$GITHUB_WORKSPACE/openvino/tests/lsan/suppressions.txt:NEOReadDebugKeys=1:DisableDeepBind=1" >> "$GITHUB_ENV" + echo "ASAN_OPTIONS=halt_on_error=0:suppressions=$GITHUB_WORKSPACE/openvino/tests/sanitizers/asan/suppressions.supp" >> "$GITHUB_ENV" + echo "LSAN_OPTIONS=suppressions=$GITHUB_WORKSPACE/openvino/tests/sanitizers/lsan/suppressions.txt:NEOReadDebugKeys=1:DisableDeepBind=1" >> "$GITHUB_ENV" - name: Extract OpenVINO packages run: | @@ -275,8 +278,9 @@ jobs: uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0 with: sparse-checkout: | - tests/lsan/suppressions.txt - tests/asan/suppressions.supp + tests/sanitizers/tsan/suppressions.txt + tests/sanitizers/lsan/suppressions.txt + tests/sanitizers/asan/suppressions.supp sparse-checkout-cone-mode: false path: 'openvino' diff --git a/cmake/developer_package/compile_flags/sanitizer.cmake b/cmake/developer_package/compile_flags/sanitizer.cmake index 73f109d726c88b..66da4495f2c3d3 100644 --- a/cmake/developer_package/compile_flags/sanitizer.cmake +++ b/cmake/developer_package/compile_flags/sanitizer.cmake @@ -17,7 +17,7 @@ if (ENABLE_SANITIZER) "https://github.com/openvinotoolkit/openvino/wiki/AddressSanitizer-and-LeakSanitizer") endif() elseif(OV_COMPILER_IS_CLANG) - set(SANITIZER_COMPILER_FLAGS "${SANITIZER_COMPILER_FLAGS} -fsanitize=address -fsanitize-blacklist=${OpenVINO_SOURCE_DIR}/tests/asan/ignore.txt") + set(SANITIZER_COMPILER_FLAGS "${SANITIZER_COMPILER_FLAGS} -fsanitize=address -fsanitize-blacklist=${OpenVINO_SOURCE_DIR}/tests/asan/sanitizers/ignore.txt") if(BUILD_SHARED_LIBS) set(SANITIZER_COMPILER_FLAGS "${SANITIZER_COMPILER_FLAGS} -shared-libasan") endif() @@ -27,7 +27,7 @@ if (ENABLE_SANITIZER) set(SANITIZER_COMPILER_FLAGS "${SANITIZER_COMPILER_FLAGS} -fsanitize-recover=address") endif() - set(SANITIZER_LINKER_FLAGS "${SANITIZER_LINKER_FLAGS} -fsanitize=address -fsanitize-blacklist=${OpenVINO_SOURCE_DIR}/tests/asan/ignore.txt") + set(SANITIZER_LINKER_FLAGS "${SANITIZER_LINKER_FLAGS} -fsanitize=address -fsanitize-blacklist=${OpenVINO_SOURCE_DIR}/tests/asan/sanitizers/ignore.txt") if(BUILD_SHARED_LIBS) set(SANITIZER_LINKER_FLAGS "${SANITIZER_LINKER_FLAGS} -shared-libasan") endif() diff --git a/tests/asan/ignore.txt b/tests/sanitizers/asan/ignore.txt similarity index 100% rename from tests/asan/ignore.txt rename to tests/sanitizers/asan/ignore.txt diff --git a/tests/asan/suppressions.supp b/tests/sanitizers/asan/suppressions.supp similarity index 100% rename from tests/asan/suppressions.supp rename to tests/sanitizers/asan/suppressions.supp diff --git a/tests/lsan/suppressions.txt b/tests/sanitizers/lsan/suppressions.txt similarity index 100% rename from tests/lsan/suppressions.txt rename to tests/sanitizers/lsan/suppressions.txt diff --git a/tests/sanitizers/tsan/suppressions.txt b/tests/sanitizers/tsan/suppressions.txt new file mode 100644 index 00000000000000..7ac08098c85ac0 --- /dev/null +++ b/tests/sanitizers/tsan/suppressions.txt @@ -0,0 +1 @@ +race:google::protobuf::* \ No newline at end of file From 28a35012e5b63cfb90ae0167b0b8d3ee88852c6f Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Wed, 18 Sep 2024 14:09:43 +0200 Subject: [PATCH 04/72] fixed path in the tests --- cmake/developer_package/compile_flags/sanitizer.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmake/developer_package/compile_flags/sanitizer.cmake b/cmake/developer_package/compile_flags/sanitizer.cmake index 66da4495f2c3d3..6c1956b84c18b5 100644 --- a/cmake/developer_package/compile_flags/sanitizer.cmake +++ b/cmake/developer_package/compile_flags/sanitizer.cmake @@ -17,7 +17,7 @@ if (ENABLE_SANITIZER) "https://github.com/openvinotoolkit/openvino/wiki/AddressSanitizer-and-LeakSanitizer") endif() elseif(OV_COMPILER_IS_CLANG) - set(SANITIZER_COMPILER_FLAGS "${SANITIZER_COMPILER_FLAGS} -fsanitize=address -fsanitize-blacklist=${OpenVINO_SOURCE_DIR}/tests/asan/sanitizers/ignore.txt") + set(SANITIZER_COMPILER_FLAGS "${SANITIZER_COMPILER_FLAGS} -fsanitize=address -fsanitize-blacklist=${OpenVINO_SOURCE_DIR}/tests/sanitizers/asan/ignore.txt") if(BUILD_SHARED_LIBS) set(SANITIZER_COMPILER_FLAGS "${SANITIZER_COMPILER_FLAGS} -shared-libasan") endif() @@ -27,7 +27,7 @@ if (ENABLE_SANITIZER) set(SANITIZER_COMPILER_FLAGS "${SANITIZER_COMPILER_FLAGS} -fsanitize-recover=address") endif() - set(SANITIZER_LINKER_FLAGS "${SANITIZER_LINKER_FLAGS} -fsanitize=address -fsanitize-blacklist=${OpenVINO_SOURCE_DIR}/tests/asan/sanitizers/ignore.txt") + set(SANITIZER_LINKER_FLAGS "${SANITIZER_LINKER_FLAGS} -fsanitize=address -fsanitize-blacklist=${OpenVINO_SOURCE_DIR}/tests/sanitizers/asan/ignore.txt") if(BUILD_SHARED_LIBS) set(SANITIZER_LINKER_FLAGS "${SANITIZER_LINKER_FLAGS} -shared-libasan") endif() From 0b3f26a02372ee09e8bad4eab7c677d70d34b225 Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Wed, 18 Sep 2024 14:26:17 +0200 Subject: [PATCH 05/72] added missed sting --- .github/workflows/linux_sanitizers.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/linux_sanitizers.yml b/.github/workflows/linux_sanitizers.yml index 79271bfd256410..d26c3e4e8baeca 100644 --- a/.github/workflows/linux_sanitizers.yml +++ b/.github/workflows/linux_sanitizers.yml @@ -56,7 +56,7 @@ jobs: BUILD_DIR: /__w/openvino/openvino/openvino_build LSAN_IGNORE: /__w/openvino/openvino/openvino/tests/sanitizers/lsan/suppressions.txt ASAN_IGNORE: /__w/openvino/openvino/openvino/tests/sanitizers/asan/suppressions.supp - TSAN_IGNORE: /__w/openvino/openvino/openvino/tests/sanitizers/lsan/suppressions.txt + TSAN_IGNORE: /__w/openvino/openvino/openvino/tests/sanitizers/tsan/suppressions.txt CXX: clang++ CC: clang @@ -259,6 +259,7 @@ jobs: echo "ASAN_OPTIONS=halt_on_error=0:suppressions=$GITHUB_WORKSPACE/openvino/tests/sanitizers/asan/suppressions.supp" >> "$GITHUB_ENV" echo "LSAN_OPTIONS=suppressions=$GITHUB_WORKSPACE/openvino/tests/sanitizers/lsan/suppressions.txt:NEOReadDebugKeys=1:DisableDeepBind=1" >> "$GITHUB_ENV" + echo "TSAN_OPTIONS=suppressions=$GITHUB_WORKSPACE/openvino/tests/sanitizers/tsan/suppressions.txt" >> "$GITHUB_ENV" - name: Extract OpenVINO packages run: | From 412d6613ca6dc4068485d40e301f243f316c0bdf Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Wed, 18 Sep 2024 14:37:34 +0200 Subject: [PATCH 06/72] update suppresions --- tests/sanitizers/tsan/suppressions.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/sanitizers/tsan/suppressions.txt b/tests/sanitizers/tsan/suppressions.txt index 7ac08098c85ac0..e69bdd1478f687 100644 --- a/tests/sanitizers/tsan/suppressions.txt +++ b/tests/sanitizers/tsan/suppressions.txt @@ -1 +1 @@ -race:google::protobuf::* \ No newline at end of file +race:thirdparty/protobuf \ No newline at end of file From 9f119607d0b5abadf0dbfcba0fa9af8ec7b87f75 Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Wed, 18 Sep 2024 15:02:19 +0200 Subject: [PATCH 07/72] set additional filters --- tests/sanitizers/tsan/suppressions.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/sanitizers/tsan/suppressions.txt b/tests/sanitizers/tsan/suppressions.txt index e69bdd1478f687..cb22876a20ae3e 100644 --- a/tests/sanitizers/tsan/suppressions.txt +++ b/tests/sanitizers/tsan/suppressions.txt @@ -1 +1,2 @@ -race:thirdparty/protobuf \ No newline at end of file +race:google::protobuf::* +race:protobuf::* \ No newline at end of file From 6c7e4354070f8501dbb6c7c2bbe5f40c995da0a2 Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Wed, 18 Sep 2024 15:21:01 +0200 Subject: [PATCH 08/72] ignore list --- cmake/developer_package/compile_flags/sanitizer.cmake | 4 ++-- tests/sanitizers/tsan/ignore.txt | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) create mode 100644 tests/sanitizers/tsan/ignore.txt diff --git a/cmake/developer_package/compile_flags/sanitizer.cmake b/cmake/developer_package/compile_flags/sanitizer.cmake index 6c1956b84c18b5..0ca39d0f5c5d9a 100644 --- a/cmake/developer_package/compile_flags/sanitizer.cmake +++ b/cmake/developer_package/compile_flags/sanitizer.cmake @@ -88,8 +88,8 @@ if(ENABLE_THREAD_SANITIZER) if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") message(FATAL_ERROR "Thread sanitizer is not supported in Windows with MSVC compiler. Please, use clang-cl or mingw") elseif(CMAKE_COMPILER_IS_GNUCXX OR OV_COMPILER_IS_CLANG) - set(SANITIZER_COMPILER_FLAGS "${SANITIZER_COMPILER_FLAGS} -fsanitize=thread") - set(SANITIZER_LINKER_FLAGS "${SANITIZER_LINKER_FLAGS} -fsanitize=thread") + set(SANITIZER_COMPILER_FLAGS "${SANITIZER_COMPILER_FLAGS} -fsanitize=thread -fsanitize-blacklist=${OpenVINO_SOURCE_DIR}/tests/sanitizers/tsan/ignore.txt") + set(SANITIZER_LINKER_FLAGS "${SANITIZER_LINKER_FLAGS} -fsanitize=thread -fsanitize-blacklist=${OpenVINO_SOURCE_DIR}/tests/sanitizers/tsan/ignore.txt") else() message(WARNING "Unsupported CXX compiler ${CMAKE_CXX_COMPILER_ID}") endif() diff --git a/tests/sanitizers/tsan/ignore.txt b/tests/sanitizers/tsan/ignore.txt new file mode 100644 index 00000000000000..970a3955092668 --- /dev/null +++ b/tests/sanitizers/tsan/ignore.txt @@ -0,0 +1,2 @@ +src:*/thirdparty/* +src:thirdparty/* \ No newline at end of file From f2a964df3b025aa1e5d10f7b4c255c4841d8128b Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Wed, 18 Sep 2024 18:53:05 +0200 Subject: [PATCH 09/72] ignore onnx too --- tests/sanitizers/tsan/suppressions.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/sanitizers/tsan/suppressions.txt b/tests/sanitizers/tsan/suppressions.txt index cb22876a20ae3e..dd34e38efdd563 100644 --- a/tests/sanitizers/tsan/suppressions.txt +++ b/tests/sanitizers/tsan/suppressions.txt @@ -1,2 +1,3 @@ race:google::protobuf::* -race:protobuf::* \ No newline at end of file +race:protobuf::* +race:onnx::* \ No newline at end of file From 416fef43ecc76129bd8cbcec96438f0ec28b332f Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Thu, 19 Sep 2024 22:46:40 +0200 Subject: [PATCH 10/72] exlude tsan from configure --- .github/workflows/linux_sanitizers.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/linux_sanitizers.yml b/.github/workflows/linux_sanitizers.yml index d26c3e4e8baeca..39b463c601ed99 100644 --- a/.github/workflows/linux_sanitizers.yml +++ b/.github/workflows/linux_sanitizers.yml @@ -133,7 +133,6 @@ jobs: run: | export ASAN_OPTIONS=halt_on_error=0:suppressions=${ASAN_IGNORE} export LSAN_OPTIONS=suppressions=${LSAN_IGNORE}:NEOReadDebugKeys=1:DisableDeepBind=1 - export TSAN_OPTIONS=suppressions=${TSAN_IGNORE} export CC=clang export CXX=clang++ cmake \ From 3f1100ac7ed265c42ac8e9ebd4dc9263ade34f3b Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Fri, 20 Sep 2024 14:44:37 +0200 Subject: [PATCH 11/72] gcc --- .github/workflows/linux_sanitizers.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/linux_sanitizers.yml b/.github/workflows/linux_sanitizers.yml index 39b463c601ed99..48d0513db0907d 100644 --- a/.github/workflows/linux_sanitizers.yml +++ b/.github/workflows/linux_sanitizers.yml @@ -57,8 +57,8 @@ jobs: LSAN_IGNORE: /__w/openvino/openvino/openvino/tests/sanitizers/lsan/suppressions.txt ASAN_IGNORE: /__w/openvino/openvino/openvino/tests/sanitizers/asan/suppressions.supp TSAN_IGNORE: /__w/openvino/openvino/openvino/tests/sanitizers/tsan/suppressions.txt - CXX: clang++ - CC: clang + CXX: g++ + CC: gcc steps: - name: Set apt retries @@ -231,8 +231,8 @@ jobs: INSTALL_DIR: ${{ github.workspace }}/install INSTALL_TEST_DIR: ${{ github.workspace }}/install/tests TBB_ENABLE_SANITIZERS: 1 - CC: clang - CXX: clang++ + CC: gcc + CXX: g++ steps: - name: Set apt retries run: echo 'Acquire::Retries "10";' > /etc/apt/apt.conf.d/80-retries From a399af4f9ffd037f2f9638190164d398ecf1e53c Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Fri, 20 Sep 2024 14:55:45 +0200 Subject: [PATCH 12/72] removed clang --- .github/workflows/linux_sanitizers.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/linux_sanitizers.yml b/.github/workflows/linux_sanitizers.yml index 48d0513db0907d..063c251ff06575 100644 --- a/.github/workflows/linux_sanitizers.yml +++ b/.github/workflows/linux_sanitizers.yml @@ -133,8 +133,8 @@ jobs: run: | export ASAN_OPTIONS=halt_on_error=0:suppressions=${ASAN_IGNORE} export LSAN_OPTIONS=suppressions=${LSAN_IGNORE}:NEOReadDebugKeys=1:DisableDeepBind=1 - export CC=clang - export CXX=clang++ + export CC=gcc + export CXX=g++ cmake \ -G "${{ env.CMAKE_GENERATOR }}" \ -DENABLE_CPPLINT=OFF \ From 0eccfff48ea145698bc27f20825628b21eb5b924 Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Fri, 20 Sep 2024 15:35:58 +0200 Subject: [PATCH 13/72] disable blacklist --- cmake/developer_package/compile_flags/sanitizer.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmake/developer_package/compile_flags/sanitizer.cmake b/cmake/developer_package/compile_flags/sanitizer.cmake index 0ca39d0f5c5d9a..6c1956b84c18b5 100644 --- a/cmake/developer_package/compile_flags/sanitizer.cmake +++ b/cmake/developer_package/compile_flags/sanitizer.cmake @@ -88,8 +88,8 @@ if(ENABLE_THREAD_SANITIZER) if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") message(FATAL_ERROR "Thread sanitizer is not supported in Windows with MSVC compiler. Please, use clang-cl or mingw") elseif(CMAKE_COMPILER_IS_GNUCXX OR OV_COMPILER_IS_CLANG) - set(SANITIZER_COMPILER_FLAGS "${SANITIZER_COMPILER_FLAGS} -fsanitize=thread -fsanitize-blacklist=${OpenVINO_SOURCE_DIR}/tests/sanitizers/tsan/ignore.txt") - set(SANITIZER_LINKER_FLAGS "${SANITIZER_LINKER_FLAGS} -fsanitize=thread -fsanitize-blacklist=${OpenVINO_SOURCE_DIR}/tests/sanitizers/tsan/ignore.txt") + set(SANITIZER_COMPILER_FLAGS "${SANITIZER_COMPILER_FLAGS} -fsanitize=thread") + set(SANITIZER_LINKER_FLAGS "${SANITIZER_LINKER_FLAGS} -fsanitize=thread") else() message(WARNING "Unsupported CXX compiler ${CMAKE_CXX_COMPILER_ID}") endif() From 0897a1b2fadaba1297f0e0a09c77afe5e4030b0e Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Mon, 23 Sep 2024 12:39:30 +0200 Subject: [PATCH 14/72] Revert "disable blacklist" This reverts commit 876023aba06f6661396a5205e19e84769966b95a. --- cmake/developer_package/compile_flags/sanitizer.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmake/developer_package/compile_flags/sanitizer.cmake b/cmake/developer_package/compile_flags/sanitizer.cmake index 6c1956b84c18b5..0ca39d0f5c5d9a 100644 --- a/cmake/developer_package/compile_flags/sanitizer.cmake +++ b/cmake/developer_package/compile_flags/sanitizer.cmake @@ -88,8 +88,8 @@ if(ENABLE_THREAD_SANITIZER) if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") message(FATAL_ERROR "Thread sanitizer is not supported in Windows with MSVC compiler. Please, use clang-cl or mingw") elseif(CMAKE_COMPILER_IS_GNUCXX OR OV_COMPILER_IS_CLANG) - set(SANITIZER_COMPILER_FLAGS "${SANITIZER_COMPILER_FLAGS} -fsanitize=thread") - set(SANITIZER_LINKER_FLAGS "${SANITIZER_LINKER_FLAGS} -fsanitize=thread") + set(SANITIZER_COMPILER_FLAGS "${SANITIZER_COMPILER_FLAGS} -fsanitize=thread -fsanitize-blacklist=${OpenVINO_SOURCE_DIR}/tests/sanitizers/tsan/ignore.txt") + set(SANITIZER_LINKER_FLAGS "${SANITIZER_LINKER_FLAGS} -fsanitize=thread -fsanitize-blacklist=${OpenVINO_SOURCE_DIR}/tests/sanitizers/tsan/ignore.txt") else() message(WARNING "Unsupported CXX compiler ${CMAKE_CXX_COMPILER_ID}") endif() From d97eab039eff96705548c9e6e164c500a1698278 Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Mon, 23 Sep 2024 12:39:38 +0200 Subject: [PATCH 15/72] Revert "removed clang" This reverts commit 3b074fe72372cb1a189bda38d0da0f2dcb09b156. --- .github/workflows/linux_sanitizers.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/linux_sanitizers.yml b/.github/workflows/linux_sanitizers.yml index 063c251ff06575..48d0513db0907d 100644 --- a/.github/workflows/linux_sanitizers.yml +++ b/.github/workflows/linux_sanitizers.yml @@ -133,8 +133,8 @@ jobs: run: | export ASAN_OPTIONS=halt_on_error=0:suppressions=${ASAN_IGNORE} export LSAN_OPTIONS=suppressions=${LSAN_IGNORE}:NEOReadDebugKeys=1:DisableDeepBind=1 - export CC=gcc - export CXX=g++ + export CC=clang + export CXX=clang++ cmake \ -G "${{ env.CMAKE_GENERATOR }}" \ -DENABLE_CPPLINT=OFF \ From 0d07a861726eedb458c495bd98eb974305934871 Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Mon, 23 Sep 2024 12:39:41 +0200 Subject: [PATCH 16/72] Revert "gcc" This reverts commit 2fac464f9d2cea55f7d43bb519b829339fc910f5. --- .github/workflows/linux_sanitizers.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/linux_sanitizers.yml b/.github/workflows/linux_sanitizers.yml index 48d0513db0907d..39b463c601ed99 100644 --- a/.github/workflows/linux_sanitizers.yml +++ b/.github/workflows/linux_sanitizers.yml @@ -57,8 +57,8 @@ jobs: LSAN_IGNORE: /__w/openvino/openvino/openvino/tests/sanitizers/lsan/suppressions.txt ASAN_IGNORE: /__w/openvino/openvino/openvino/tests/sanitizers/asan/suppressions.supp TSAN_IGNORE: /__w/openvino/openvino/openvino/tests/sanitizers/tsan/suppressions.txt - CXX: g++ - CC: gcc + CXX: clang++ + CC: clang steps: - name: Set apt retries @@ -231,8 +231,8 @@ jobs: INSTALL_DIR: ${{ github.workspace }}/install INSTALL_TEST_DIR: ${{ github.workspace }}/install/tests TBB_ENABLE_SANITIZERS: 1 - CC: gcc - CXX: g++ + CC: clang + CXX: clang++ steps: - name: Set apt retries run: echo 'Acquire::Retries "10";' > /etc/apt/apt.conf.d/80-retries From 036fb3a21edf9e52a5a563f8029cb8296109d8a0 Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Mon, 23 Sep 2024 12:41:00 +0200 Subject: [PATCH 17/72] disable onnx fe --- .github/workflows/linux_sanitizers.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/linux_sanitizers.yml b/.github/workflows/linux_sanitizers.yml index 39b463c601ed99..0293327782a1cd 100644 --- a/.github/workflows/linux_sanitizers.yml +++ b/.github/workflows/linux_sanitizers.yml @@ -150,7 +150,7 @@ jobs: -DENABLE_OV_PADDLE_FRONTEND=ON \ -DENABLE_OV_PYTORCH_FRONTEND=ON \ -DENABLE_OV_JAX_FRONTEND=ON \ - -DENABLE_OV_ONNX_FRONTEND=ON \ + -DENABLE_OV_ONNX_FRONTEND=OFF \ -DENABLE_ONEDNN_FOR_GPU=OFF \ -DCMAKE_COMPILE_WARNING_AS_ERROR=OFF \ ${{ matrix.SANITIZER_CMAKE_OPTION }} \ From fa02e6355c11c3a74f5029a668c509b06e619fbc Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Mon, 23 Sep 2024 12:48:27 +0200 Subject: [PATCH 18/72] ENABLE_SYSTEM_PROTOBUF=ON --- .github/workflows/linux_sanitizers.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/linux_sanitizers.yml b/.github/workflows/linux_sanitizers.yml index 0293327782a1cd..4fc7457c2b3fa2 100644 --- a/.github/workflows/linux_sanitizers.yml +++ b/.github/workflows/linux_sanitizers.yml @@ -152,6 +152,7 @@ jobs: -DENABLE_OV_JAX_FRONTEND=ON \ -DENABLE_OV_ONNX_FRONTEND=OFF \ -DENABLE_ONEDNN_FOR_GPU=OFF \ + -DENABLE_SYSTEM_PROTOBUF=ON \ -DCMAKE_COMPILE_WARNING_AS_ERROR=OFF \ ${{ matrix.SANITIZER_CMAKE_OPTION }} \ -S ${OPENVINO_REPO} \ From 1a52e599f21f05eb23cd188ecdbdf41946290f7c Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Mon, 23 Sep 2024 20:33:24 +0200 Subject: [PATCH 19/72] fixed protobuf builds --- .../developer_package/compile_flags/sanitizer.cmake | 4 ++-- thirdparty/dependencies.cmake | 12 +++++++++++- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/cmake/developer_package/compile_flags/sanitizer.cmake b/cmake/developer_package/compile_flags/sanitizer.cmake index 0ca39d0f5c5d9a..6c1956b84c18b5 100644 --- a/cmake/developer_package/compile_flags/sanitizer.cmake +++ b/cmake/developer_package/compile_flags/sanitizer.cmake @@ -88,8 +88,8 @@ if(ENABLE_THREAD_SANITIZER) if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") message(FATAL_ERROR "Thread sanitizer is not supported in Windows with MSVC compiler. Please, use clang-cl or mingw") elseif(CMAKE_COMPILER_IS_GNUCXX OR OV_COMPILER_IS_CLANG) - set(SANITIZER_COMPILER_FLAGS "${SANITIZER_COMPILER_FLAGS} -fsanitize=thread -fsanitize-blacklist=${OpenVINO_SOURCE_DIR}/tests/sanitizers/tsan/ignore.txt") - set(SANITIZER_LINKER_FLAGS "${SANITIZER_LINKER_FLAGS} -fsanitize=thread -fsanitize-blacklist=${OpenVINO_SOURCE_DIR}/tests/sanitizers/tsan/ignore.txt") + set(SANITIZER_COMPILER_FLAGS "${SANITIZER_COMPILER_FLAGS} -fsanitize=thread") + set(SANITIZER_LINKER_FLAGS "${SANITIZER_LINKER_FLAGS} -fsanitize=thread") else() message(WARNING "Unsupported CXX compiler ${CMAKE_CXX_COMPILER_ID}") endif() diff --git a/thirdparty/dependencies.cmake b/thirdparty/dependencies.cmake index 8313ca73178283..3455567b801a13 100644 --- a/thirdparty/dependencies.cmake +++ b/thirdparty/dependencies.cmake @@ -357,6 +357,12 @@ if(ENABLE_OV_PADDLE_FRONTEND OR ENABLE_OV_ONNX_FRONTEND OR ENABLE_OV_TF_FRONTEND endif() else() add_subdirectory(thirdparty/protobuf EXCLUDE_FROM_ALL) + + foreach(proto_target protoc libprotobuf) + target_compile_options(${proto_target} PUBLIC -fno-sanitize=thread) + target_link_options(${proto_target} PUBLIC -fno-sanitize=thread) + endforeach() + endif() # forward additional variables used in the other places @@ -497,7 +503,11 @@ if(ENABLE_OV_ONNX_FRONTEND) # conan and vcpkg create imported targets 'onnx' and 'onnx_proto' else() add_subdirectory(thirdparty/onnx) - endif() + foreach(onnx_target onnx onnx_proto) + target_compile_options(${onnx_target} PUBLIC -fno-sanitize=thread) + target_link_options(${onnx_target} PUBLIC -fno-sanitize=thread) + endforeach() + endif() endif() # From 1dad8fe7872f620a56f71c3ea3ee778b379fe7a9 Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Mon, 23 Sep 2024 21:52:29 +0200 Subject: [PATCH 20/72] Revert "disable onnx fe" This reverts commit 9eae0af35d6d586aeddd6c341a93eccaaf2f9563. --- .github/workflows/linux_sanitizers.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/linux_sanitizers.yml b/.github/workflows/linux_sanitizers.yml index 4fc7457c2b3fa2..85e92f34fe6a4c 100644 --- a/.github/workflows/linux_sanitizers.yml +++ b/.github/workflows/linux_sanitizers.yml @@ -150,7 +150,7 @@ jobs: -DENABLE_OV_PADDLE_FRONTEND=ON \ -DENABLE_OV_PYTORCH_FRONTEND=ON \ -DENABLE_OV_JAX_FRONTEND=ON \ - -DENABLE_OV_ONNX_FRONTEND=OFF \ + -DENABLE_OV_ONNX_FRONTEND=ON \ -DENABLE_ONEDNN_FOR_GPU=OFF \ -DENABLE_SYSTEM_PROTOBUF=ON \ -DCMAKE_COMPILE_WARNING_AS_ERROR=OFF \ From a8d8ce208c716336cff13472d39602e7d03fe9d6 Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Mon, 23 Sep 2024 21:52:47 +0200 Subject: [PATCH 21/72] Revert "ENABLE_SYSTEM_PROTOBUF=ON" This reverts commit d05eaf70e217cd0e07a79ac8c5a419250cd67bfd. --- .github/workflows/linux_sanitizers.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/linux_sanitizers.yml b/.github/workflows/linux_sanitizers.yml index 85e92f34fe6a4c..39b463c601ed99 100644 --- a/.github/workflows/linux_sanitizers.yml +++ b/.github/workflows/linux_sanitizers.yml @@ -152,7 +152,6 @@ jobs: -DENABLE_OV_JAX_FRONTEND=ON \ -DENABLE_OV_ONNX_FRONTEND=ON \ -DENABLE_ONEDNN_FOR_GPU=OFF \ - -DENABLE_SYSTEM_PROTOBUF=ON \ -DCMAKE_COMPILE_WARNING_AS_ERROR=OFF \ ${{ matrix.SANITIZER_CMAKE_OPTION }} \ -S ${OPENVINO_REPO} \ From 7b39c020c7b7f41720a60a6dff2e33d473f14ef9 Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Tue, 24 Sep 2024 08:18:08 +0200 Subject: [PATCH 22/72] add condition for ENABLE_THREAD_SANITIZER --- thirdparty/dependencies.cmake | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/thirdparty/dependencies.cmake b/thirdparty/dependencies.cmake index 3455567b801a13..9102b103f47fec 100644 --- a/thirdparty/dependencies.cmake +++ b/thirdparty/dependencies.cmake @@ -357,12 +357,12 @@ if(ENABLE_OV_PADDLE_FRONTEND OR ENABLE_OV_ONNX_FRONTEND OR ENABLE_OV_TF_FRONTEND endif() else() add_subdirectory(thirdparty/protobuf EXCLUDE_FROM_ALL) - - foreach(proto_target protoc libprotobuf) - target_compile_options(${proto_target} PUBLIC -fno-sanitize=thread) - target_link_options(${proto_target} PUBLIC -fno-sanitize=thread) - endforeach() - + if(ENABLE_THREAD_SANITIZER) + foreach(proto_target protoc libprotobuf) + target_compile_options(${proto_target} PUBLIC -fno-sanitize=thread) + target_link_options(${proto_target} PUBLIC -fno-sanitize=thread) + endforeach() + endif() endif() # forward additional variables used in the other places @@ -503,10 +503,12 @@ if(ENABLE_OV_ONNX_FRONTEND) # conan and vcpkg create imported targets 'onnx' and 'onnx_proto' else() add_subdirectory(thirdparty/onnx) - foreach(onnx_target onnx onnx_proto) - target_compile_options(${onnx_target} PUBLIC -fno-sanitize=thread) - target_link_options(${onnx_target} PUBLIC -fno-sanitize=thread) - endforeach() + if(ENABLE_THREAD_SANITIZER) + foreach(onnx_target onnx onnx_proto) + target_compile_options(${onnx_target} PUBLIC -fno-sanitize=thread) + target_link_options(${onnx_target} PUBLIC -fno-sanitize=thread) + endforeach() + endif() endif() endif() From fc82d2a3065fcac35c41b6805bfae7c846d3517b Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Tue, 24 Sep 2024 10:44:44 +0200 Subject: [PATCH 23/72] final variant --- thirdparty/dependencies.cmake | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/thirdparty/dependencies.cmake b/thirdparty/dependencies.cmake index 9102b103f47fec..627ec2e3af9282 100644 --- a/thirdparty/dependencies.cmake +++ b/thirdparty/dependencies.cmake @@ -357,10 +357,13 @@ if(ENABLE_OV_PADDLE_FRONTEND OR ENABLE_OV_ONNX_FRONTEND OR ENABLE_OV_TF_FRONTEND endif() else() add_subdirectory(thirdparty/protobuf EXCLUDE_FROM_ALL) + # protobuf fails to build with -fsanitize=thread if(ENABLE_THREAD_SANITIZER) - foreach(proto_target protoc libprotobuf) - target_compile_options(${proto_target} PUBLIC -fno-sanitize=thread) - target_link_options(${proto_target} PUBLIC -fno-sanitize=thread) + foreach(proto_target protoc libprotobuf libprotobuf-lite) + if(TARGET ${proto_target}) + target_compile_options(${proto_target} PUBLIC -fno-sanitize=thread) + target_link_options(${proto_target} PUBLIC -fno-sanitize=thread) + endif() endforeach() endif() endif() @@ -503,12 +506,6 @@ if(ENABLE_OV_ONNX_FRONTEND) # conan and vcpkg create imported targets 'onnx' and 'onnx_proto' else() add_subdirectory(thirdparty/onnx) - if(ENABLE_THREAD_SANITIZER) - foreach(onnx_target onnx onnx_proto) - target_compile_options(${onnx_target} PUBLIC -fno-sanitize=thread) - target_link_options(${onnx_target} PUBLIC -fno-sanitize=thread) - endforeach() - endif() endif() endif() From 44c65478db60016b449352098c4b227fec95cc80 Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Tue, 24 Sep 2024 10:46:49 +0200 Subject: [PATCH 24/72] revert tsan blacklist --- .github/workflows/linux_sanitizers.yml | 4 ---- tests/sanitizers/tsan/ignore.txt | 2 -- tests/sanitizers/tsan/suppressions.txt | 3 --- 3 files changed, 9 deletions(-) delete mode 100644 tests/sanitizers/tsan/ignore.txt delete mode 100644 tests/sanitizers/tsan/suppressions.txt diff --git a/.github/workflows/linux_sanitizers.yml b/.github/workflows/linux_sanitizers.yml index 39b463c601ed99..1d071bb94a8032 100644 --- a/.github/workflows/linux_sanitizers.yml +++ b/.github/workflows/linux_sanitizers.yml @@ -56,7 +56,6 @@ jobs: BUILD_DIR: /__w/openvino/openvino/openvino_build LSAN_IGNORE: /__w/openvino/openvino/openvino/tests/sanitizers/lsan/suppressions.txt ASAN_IGNORE: /__w/openvino/openvino/openvino/tests/sanitizers/asan/suppressions.supp - TSAN_IGNORE: /__w/openvino/openvino/openvino/tests/sanitizers/tsan/suppressions.txt CXX: clang++ CC: clang @@ -161,7 +160,6 @@ jobs: run: | export ASAN_OPTIONS=halt_on_error=0:suppressions=${ASAN_IGNORE} export LSAN_OPTIONS=suppressions=${LSAN_IGNORE}:NEOReadDebugKeys=1:DisableDeepBind=1 - export TSAN_OPTIONS=suppressions=${TSAN_IGNORE} cmake --build ${BUILD_DIR} --parallel --config ${{ env.CMAKE_BUILD_TYPE }} - name: Cmake install - OpenVINO @@ -258,7 +256,6 @@ jobs: echo "ASAN_OPTIONS=halt_on_error=0:suppressions=$GITHUB_WORKSPACE/openvino/tests/sanitizers/asan/suppressions.supp" >> "$GITHUB_ENV" echo "LSAN_OPTIONS=suppressions=$GITHUB_WORKSPACE/openvino/tests/sanitizers/lsan/suppressions.txt:NEOReadDebugKeys=1:DisableDeepBind=1" >> "$GITHUB_ENV" - echo "TSAN_OPTIONS=suppressions=$GITHUB_WORKSPACE/openvino/tests/sanitizers/tsan/suppressions.txt" >> "$GITHUB_ENV" - name: Extract OpenVINO packages run: | @@ -278,7 +275,6 @@ jobs: uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0 with: sparse-checkout: | - tests/sanitizers/tsan/suppressions.txt tests/sanitizers/lsan/suppressions.txt tests/sanitizers/asan/suppressions.supp sparse-checkout-cone-mode: false diff --git a/tests/sanitizers/tsan/ignore.txt b/tests/sanitizers/tsan/ignore.txt deleted file mode 100644 index 970a3955092668..00000000000000 --- a/tests/sanitizers/tsan/ignore.txt +++ /dev/null @@ -1,2 +0,0 @@ -src:*/thirdparty/* -src:thirdparty/* \ No newline at end of file diff --git a/tests/sanitizers/tsan/suppressions.txt b/tests/sanitizers/tsan/suppressions.txt deleted file mode 100644 index dd34e38efdd563..00000000000000 --- a/tests/sanitizers/tsan/suppressions.txt +++ /dev/null @@ -1,3 +0,0 @@ -race:google::protobuf::* -race:protobuf::* -race:onnx::* \ No newline at end of file From a217ee7c10fd64050b3d06c17b10d8988c53097a Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Tue, 24 Sep 2024 11:59:26 +0200 Subject: [PATCH 25/72] rerfactoring --- .github/workflows/linux_sanitizers.yml | 45 ++++---------------------- 1 file changed, 6 insertions(+), 39 deletions(-) diff --git a/.github/workflows/linux_sanitizers.yml b/.github/workflows/linux_sanitizers.yml index 1d071bb94a8032..7a886132f75009 100644 --- a/.github/workflows/linux_sanitizers.yml +++ b/.github/workflows/linux_sanitizers.yml @@ -1,4 +1,4 @@ -name: Linux Sanitizers (Ubuntu 20.04, Python 3.11) +name: Linux Sanitizers (Ubuntu 20.04) on: schedule: # run daily at 00:00 @@ -17,8 +17,6 @@ concurrency: permissions: read-all env: - PIP_CACHE_PATH: /mount/caches/pip/linux - PYTHON_VERSION: '3.11' TARGET_BRANCH: ${{ github.base_ref || github.event.merge_group.base_ref || github.ref }} jobs: @@ -42,7 +40,7 @@ jobs: SANITIZER_CMAKE_OPTION: '-DENABLE_SANITIZER=ON' - SANITIZER: 'UndefinedBehavior' SANITIZER_CMAKE_OPTION: '-DENABLE_UB_SANITIZER=ON' - - SANITIZER: 'Thread' # Problems with protobuf + - SANITIZER: 'Thread' SANITIZER_CMAKE_OPTION: '-DENABLE_THREAD_SANITIZER=ON' env: DEBIAN_FRONTEND: noninteractive # to prevent apt-get from waiting user input @@ -98,42 +96,12 @@ jobs: bash ${OPENVINO_REPO}/install_build_dependencies.sh apt --assume-yes install clang lld - - name: Setup Python ${{ env.PYTHON_VERSION }} - uses: ./openvino/.github/actions/setup_python - with: - version: ${{ env.PYTHON_VERSION }} - pip-cache-path: ${{ env.PIP_CACHE_PATH }} - should-setup-pip-paths: 'true' - self-hosted-runner: 'true' - show-cache-info: 'true' - - - name: Install python dependencies - run: | - # For Python API: build and wheel packaging - python3 -m pip install -r ${OPENVINO_REPO}/src/bindings/python/wheel/requirements-dev.txt - - # For running ONNX frontend unit tests - python3 -m pip install --force-reinstall -r ${OPENVINO_REPO}/src/frontends/onnx/tests/requirements.txt - - # For running TensorFlow frontend unit tests - python3 -m pip install -r ${OPENVINO_REPO}/src/frontends/tensorflow/tests/requirements.txt - - # For running TensorFlow Lite frontend unit tests - python3 -m pip install -r ${OPENVINO_REPO}/src/frontends/tensorflow_lite/tests/requirements.txt - - # For running Paddle frontend unit tests - python3 -m pip install -r ${OPENVINO_REPO}/src/frontends/paddle/tests/requirements.txt - # # Build # - name: CMake configure - OpenVINO run: | - export ASAN_OPTIONS=halt_on_error=0:suppressions=${ASAN_IGNORE} - export LSAN_OPTIONS=suppressions=${LSAN_IGNORE}:NEOReadDebugKeys=1:DisableDeepBind=1 - export CC=clang - export CXX=clang++ cmake \ -G "${{ env.CMAKE_GENERATOR }}" \ -DENABLE_CPPLINT=OFF \ @@ -158,9 +126,10 @@ jobs: - name: Cmake build - OpenVINO run: | - export ASAN_OPTIONS=halt_on_error=0:suppressions=${ASAN_IGNORE} - export LSAN_OPTIONS=suppressions=${LSAN_IGNORE}:NEOReadDebugKeys=1:DisableDeepBind=1 cmake --build ${BUILD_DIR} --parallel --config ${{ env.CMAKE_BUILD_TYPE }} + env: + ASAN_OPTIONS: halt_on_error=0:suppressions=${ASAN_IGNORE} + LSAN_OPTIONS: suppressions=${LSAN_IGNORE}:NEOReadDebugKeys=1:DisableDeepBind=1 - name: Cmake install - OpenVINO run: | @@ -223,14 +192,12 @@ jobs: include: - SANITIZER: 'AddressAndLeak' - SANITIZER: 'UndefinedBehavior' -# - SANITIZER: 'Thread' # Problems with protobuf at the Build stage + - SANITIZER: 'Thread' env: DEBIAN_FRONTEND: noninteractive # to prevent apt-get from waiting user input INSTALL_DIR: ${{ github.workspace }}/install INSTALL_TEST_DIR: ${{ github.workspace }}/install/tests TBB_ENABLE_SANITIZERS: 1 - CC: clang - CXX: clang++ steps: - name: Set apt retries run: echo 'Acquire::Retries "10";' > /etc/apt/apt.conf.d/80-retries From 809b42dacdd5e62a8e1fe937ac78b5536db2b26c Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Tue, 24 Sep 2024 10:55:49 +0200 Subject: [PATCH 26/72] Apply suggestions from code review --- thirdparty/dependencies.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/thirdparty/dependencies.cmake b/thirdparty/dependencies.cmake index 627ec2e3af9282..976e92e8634c39 100644 --- a/thirdparty/dependencies.cmake +++ b/thirdparty/dependencies.cmake @@ -506,7 +506,7 @@ if(ENABLE_OV_ONNX_FRONTEND) # conan and vcpkg create imported targets 'onnx' and 'onnx_proto' else() add_subdirectory(thirdparty/onnx) - endif() + endif() endif() # From 02b466e92d7cdcb0388a265650388d2c532ff751 Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Tue, 24 Sep 2024 12:17:07 +0200 Subject: [PATCH 27/72] fixed envs --- .github/workflows/linux_sanitizers.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/linux_sanitizers.yml b/.github/workflows/linux_sanitizers.yml index 7a886132f75009..d5f7359a987b6a 100644 --- a/.github/workflows/linux_sanitizers.yml +++ b/.github/workflows/linux_sanitizers.yml @@ -128,8 +128,8 @@ jobs: run: | cmake --build ${BUILD_DIR} --parallel --config ${{ env.CMAKE_BUILD_TYPE }} env: - ASAN_OPTIONS: halt_on_error=0:suppressions=${ASAN_IGNORE} - LSAN_OPTIONS: suppressions=${LSAN_IGNORE}:NEOReadDebugKeys=1:DisableDeepBind=1 + ASAN_OPTIONS: halt_on_error=0:suppressions=${{ env.ASAN_IGNORE }} + LSAN_OPTIONS: suppressions=${{ env.LSAN_IGNORE }}:NEOReadDebugKeys=1:DisableDeepBind=1 - name: Cmake install - OpenVINO run: | @@ -262,7 +262,6 @@ jobs: if: always() run: | source ${INSTALL_DIR}/setupvars.sh - ${INSTALL_TEST_DIR}/ov_inference_functional_tests --gtest_print_time=1 \ --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-InferenceFunc.xml From 0f5fe7ead3fec327154efd7d7694734d12b771e2 Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Tue, 24 Sep 2024 12:51:08 +0200 Subject: [PATCH 28/72] use docker images --- .github/dockerfiles/docker_tag | 2 +- .../ov_build/ubuntu_20_04_x64/Dockerfile | 2 + .github/workflows/linux_sanitizers.yml | 80 ++++++++++++------- 3 files changed, 54 insertions(+), 30 deletions(-) diff --git a/.github/dockerfiles/docker_tag b/.github/dockerfiles/docker_tag index 413b6019a3b0be..5d0bfbf0157ba3 100644 --- a/.github/dockerfiles/docker_tag +++ b/.github/dockerfiles/docker_tag @@ -1 +1 @@ -pr-26586 \ No newline at end of file +pr-26634 \ No newline at end of file diff --git a/.github/dockerfiles/ov_build/ubuntu_20_04_x64/Dockerfile b/.github/dockerfiles/ov_build/ubuntu_20_04_x64/Dockerfile index 886ed15a634974..d5d14f96e137ab 100644 --- a/.github/dockerfiles/ov_build/ubuntu_20_04_x64/Dockerfile +++ b/.github/dockerfiles/ov_build/ubuntu_20_04_x64/Dockerfile @@ -31,6 +31,8 @@ RUN apt-get update && \ # Compiler \ gcc-10 \ g++-10 \ + clang \ + lld \ && \ rm -rf /var/lib/apt/lists/* diff --git a/.github/workflows/linux_sanitizers.yml b/.github/workflows/linux_sanitizers.yml index d5f7359a987b6a..6fa329b9625702 100644 --- a/.github/workflows/linux_sanitizers.yml +++ b/.github/workflows/linux_sanitizers.yml @@ -20,7 +20,55 @@ env: TARGET_BRANCH: ${{ github.base_ref || github.event.merge_group.base_ref || github.ref }} jobs: + Smart_CI: + runs-on: ubuntu-latest + outputs: + affected_components: "${{ steps.smart_ci.outputs.affected_components }}" + changed_components: "${{ steps.smart_ci.outputs.changed_components }}" + skip_workflow: "${{ steps.smart_ci.outputs.skip_workflow }}" + steps: + - name: checkout action + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + with: + sparse-checkout: .github/actions/smart-ci + + - name: Get affected components + id: smart_ci + uses: ./.github/actions/smart-ci + with: + repository: ${{ github.repository }} + pr: ${{ github.event.number }} + commit_sha: ${{ github.sha }} + ref_name: ${{ github.ref_name }} + component_pattern: "category: (.*)" + repo_token: ${{ secrets.GITHUB_TOKEN }} + skip_when_only_listed_labels_set: 'docs' + skip_when_only_listed_files_changed: '*.md,*.rst,*.png,*.jpg,*.svg,*/layer_tests_summary/*,*/conformance/*' + + Docker: + needs: Smart_CI + runs-on: aks-linux-4-cores-16gb-docker-build + container: + image: openvinogithubactions.azurecr.io/docker_build:0.2 + volumes: + - /mount:/mount + outputs: + images: "${{ steps.handle_docker.outputs.images }}" + steps: + - name: Checkout + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + + - uses: ./.github/actions/handle_docker + id: handle_docker + with: + images: | + ov_build/ubuntu_22_04_x64 + registry: 'openvinogithubactions.azurecr.io' + dockerfiles_root_dir: '.github/dockerfiles' + changed_components: ${{ needs.smart_ci.outputs.changed_components }} + Build: + needs: [Smart_CI, Docker] timeout-minutes: 500 defaults: run: @@ -28,7 +76,7 @@ jobs: runs-on: aks-linux-16-cores-32gb if: ${{ github.repository_owner == 'openvinotoolkit' }} container: - image: openvinogithubactions.azurecr.io/dockerhub/ubuntu:20.04 + image: ${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_22_04_x64 }} volumes: - /mount:/mount strategy: @@ -58,14 +106,6 @@ jobs: CC: clang steps: - - name: Set apt retries - run: echo 'Acquire::Retries "10";' > /etc/apt/apt.conf.d/80-retries - - - name: Install git - run: | - apt-get update - apt-get install --assume-yes --no-install-recommends git ca-certificates - - name: Clone OpenVINO uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0 with: @@ -87,15 +127,6 @@ jobs: - name: System info uses: ./openvino/.github/actions/system_info - # - # Dependencies - # - - - name: Install build dependencies - run: | - bash ${OPENVINO_REPO}/install_build_dependencies.sh - apt --assume-yes install clang lld - # # Build # @@ -177,11 +208,11 @@ jobs: CXX_Unit_Tests: name: C++ unit tests if: always() - needs: Build + needs: [Docker, Build] timeout-minutes: 100 runs-on: 'aks-linux-16-cores-32gb' container: - image: 'openvinogithubactions.azurecr.io/dockerhub/ubuntu:20.04' + image: ${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_22_04_x64 }} defaults: run: shell: bash @@ -194,14 +225,10 @@ jobs: - SANITIZER: 'UndefinedBehavior' - SANITIZER: 'Thread' env: - DEBIAN_FRONTEND: noninteractive # to prevent apt-get from waiting user input INSTALL_DIR: ${{ github.workspace }}/install INSTALL_TEST_DIR: ${{ github.workspace }}/install/tests TBB_ENABLE_SANITIZERS: 1 steps: - - name: Set apt retries - run: echo 'Acquire::Retries "10";' > /etc/apt/apt.conf.d/80-retries - - name: Download OpenVINO package uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 with: @@ -233,11 +260,6 @@ jobs: tar -xzf openvino_tests.tar.gz -C $INSTALL_DIR popd - - name: Install dependencies (Linux) - run: | - $INSTALL_DIR/install_dependencies/install_openvino_dependencies.sh -c=core -c=dev -c=gpu -y - apt update && apt --assume-yes install clang lld - - name: Fetch Sanitizer Suppression Lists uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0 with: From 8d10090d348100b75c518985a889a25b1e5e054c Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Tue, 24 Sep 2024 13:08:22 +0200 Subject: [PATCH 29/72] fixed image name --- .github/workflows/linux_sanitizers.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/linux_sanitizers.yml b/.github/workflows/linux_sanitizers.yml index 6fa329b9625702..f2863a7a3eb05c 100644 --- a/.github/workflows/linux_sanitizers.yml +++ b/.github/workflows/linux_sanitizers.yml @@ -62,7 +62,7 @@ jobs: id: handle_docker with: images: | - ov_build/ubuntu_22_04_x64 + ov_build/ubuntu_20_04_x64 registry: 'openvinogithubactions.azurecr.io' dockerfiles_root_dir: '.github/dockerfiles' changed_components: ${{ needs.smart_ci.outputs.changed_components }} @@ -76,7 +76,7 @@ jobs: runs-on: aks-linux-16-cores-32gb if: ${{ github.repository_owner == 'openvinotoolkit' }} container: - image: ${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_22_04_x64 }} + image: ${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_20_04_x64 }} volumes: - /mount:/mount strategy: @@ -212,7 +212,7 @@ jobs: timeout-minutes: 100 runs-on: 'aks-linux-16-cores-32gb' container: - image: ${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_22_04_x64 }} + image: ${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_20_04_x64 }} defaults: run: shell: bash From 27cb93271764c3b63d50d82f241f9a57907be4fa Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Tue, 24 Sep 2024 13:16:02 +0200 Subject: [PATCH 30/72] dedicated sanitizers image --- .../ov_build/sanitizers/Dockerfile | 35 +++++++++++++++++++ .github/workflows/linux_sanitizers.yml | 4 +-- 2 files changed, 37 insertions(+), 2 deletions(-) create mode 100644 .github/dockerfiles/ov_build/sanitizers/Dockerfile diff --git a/.github/dockerfiles/ov_build/sanitizers/Dockerfile b/.github/dockerfiles/ov_build/sanitizers/Dockerfile new file mode 100644 index 00000000000000..9c99668d0aded1 --- /dev/null +++ b/.github/dockerfiles/ov_build/sanitizers/Dockerfile @@ -0,0 +1,35 @@ +ARG REGISTRY="docker.io" +FROM ${REGISTRY}/library/ubuntu:20.04 + +USER root + +# APT configuration +RUN echo 'Acquire::Retries "10";' > /etc/apt/apt.conf && \ + echo 'APT::Get::Assume-Yes "true";' >> /etc/apt/apt.conf && \ + echo 'APT::Get::Fix-Broken "true";' >> /etc/apt/apt.conf && \ + echo 'APT::Get::no-install-recommends "true";' >> /etc/apt/apt.conf + +ENV DEBIAN_FRONTEND="noninteractive" \ + TZ="Europe/London" + +RUN apt-get update && \ + apt-get install software-properties-common && \ + add-apt-repository --yes --no-update ppa:git-core/ppa && \ + add-apt-repository --yes --no-update ppa:deadsnakes/ppa && \ + apt-get update && \ + apt-get install \ + curl \ + git \ + gpg-agent \ + tzdata \ + # Compiler \ + clang \ + lld \ + && \ + rm -rf /var/lib/apt/lists/* + +# Install build dependencies +ADD install_build_dependencies.sh /install_build_dependencies.sh +RUN chmod +x /install_build_dependencies.sh && \ + /install_build_dependencies.sh && \ + rm -rf /var/lib/apt/lists/* diff --git a/.github/workflows/linux_sanitizers.yml b/.github/workflows/linux_sanitizers.yml index f2863a7a3eb05c..c889936552d5c0 100644 --- a/.github/workflows/linux_sanitizers.yml +++ b/.github/workflows/linux_sanitizers.yml @@ -62,7 +62,7 @@ jobs: id: handle_docker with: images: | - ov_build/ubuntu_20_04_x64 + ov_build/sanitizers registry: 'openvinogithubactions.azurecr.io' dockerfiles_root_dir: '.github/dockerfiles' changed_components: ${{ needs.smart_ci.outputs.changed_components }} @@ -76,7 +76,7 @@ jobs: runs-on: aks-linux-16-cores-32gb if: ${{ github.repository_owner == 'openvinotoolkit' }} container: - image: ${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_20_04_x64 }} + image: ${{ fromJSON(needs.docker.outputs.images).ov_build.sanitizers }} volumes: - /mount:/mount strategy: From 9fc4eba9c4494904387548a4a8026dec982493e8 Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Tue, 24 Sep 2024 13:30:06 +0200 Subject: [PATCH 31/72] ccache --- .github/workflows/linux_sanitizers.yml | 29 ++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/.github/workflows/linux_sanitizers.yml b/.github/workflows/linux_sanitizers.yml index c889936552d5c0..d07c9b9b79157e 100644 --- a/.github/workflows/linux_sanitizers.yml +++ b/.github/workflows/linux_sanitizers.yml @@ -102,8 +102,12 @@ jobs: BUILD_DIR: /__w/openvino/openvino/openvino_build LSAN_IGNORE: /__w/openvino/openvino/openvino/tests/sanitizers/lsan/suppressions.txt ASAN_IGNORE: /__w/openvino/openvino/openvino/tests/sanitizers/asan/suppressions.supp - CXX: clang++ - CC: clang + CCACHE_REMOTE_DIR: /mount/caches/ccache/sanitizers/${{ matrix.SANITIZER }}/${{ github.base_ref || github.ref_name }} + CCACHE_DIR: /__w/openvino/openvino/ccache + CCACHE_TEMPDIR: /__w/openvino/openvino/ccache_temp + CCACHE_MAXSIZE: 2G + CXX: "ccache clang++" + CC: "ccache clang" steps: - name: Clone OpenVINO @@ -127,9 +131,27 @@ jobs: - name: System info uses: ./openvino/.github/actions/system_info + - name: Setup ccache + id: ccache_restore + uses: ./openvino/.github/actions/cache + with: + # save-always: ${{ github.event_name == 'push' && 'true' || 'false' }} + # cleanup-always: ${{ github.event_name == 'push' && 'true' || 'false' }} + save-always: true + cleanup-always: true + cache-size: 10 + max-cache-size: 50 + cache-path: ${{ env.CCACHE_REMOTE_DIR }} + path: ${{ env.CCACHE_DIR }} + key: ${{ runner.os }}-${{ runner.arch }}-ccache-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-${{ runner.arch }}-ccache + # # Build # + - name: Clean ccache stats + run: ccache --zero-stats - name: CMake configure - OpenVINO run: | @@ -161,6 +183,9 @@ jobs: env: ASAN_OPTIONS: halt_on_error=0:suppressions=${{ env.ASAN_IGNORE }} LSAN_OPTIONS: suppressions=${{ env.LSAN_IGNORE }}:NEOReadDebugKeys=1:DisableDeepBind=1 + + - name: Show ccache stats + run: ccache --show-stats - name: Cmake install - OpenVINO run: | From d9091657e5e9e2f8746151af69a561853e071e79 Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Tue, 24 Sep 2024 13:50:06 +0200 Subject: [PATCH 32/72] set CMAKE_CXX_COMPILER_LAUNCHER --- .github/workflows/linux_sanitizers.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/linux_sanitizers.yml b/.github/workflows/linux_sanitizers.yml index d07c9b9b79157e..fe88fbacd7e1ba 100644 --- a/.github/workflows/linux_sanitizers.yml +++ b/.github/workflows/linux_sanitizers.yml @@ -102,12 +102,14 @@ jobs: BUILD_DIR: /__w/openvino/openvino/openvino_build LSAN_IGNORE: /__w/openvino/openvino/openvino/tests/sanitizers/lsan/suppressions.txt ASAN_IGNORE: /__w/openvino/openvino/openvino/tests/sanitizers/asan/suppressions.supp + MAKE_CXX_COMPILER_LAUNCHER: ccache + CMAKE_C_COMPILER_LAUNCHER: ccache CCACHE_REMOTE_DIR: /mount/caches/ccache/sanitizers/${{ matrix.SANITIZER }}/${{ github.base_ref || github.ref_name }} CCACHE_DIR: /__w/openvino/openvino/ccache CCACHE_TEMPDIR: /__w/openvino/openvino/ccache_temp CCACHE_MAXSIZE: 2G - CXX: "ccache clang++" - CC: "ccache clang" + CXX: clang++ + CC: clang steps: - name: Clone OpenVINO From f119c03801171f71eaa3fd51726577b03cfb7396 Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Tue, 24 Sep 2024 14:03:21 +0200 Subject: [PATCH 33/72] fixed typo --- .github/workflows/linux_sanitizers.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/linux_sanitizers.yml b/.github/workflows/linux_sanitizers.yml index fe88fbacd7e1ba..565be637098283 100644 --- a/.github/workflows/linux_sanitizers.yml +++ b/.github/workflows/linux_sanitizers.yml @@ -102,7 +102,7 @@ jobs: BUILD_DIR: /__w/openvino/openvino/openvino_build LSAN_IGNORE: /__w/openvino/openvino/openvino/tests/sanitizers/lsan/suppressions.txt ASAN_IGNORE: /__w/openvino/openvino/openvino/tests/sanitizers/asan/suppressions.supp - MAKE_CXX_COMPILER_LAUNCHER: ccache + CMAKE_CXX_COMPILER_LAUNCHER: ccache CMAKE_C_COMPILER_LAUNCHER: ccache CCACHE_REMOTE_DIR: /mount/caches/ccache/sanitizers/${{ matrix.SANITIZER }}/${{ github.base_ref || github.ref_name }} CCACHE_DIR: /__w/openvino/openvino/ccache From c134d2fddfb2a93adc0f024c5a9297378e882e82 Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Tue, 24 Sep 2024 16:39:20 +0200 Subject: [PATCH 34/72] changed test image --- .github/workflows/linux_sanitizers.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/linux_sanitizers.yml b/.github/workflows/linux_sanitizers.yml index 565be637098283..59d56827ea44e3 100644 --- a/.github/workflows/linux_sanitizers.yml +++ b/.github/workflows/linux_sanitizers.yml @@ -239,7 +239,7 @@ jobs: timeout-minutes: 100 runs-on: 'aks-linux-16-cores-32gb' container: - image: ${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_20_04_x64 }} + image: ${{ fromJSON(needs.docker.outputs.images).ov_build.sanitizers }} defaults: run: shell: bash From 2c67e756b184b5023671a9a5ff990441e4013039 Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Tue, 24 Sep 2024 18:58:40 +0200 Subject: [PATCH 35/72] improved cache --- .github/workflows/linux_sanitizers.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/linux_sanitizers.yml b/.github/workflows/linux_sanitizers.yml index 59d56827ea44e3..6d87f650382582 100644 --- a/.github/workflows/linux_sanitizers.yml +++ b/.github/workflows/linux_sanitizers.yml @@ -107,7 +107,8 @@ jobs: CCACHE_REMOTE_DIR: /mount/caches/ccache/sanitizers/${{ matrix.SANITIZER }}/${{ github.base_ref || github.ref_name }} CCACHE_DIR: /__w/openvino/openvino/ccache CCACHE_TEMPDIR: /__w/openvino/openvino/ccache_temp - CCACHE_MAXSIZE: 2G + CCACHE_MAXSIZE: 3G + CCACHE_COMPILERCHECK: content CXX: clang++ CC: clang @@ -154,6 +155,9 @@ jobs: # - name: Clean ccache stats run: ccache --zero-stats + + - name: Show ccache stats + run: ccache --show-stats - name: CMake configure - OpenVINO run: | From b7a0bc151e771245a33577fc94df80582ecbfb67 Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Tue, 24 Sep 2024 19:29:55 +0200 Subject: [PATCH 36/72] revert test envs --- .github/workflows/linux_sanitizers.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/linux_sanitizers.yml b/.github/workflows/linux_sanitizers.yml index 6d87f650382582..3bd35e275c72de 100644 --- a/.github/workflows/linux_sanitizers.yml +++ b/.github/workflows/linux_sanitizers.yml @@ -259,6 +259,8 @@ jobs: INSTALL_DIR: ${{ github.workspace }}/install INSTALL_TEST_DIR: ${{ github.workspace }}/install/tests TBB_ENABLE_SANITIZERS: 1 + CXX: clang++ + CC: clang steps: - name: Download OpenVINO package uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 From e3512c8e1deaec258e4d0eb2518e5259e4293061 Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Tue, 24 Sep 2024 23:51:04 +0200 Subject: [PATCH 37/72] di not cleanup build --- .github/workflows/linux_sanitizers.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/linux_sanitizers.yml b/.github/workflows/linux_sanitizers.yml index 3bd35e275c72de..253474abd68c42 100644 --- a/.github/workflows/linux_sanitizers.yml +++ b/.github/workflows/linux_sanitizers.yml @@ -199,9 +199,6 @@ jobs: cmake -DCMAKE_INSTALL_PREFIX=${INSTALL_TEST_DIR} -DCOMPONENT=tests -P ${BUILD_DIR}/cmake_install.cmake --config ${{ env.CMAKE_BUILD_TYPE }} cmake -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} -DCOMPONENT=python_wheels -P ${BUILD_DIR}/cmake_install.cmake --config ${{ env.CMAKE_BUILD_TYPE }} - - name: Remove unused files to free space - run: rm -rf ${BUILD_DIR}/* - # # Tests # From c0854a30218d10340da93f6cbae8691f793a8505 Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Wed, 25 Sep 2024 10:19:13 +0200 Subject: [PATCH 38/72] increased ccache size --- .github/workflows/linux_sanitizers.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/linux_sanitizers.yml b/.github/workflows/linux_sanitizers.yml index 253474abd68c42..d60e2e44ec87ca 100644 --- a/.github/workflows/linux_sanitizers.yml +++ b/.github/workflows/linux_sanitizers.yml @@ -91,7 +91,6 @@ jobs: - SANITIZER: 'Thread' SANITIZER_CMAKE_OPTION: '-DENABLE_THREAD_SANITIZER=ON' env: - DEBIAN_FRONTEND: noninteractive # to prevent apt-get from waiting user input CMAKE_BUILD_TYPE: 'RelWithDebInfo' CMAKE_GENERATOR: 'Ninja' GITHUB_WORKSPACE: '/__w/openvino/openvino' @@ -107,8 +106,7 @@ jobs: CCACHE_REMOTE_DIR: /mount/caches/ccache/sanitizers/${{ matrix.SANITIZER }}/${{ github.base_ref || github.ref_name }} CCACHE_DIR: /__w/openvino/openvino/ccache CCACHE_TEMPDIR: /__w/openvino/openvino/ccache_temp - CCACHE_MAXSIZE: 3G - CCACHE_COMPILERCHECK: content + CCACHE_MAXSIZE: 15G CXX: clang++ CC: clang From bd466ea88522ad0e9c1321f40b3e712c55b1feb0 Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Wed, 25 Sep 2024 12:05:00 +0200 Subject: [PATCH 39/72] ccache size 20g --- .github/workflows/linux_sanitizers.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/linux_sanitizers.yml b/.github/workflows/linux_sanitizers.yml index d60e2e44ec87ca..810b394c555b55 100644 --- a/.github/workflows/linux_sanitizers.yml +++ b/.github/workflows/linux_sanitizers.yml @@ -106,7 +106,7 @@ jobs: CCACHE_REMOTE_DIR: /mount/caches/ccache/sanitizers/${{ matrix.SANITIZER }}/${{ github.base_ref || github.ref_name }} CCACHE_DIR: /__w/openvino/openvino/ccache CCACHE_TEMPDIR: /__w/openvino/openvino/ccache_temp - CCACHE_MAXSIZE: 15G + CCACHE_MAXSIZE: 20G CXX: clang++ CC: clang From abf58b263d2746c0e766ad88786d0e4ce60e6bf3 Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Wed, 25 Sep 2024 13:00:40 +0200 Subject: [PATCH 40/72] changed steps conditionsa --- .github/workflows/linux_sanitizers.yml | 46 +++++++++++++------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/.github/workflows/linux_sanitizers.yml b/.github/workflows/linux_sanitizers.yml index 810b394c555b55..08650df48f35a4 100644 --- a/.github/workflows/linux_sanitizers.yml +++ b/.github/workflows/linux_sanitizers.yml @@ -302,28 +302,28 @@ jobs: # - name: OpenVINO Core Unit Tests - if: always() + if: !cancelled() run: | source ${INSTALL_DIR}/setupvars.sh ${INSTALL_TEST_DIR}/ov_core_unit_tests --gtest_print_time=1 --gtest_filter=-*IE_GPU* \ --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-OVCoreUT.xml - name: OpenVINO Inference Functional Tests - if: always() + if: !cancelled() run: | source ${INSTALL_DIR}/setupvars.sh ${INSTALL_TEST_DIR}/ov_inference_functional_tests --gtest_print_time=1 \ --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-InferenceFunc.xml - name: OpenVINO Inference Unit Tests - if: always() + if: !cancelled() run: | source ${INSTALL_DIR}/setupvars.sh ${INSTALL_TEST_DIR}/ov_inference_unit_tests --gtest_print_time=1 \ --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-InferenceUnit.xml - name: Low Precision Transformations Tests - if: always() + if: !cancelled() run: | source ${INSTALL_DIR}/setupvars.sh @@ -331,28 +331,28 @@ jobs: --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-LpTransformations.xml - name: OpenVINO Conditional compilation tests - if: always() + if: !cancelled() run: | source ${INSTALL_DIR}/setupvars.sh ${INSTALL_TEST_DIR}/ov_conditional_compilation_tests --gtest_print_time=1 \ --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-ConditionalCompilation.xml - name: IR frontend tests - if: always() + if: !cancelled() run: | source ${INSTALL_DIR}/setupvars.sh ${INSTALL_TEST_DIR}/ov_ir_frontend_tests --gtest_print_time=1 \ --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-IRFrontend.xml - name: PaddlePaddle frontend tests - if: always() + if: !cancelled() run: | source ${INSTALL_DIR}/setupvars.sh ${INSTALL_TEST_DIR}/paddle_tests --gtest_print_time=1 \ --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-PaddleTests.xml - name: ONNX frontend tests - if: always() + if: !cancelled() run: | source ${INSTALL_DIR}/setupvars.sh ${INSTALL_TEST_DIR}/ov_onnx_frontend_tests --gtest_print_time=1 \ @@ -360,14 +360,14 @@ jobs: --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-ONNXFrontend.xml - name: TensorFlow Common frontend tests - if: always() + if: !cancelled() run: | source ${INSTALL_DIR}/setupvars.sh ${INSTALL_TEST_DIR}/ov_tensorflow_common_tests --gtest_print_time=1 \ --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-TensorFlowCommonFrontend.xml - name: TensorFlow frontend tests - if: always() + if: !cancelled() run: | source ${INSTALL_DIR}/setupvars.sh @@ -390,56 +390,56 @@ jobs: --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-Transformations.xml - name: Common test utils tests - if: always() + if: !cancelled() run: | source ${INSTALL_DIR}/setupvars.sh ${INSTALL_TEST_DIR}/ov_util_tests --gtest_print_time=1 \ --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-CommonUtilTests.xml - name: Snippets func tests - if: always() + if: !cancelled() run: | source ${INSTALL_DIR}/setupvars.sh ${INSTALL_TEST_DIR}/ov_snippets_func_tests --gtest_print_time=1 \ --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-SnippetsFuncTests.xml - name: CPU plugin unit tests - if: always() + if: !cancelled() run: | source ${INSTALL_DIR}/setupvars.sh ${INSTALL_TEST_DIR}/ov_cpu_unit_tests --gtest_print_time=1 \ --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-CPUUnitTests.xml - name: ov_subgraphs_dumper_tests tests - if: always() + if: !cancelled() run: | source ${INSTALL_DIR}/setupvars.sh ${INSTALL_TEST_DIR}/ov_subgraphs_dumper_tests --gtest_print_time=1 \ --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-ov_subgraphs_dumper_tests.xml - name: Template OpImpl tests - if: always() + if: !cancelled() run: | source ${INSTALL_DIR}/setupvars.sh ${INSTALL_TEST_DIR}/ov_op_conformance_tests --gtest_print_time=1 --device=TEMPLATE --gtest_filter=*OpImpl*\ --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-OpImplTests.xml - name: AUTO unit tests - if: always() + if: !cancelled() run: | source ${INSTALL_DIR}/setupvars.sh ${INSTALL_TEST_DIR}/ov_auto_unit_tests --gtest_print_time=1 \ --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-ov_auto_unit_tests.xml - name: AUTO func Tests - if: always() + if: !cancelled() run: | source ${{ env.INSTALL_DIR }}/setupvars.sh ${{ env.INSTALL_TEST_DIR }}/ov_auto_func_tests --gtest_print_time=1 \ --gtest_output=xml:${{ env.INSTALL_TEST_DIR }}/TEST-ov_auto_func_tests.xml - name: Template plugin func tests - if: always() + if: !cancelled() run: | source ${INSTALL_DIR}/setupvars.sh ${INSTALL_TEST_DIR}/ov_template_func_tests --gtest_print_time=1 \ @@ -447,32 +447,32 @@ jobs: --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-TemplateFuncTests.xml - name: OpenVINO C API tests - if: always() + if: !cancelled() run: | source ${INSTALL_DIR}/setupvars.sh ${INSTALL_TEST_DIR}/ov_capi_test --gtest_print_time=1 \ --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-OpenVINOCAPITests.xml - name: AutoBatch unit tests - if: always() + if: !cancelled() run: | source ${INSTALL_DIR}/setupvars.sh ${INSTALL_TEST_DIR}/ov_auto_batch_unit_tests --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-ov_auto_batch_unit_tests.xml - name: AutoBatch func tests - if: always() + if: !cancelled() run: | source ${INSTALL_DIR}/setupvars.sh ${INSTALL_TEST_DIR}/ov_auto_batch_func_tests --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-ov_auto_batch_func_tests.xml --gtest_filter="*smoke*" - name: Proxy Plugin func tests - if: always() + if: !cancelled() run: | source ${INSTALL_DIR}/setupvars.sh ${INSTALL_TEST_DIR}/ov_proxy_plugin_tests --gtest_print_time=1 --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-OVProxyTests.xml - name: Hetero unit tests - if: always() + if: !cancelled() run: | source ${{ env.INSTALL_DIR }}/setupvars.sh ${{ env.INSTALL_TEST_DIR }}/ov_hetero_unit_tests --gtest_print_time=1 --gtest_output=xml:${{ env.INSTALL_TEST_DIR }}/TEST-OVHeteroUnitTests.xml From 61196543257bb7e9978fe7c5c36b6b1165856aa4 Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Wed, 25 Sep 2024 13:13:45 +0200 Subject: [PATCH 41/72] fixed syntax --- .github/workflows/linux_sanitizers.yml | 46 +++++++++++++------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/.github/workflows/linux_sanitizers.yml b/.github/workflows/linux_sanitizers.yml index 08650df48f35a4..3c778c353d1670 100644 --- a/.github/workflows/linux_sanitizers.yml +++ b/.github/workflows/linux_sanitizers.yml @@ -302,28 +302,28 @@ jobs: # - name: OpenVINO Core Unit Tests - if: !cancelled() + if: ${{ !cancelled() }} run: | source ${INSTALL_DIR}/setupvars.sh ${INSTALL_TEST_DIR}/ov_core_unit_tests --gtest_print_time=1 --gtest_filter=-*IE_GPU* \ --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-OVCoreUT.xml - name: OpenVINO Inference Functional Tests - if: !cancelled() + if: ${{ !cancelled() }} run: | source ${INSTALL_DIR}/setupvars.sh ${INSTALL_TEST_DIR}/ov_inference_functional_tests --gtest_print_time=1 \ --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-InferenceFunc.xml - name: OpenVINO Inference Unit Tests - if: !cancelled() + if: ${{ !cancelled() }} run: | source ${INSTALL_DIR}/setupvars.sh ${INSTALL_TEST_DIR}/ov_inference_unit_tests --gtest_print_time=1 \ --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-InferenceUnit.xml - name: Low Precision Transformations Tests - if: !cancelled() + if: ${{ !cancelled() }} run: | source ${INSTALL_DIR}/setupvars.sh @@ -331,28 +331,28 @@ jobs: --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-LpTransformations.xml - name: OpenVINO Conditional compilation tests - if: !cancelled() + if: ${{ !cancelled() }} run: | source ${INSTALL_DIR}/setupvars.sh ${INSTALL_TEST_DIR}/ov_conditional_compilation_tests --gtest_print_time=1 \ --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-ConditionalCompilation.xml - name: IR frontend tests - if: !cancelled() + if: ${{ !cancelled() }} run: | source ${INSTALL_DIR}/setupvars.sh ${INSTALL_TEST_DIR}/ov_ir_frontend_tests --gtest_print_time=1 \ --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-IRFrontend.xml - name: PaddlePaddle frontend tests - if: !cancelled() + if: ${{ !cancelled() }} run: | source ${INSTALL_DIR}/setupvars.sh ${INSTALL_TEST_DIR}/paddle_tests --gtest_print_time=1 \ --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-PaddleTests.xml - name: ONNX frontend tests - if: !cancelled() + if: ${{ !cancelled() }} run: | source ${INSTALL_DIR}/setupvars.sh ${INSTALL_TEST_DIR}/ov_onnx_frontend_tests --gtest_print_time=1 \ @@ -360,14 +360,14 @@ jobs: --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-ONNXFrontend.xml - name: TensorFlow Common frontend tests - if: !cancelled() + if: ${{ !cancelled() }} run: | source ${INSTALL_DIR}/setupvars.sh ${INSTALL_TEST_DIR}/ov_tensorflow_common_tests --gtest_print_time=1 \ --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-TensorFlowCommonFrontend.xml - name: TensorFlow frontend tests - if: !cancelled() + if: ${{ !cancelled() }} run: | source ${INSTALL_DIR}/setupvars.sh @@ -390,56 +390,56 @@ jobs: --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-Transformations.xml - name: Common test utils tests - if: !cancelled() + if: ${{ !cancelled() }} run: | source ${INSTALL_DIR}/setupvars.sh ${INSTALL_TEST_DIR}/ov_util_tests --gtest_print_time=1 \ --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-CommonUtilTests.xml - name: Snippets func tests - if: !cancelled() + if: ${{ !cancelled() }} run: | source ${INSTALL_DIR}/setupvars.sh ${INSTALL_TEST_DIR}/ov_snippets_func_tests --gtest_print_time=1 \ --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-SnippetsFuncTests.xml - name: CPU plugin unit tests - if: !cancelled() + if: ${{ !cancelled() }} run: | source ${INSTALL_DIR}/setupvars.sh ${INSTALL_TEST_DIR}/ov_cpu_unit_tests --gtest_print_time=1 \ --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-CPUUnitTests.xml - name: ov_subgraphs_dumper_tests tests - if: !cancelled() + if: ${{ !cancelled() }} run: | source ${INSTALL_DIR}/setupvars.sh ${INSTALL_TEST_DIR}/ov_subgraphs_dumper_tests --gtest_print_time=1 \ --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-ov_subgraphs_dumper_tests.xml - name: Template OpImpl tests - if: !cancelled() + if: ${{ !cancelled() }} run: | source ${INSTALL_DIR}/setupvars.sh ${INSTALL_TEST_DIR}/ov_op_conformance_tests --gtest_print_time=1 --device=TEMPLATE --gtest_filter=*OpImpl*\ --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-OpImplTests.xml - name: AUTO unit tests - if: !cancelled() + if: ${{ !cancelled() }} run: | source ${INSTALL_DIR}/setupvars.sh ${INSTALL_TEST_DIR}/ov_auto_unit_tests --gtest_print_time=1 \ --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-ov_auto_unit_tests.xml - name: AUTO func Tests - if: !cancelled() + if: ${{ !cancelled() }} run: | source ${{ env.INSTALL_DIR }}/setupvars.sh ${{ env.INSTALL_TEST_DIR }}/ov_auto_func_tests --gtest_print_time=1 \ --gtest_output=xml:${{ env.INSTALL_TEST_DIR }}/TEST-ov_auto_func_tests.xml - name: Template plugin func tests - if: !cancelled() + if: ${{ !cancelled() }} run: | source ${INSTALL_DIR}/setupvars.sh ${INSTALL_TEST_DIR}/ov_template_func_tests --gtest_print_time=1 \ @@ -447,32 +447,32 @@ jobs: --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-TemplateFuncTests.xml - name: OpenVINO C API tests - if: !cancelled() + if: ${{ !cancelled() }} run: | source ${INSTALL_DIR}/setupvars.sh ${INSTALL_TEST_DIR}/ov_capi_test --gtest_print_time=1 \ --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-OpenVINOCAPITests.xml - name: AutoBatch unit tests - if: !cancelled() + if: ${{ !cancelled() }} run: | source ${INSTALL_DIR}/setupvars.sh ${INSTALL_TEST_DIR}/ov_auto_batch_unit_tests --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-ov_auto_batch_unit_tests.xml - name: AutoBatch func tests - if: !cancelled() + if: ${{ !cancelled() }} run: | source ${INSTALL_DIR}/setupvars.sh ${INSTALL_TEST_DIR}/ov_auto_batch_func_tests --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-ov_auto_batch_func_tests.xml --gtest_filter="*smoke*" - name: Proxy Plugin func tests - if: !cancelled() + if: ${{ !cancelled() }} run: | source ${INSTALL_DIR}/setupvars.sh ${INSTALL_TEST_DIR}/ov_proxy_plugin_tests --gtest_print_time=1 --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-OVProxyTests.xml - name: Hetero unit tests - if: !cancelled() + if: ${{ !cancelled() }} run: | source ${{ env.INSTALL_DIR }}/setupvars.sh ${{ env.INSTALL_TEST_DIR }}/ov_hetero_unit_tests --gtest_print_time=1 --gtest_output=xml:${{ env.INSTALL_TEST_DIR }}/TEST-OVHeteroUnitTests.xml From 26f13f8d7814f69a8ce78816be934973492c7b06 Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Wed, 25 Sep 2024 14:07:04 +0200 Subject: [PATCH 42/72] use gcc10 --- .github/dockerfiles/ov_build/sanitizers/Dockerfile | 8 ++++++-- .github/workflows/linux_sanitizers.yml | 5 +---- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/dockerfiles/ov_build/sanitizers/Dockerfile b/.github/dockerfiles/ov_build/sanitizers/Dockerfile index 9c99668d0aded1..59f9fc62611c09 100644 --- a/.github/dockerfiles/ov_build/sanitizers/Dockerfile +++ b/.github/dockerfiles/ov_build/sanitizers/Dockerfile @@ -23,8 +23,8 @@ RUN apt-get update && \ gpg-agent \ tzdata \ # Compiler \ - clang \ - lld \ + gcc-10 \ + g++-10 \ && \ rm -rf /var/lib/apt/lists/* @@ -33,3 +33,7 @@ ADD install_build_dependencies.sh /install_build_dependencies.sh RUN chmod +x /install_build_dependencies.sh && \ /install_build_dependencies.sh && \ rm -rf /var/lib/apt/lists/* + +# Set gcc-10 as a default compiler +RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 30 && \ + update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-10 30 \ No newline at end of file diff --git a/.github/workflows/linux_sanitizers.yml b/.github/workflows/linux_sanitizers.yml index 3c778c353d1670..e9de7aa1398a47 100644 --- a/.github/workflows/linux_sanitizers.yml +++ b/.github/workflows/linux_sanitizers.yml @@ -107,8 +107,6 @@ jobs: CCACHE_DIR: /__w/openvino/openvino/ccache CCACHE_TEMPDIR: /__w/openvino/openvino/ccache_temp CCACHE_MAXSIZE: 20G - CXX: clang++ - CC: clang steps: - name: Clone OpenVINO @@ -254,8 +252,7 @@ jobs: INSTALL_DIR: ${{ github.workspace }}/install INSTALL_TEST_DIR: ${{ github.workspace }}/install/tests TBB_ENABLE_SANITIZERS: 1 - CXX: clang++ - CC: clang + steps: - name: Download OpenVINO package uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 From 23b0e94d6a072f9b4a9348af4bef35384240942f Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Wed, 25 Sep 2024 14:08:42 +0200 Subject: [PATCH 43/72] -revert lubsan --- cache | 1 + src/plugins/intel_npu/thirdparty/level-zero | 1 + 2 files changed, 2 insertions(+) create mode 160000 cache create mode 160000 src/plugins/intel_npu/thirdparty/level-zero diff --git a/cache b/cache new file mode 160000 index 00000000000000..0c45773b623bea --- /dev/null +++ b/cache @@ -0,0 +1 @@ +Subproject commit 0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 diff --git a/src/plugins/intel_npu/thirdparty/level-zero b/src/plugins/intel_npu/thirdparty/level-zero new file mode 160000 index 00000000000000..fd459f28041c17 --- /dev/null +++ b/src/plugins/intel_npu/thirdparty/level-zero @@ -0,0 +1 @@ +Subproject commit fd459f28041c176df23a0b7b791ff20a7689d237 From cf6291648947abaf4e63e1787be622f900a19f65 Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Wed, 25 Sep 2024 14:48:01 +0200 Subject: [PATCH 44/72] -revert lubsan --- cache | 1 - src/plugins/intel_npu/thirdparty/level-zero | 1 - 2 files changed, 2 deletions(-) delete mode 160000 cache delete mode 160000 src/plugins/intel_npu/thirdparty/level-zero diff --git a/cache b/cache deleted file mode 160000 index 0c45773b623bea..00000000000000 --- a/cache +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 diff --git a/src/plugins/intel_npu/thirdparty/level-zero b/src/plugins/intel_npu/thirdparty/level-zero deleted file mode 160000 index fd459f28041c17..00000000000000 --- a/src/plugins/intel_npu/thirdparty/level-zero +++ /dev/null @@ -1 +0,0 @@ -Subproject commit fd459f28041c176df23a0b7b791ff20a7689d237 From 7288c030115f9492376d1bb363bc89f8d0b4a095 Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Wed, 25 Sep 2024 21:41:47 +0200 Subject: [PATCH 45/72] debug --- .github/workflows/linux_sanitizers.yml | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/.github/workflows/linux_sanitizers.yml b/.github/workflows/linux_sanitizers.yml index e9de7aa1398a47..a1202fc45e0d19 100644 --- a/.github/workflows/linux_sanitizers.yml +++ b/.github/workflows/linux_sanitizers.yml @@ -106,7 +106,9 @@ jobs: CCACHE_REMOTE_DIR: /mount/caches/ccache/sanitizers/${{ matrix.SANITIZER }}/${{ github.base_ref || github.ref_name }} CCACHE_DIR: /__w/openvino/openvino/ccache CCACHE_TEMPDIR: /__w/openvino/openvino/ccache_temp - CCACHE_MAXSIZE: 20G + CCACHE_MAXSIZE: 30G + CCACHE_LOGFILE: /__w/openvino/openvino/openvino_build/ccache.log + CCACHE_DEBUG: 1 steps: - name: Clone OpenVINO @@ -161,18 +163,18 @@ jobs: -G "${{ env.CMAKE_GENERATOR }}" \ -DENABLE_CPPLINT=OFF \ -DENABLE_NCC_STYLE=OFF \ - -DENABLE_TESTS=ON \ + -DENABLE_TESTS=OFF \ -DENABLE_STRICT_DEPENDENCIES=OFF \ -DENABLE_SYSTEM_OPENCL=ON \ -DCMAKE_VERBOSE_MAKEFILE=ON \ -DCPACK_GENERATOR=TGZ \ -DBUILD_SHARED_LIBS=ON \ - -DENABLE_OV_TF_FRONTEND=ON \ - -DENABLE_OV_TF_LITE_FRONTEND=ON \ - -DENABLE_OV_PADDLE_FRONTEND=ON \ - -DENABLE_OV_PYTORCH_FRONTEND=ON \ - -DENABLE_OV_JAX_FRONTEND=ON \ - -DENABLE_OV_ONNX_FRONTEND=ON \ + -DENABLE_OV_TF_FRONTEND=OFF \ + -DENABLE_OV_TF_LITE_FRONTEND=OFF \ + -DENABLE_OV_PADDLE_FRONTEND=OFF \ + -DENABLE_OV_PYTORCH_FRONTEND=OFF \ + -DENABLE_OV_JAX_FRONTEND=OFF \ + -DENABLE_OV_ONNX_FRONTEND=OFF \ -DENABLE_ONEDNN_FOR_GPU=OFF \ -DCMAKE_COMPILE_WARNING_AS_ERROR=OFF \ ${{ matrix.SANITIZER_CMAKE_OPTION }} \ @@ -212,6 +214,13 @@ jobs: # # Upload build artifacts # + - name: Upload ccache log + if: ${{ always() }} + uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0 + with: + name: ccache_log_${{ matrix.SANITIZER }} + path: ${{ env.CCACHE_LOGFILE }} + if-no-files-found: 'error' - name: Upload openvino package if: ${{ always() }} From 7dfe47a45dbc6d76b7c9478bd71692fdc5b24395 Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Wed, 25 Sep 2024 22:07:08 +0200 Subject: [PATCH 46/72] tmp disable tests --- .github/workflows/linux_sanitizers.yml | 46 +++++++++++++------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/.github/workflows/linux_sanitizers.yml b/.github/workflows/linux_sanitizers.yml index a1202fc45e0d19..fb02e00d1c8f31 100644 --- a/.github/workflows/linux_sanitizers.yml +++ b/.github/workflows/linux_sanitizers.yml @@ -308,28 +308,28 @@ jobs: # - name: OpenVINO Core Unit Tests - if: ${{ !cancelled() }} + # if: ${{ !cancelled() }} run: | source ${INSTALL_DIR}/setupvars.sh ${INSTALL_TEST_DIR}/ov_core_unit_tests --gtest_print_time=1 --gtest_filter=-*IE_GPU* \ --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-OVCoreUT.xml - name: OpenVINO Inference Functional Tests - if: ${{ !cancelled() }} + # if: ${{ !cancelled() }} run: | source ${INSTALL_DIR}/setupvars.sh ${INSTALL_TEST_DIR}/ov_inference_functional_tests --gtest_print_time=1 \ --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-InferenceFunc.xml - name: OpenVINO Inference Unit Tests - if: ${{ !cancelled() }} + # if: ${{ !cancelled() }} run: | source ${INSTALL_DIR}/setupvars.sh ${INSTALL_TEST_DIR}/ov_inference_unit_tests --gtest_print_time=1 \ --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-InferenceUnit.xml - name: Low Precision Transformations Tests - if: ${{ !cancelled() }} + # if: ${{ !cancelled() }} run: | source ${INSTALL_DIR}/setupvars.sh @@ -337,28 +337,28 @@ jobs: --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-LpTransformations.xml - name: OpenVINO Conditional compilation tests - if: ${{ !cancelled() }} + # if: ${{ !cancelled() }} run: | source ${INSTALL_DIR}/setupvars.sh ${INSTALL_TEST_DIR}/ov_conditional_compilation_tests --gtest_print_time=1 \ --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-ConditionalCompilation.xml - name: IR frontend tests - if: ${{ !cancelled() }} + # if: ${{ !cancelled() }} run: | source ${INSTALL_DIR}/setupvars.sh ${INSTALL_TEST_DIR}/ov_ir_frontend_tests --gtest_print_time=1 \ --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-IRFrontend.xml - name: PaddlePaddle frontend tests - if: ${{ !cancelled() }} + # if: ${{ !cancelled() }} run: | source ${INSTALL_DIR}/setupvars.sh ${INSTALL_TEST_DIR}/paddle_tests --gtest_print_time=1 \ --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-PaddleTests.xml - name: ONNX frontend tests - if: ${{ !cancelled() }} + # if: ${{ !cancelled() }} run: | source ${INSTALL_DIR}/setupvars.sh ${INSTALL_TEST_DIR}/ov_onnx_frontend_tests --gtest_print_time=1 \ @@ -366,14 +366,14 @@ jobs: --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-ONNXFrontend.xml - name: TensorFlow Common frontend tests - if: ${{ !cancelled() }} + # if: ${{ !cancelled() }} run: | source ${INSTALL_DIR}/setupvars.sh ${INSTALL_TEST_DIR}/ov_tensorflow_common_tests --gtest_print_time=1 \ --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-TensorFlowCommonFrontend.xml - name: TensorFlow frontend tests - if: ${{ !cancelled() }} + # if: ${{ !cancelled() }} run: | source ${INSTALL_DIR}/setupvars.sh @@ -396,56 +396,56 @@ jobs: --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-Transformations.xml - name: Common test utils tests - if: ${{ !cancelled() }} + # if: ${{ !cancelled() }} run: | source ${INSTALL_DIR}/setupvars.sh ${INSTALL_TEST_DIR}/ov_util_tests --gtest_print_time=1 \ --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-CommonUtilTests.xml - name: Snippets func tests - if: ${{ !cancelled() }} + # if: ${{ !cancelled() }} run: | source ${INSTALL_DIR}/setupvars.sh ${INSTALL_TEST_DIR}/ov_snippets_func_tests --gtest_print_time=1 \ --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-SnippetsFuncTests.xml - name: CPU plugin unit tests - if: ${{ !cancelled() }} + # if: ${{ !cancelled() }} run: | source ${INSTALL_DIR}/setupvars.sh ${INSTALL_TEST_DIR}/ov_cpu_unit_tests --gtest_print_time=1 \ --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-CPUUnitTests.xml - name: ov_subgraphs_dumper_tests tests - if: ${{ !cancelled() }} + # if: ${{ !cancelled() }} run: | source ${INSTALL_DIR}/setupvars.sh ${INSTALL_TEST_DIR}/ov_subgraphs_dumper_tests --gtest_print_time=1 \ --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-ov_subgraphs_dumper_tests.xml - name: Template OpImpl tests - if: ${{ !cancelled() }} + # if: ${{ !cancelled() }} run: | source ${INSTALL_DIR}/setupvars.sh ${INSTALL_TEST_DIR}/ov_op_conformance_tests --gtest_print_time=1 --device=TEMPLATE --gtest_filter=*OpImpl*\ --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-OpImplTests.xml - name: AUTO unit tests - if: ${{ !cancelled() }} + # if: ${{ !cancelled() }} run: | source ${INSTALL_DIR}/setupvars.sh ${INSTALL_TEST_DIR}/ov_auto_unit_tests --gtest_print_time=1 \ --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-ov_auto_unit_tests.xml - name: AUTO func Tests - if: ${{ !cancelled() }} + # if: ${{ !cancelled() }} run: | source ${{ env.INSTALL_DIR }}/setupvars.sh ${{ env.INSTALL_TEST_DIR }}/ov_auto_func_tests --gtest_print_time=1 \ --gtest_output=xml:${{ env.INSTALL_TEST_DIR }}/TEST-ov_auto_func_tests.xml - name: Template plugin func tests - if: ${{ !cancelled() }} + # if: ${{ !cancelled() }} run: | source ${INSTALL_DIR}/setupvars.sh ${INSTALL_TEST_DIR}/ov_template_func_tests --gtest_print_time=1 \ @@ -453,32 +453,32 @@ jobs: --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-TemplateFuncTests.xml - name: OpenVINO C API tests - if: ${{ !cancelled() }} + # if: ${{ !cancelled() }} run: | source ${INSTALL_DIR}/setupvars.sh ${INSTALL_TEST_DIR}/ov_capi_test --gtest_print_time=1 \ --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-OpenVINOCAPITests.xml - name: AutoBatch unit tests - if: ${{ !cancelled() }} + # if: ${{ !cancelled() }} run: | source ${INSTALL_DIR}/setupvars.sh ${INSTALL_TEST_DIR}/ov_auto_batch_unit_tests --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-ov_auto_batch_unit_tests.xml - name: AutoBatch func tests - if: ${{ !cancelled() }} + # if: ${{ !cancelled() }} run: | source ${INSTALL_DIR}/setupvars.sh ${INSTALL_TEST_DIR}/ov_auto_batch_func_tests --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-ov_auto_batch_func_tests.xml --gtest_filter="*smoke*" - name: Proxy Plugin func tests - if: ${{ !cancelled() }} + # if: ${{ !cancelled() }} run: | source ${INSTALL_DIR}/setupvars.sh ${INSTALL_TEST_DIR}/ov_proxy_plugin_tests --gtest_print_time=1 --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-OVProxyTests.xml - name: Hetero unit tests - if: ${{ !cancelled() }} + # if: ${{ !cancelled() }} run: | source ${{ env.INSTALL_DIR }}/setupvars.sh ${{ env.INSTALL_TEST_DIR }}/ov_hetero_unit_tests --gtest_print_time=1 --gtest_output=xml:${{ env.INSTALL_TEST_DIR }}/TEST-OVHeteroUnitTests.xml From 26e2713c0dbe8ac468dcf5f2de3906d4ad18a2e5 Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Thu, 26 Sep 2024 14:36:40 +0200 Subject: [PATCH 47/72] check compilers --- .github/workflows/linux_sanitizers.yml | 14 +++++++------- thirdparty/dependencies.cmake | 4 ++-- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/linux_sanitizers.yml b/.github/workflows/linux_sanitizers.yml index fb02e00d1c8f31..a623eb55eb90ad 100644 --- a/.github/workflows/linux_sanitizers.yml +++ b/.github/workflows/linux_sanitizers.yml @@ -106,7 +106,7 @@ jobs: CCACHE_REMOTE_DIR: /mount/caches/ccache/sanitizers/${{ matrix.SANITIZER }}/${{ github.base_ref || github.ref_name }} CCACHE_DIR: /__w/openvino/openvino/ccache CCACHE_TEMPDIR: /__w/openvino/openvino/ccache_temp - CCACHE_MAXSIZE: 30G + CCACHE_MAXSIZE: 20G CCACHE_LOGFILE: /__w/openvino/openvino/openvino_build/ccache.log CCACHE_DEBUG: 1 @@ -169,12 +169,12 @@ jobs: -DCMAKE_VERBOSE_MAKEFILE=ON \ -DCPACK_GENERATOR=TGZ \ -DBUILD_SHARED_LIBS=ON \ - -DENABLE_OV_TF_FRONTEND=OFF \ - -DENABLE_OV_TF_LITE_FRONTEND=OFF \ - -DENABLE_OV_PADDLE_FRONTEND=OFF \ - -DENABLE_OV_PYTORCH_FRONTEND=OFF \ - -DENABLE_OV_JAX_FRONTEND=OFF \ - -DENABLE_OV_ONNX_FRONTEND=OFF \ + -DENABLE_OV_TF_FRONTEND=ON \ + -DENABLE_OV_TF_LITE_FRONTEND=ON \ + -DENABLE_OV_PADDLE_FRONTEND=ON \ + -DENABLE_OV_PYTORCH_FRONTEND=ON \ + -DENABLE_OV_JAX_FRONTEND=ON \ + -DENABLE_OV_ONNX_FRONTEND=ON \ -DENABLE_ONEDNN_FOR_GPU=OFF \ -DCMAKE_COMPILE_WARNING_AS_ERROR=OFF \ ${{ matrix.SANITIZER_CMAKE_OPTION }} \ diff --git a/thirdparty/dependencies.cmake b/thirdparty/dependencies.cmake index 976e92e8634c39..0e8536a1714a35 100644 --- a/thirdparty/dependencies.cmake +++ b/thirdparty/dependencies.cmake @@ -357,8 +357,8 @@ if(ENABLE_OV_PADDLE_FRONTEND OR ENABLE_OV_ONNX_FRONTEND OR ENABLE_OV_TF_FRONTEND endif() else() add_subdirectory(thirdparty/protobuf EXCLUDE_FROM_ALL) - # protobuf fails to build with -fsanitize=thread - if(ENABLE_THREAD_SANITIZER) + # protobuf fails to build with -fsanitize=thread by clang + if(ENABLE_THREAD_SANITIZER AND OV_COMPILER_IS_CLANG) foreach(proto_target protoc libprotobuf libprotobuf-lite) if(TARGET ${proto_target}) target_compile_options(${proto_target} PUBLIC -fno-sanitize=thread) From f9177afe65f5c2b1421efa77c6444505315102c5 Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Thu, 26 Sep 2024 15:55:29 +0200 Subject: [PATCH 48/72] enabled tests --- .github/workflows/linux_sanitizers.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/linux_sanitizers.yml b/.github/workflows/linux_sanitizers.yml index a623eb55eb90ad..646c771cbf7d57 100644 --- a/.github/workflows/linux_sanitizers.yml +++ b/.github/workflows/linux_sanitizers.yml @@ -163,7 +163,7 @@ jobs: -G "${{ env.CMAKE_GENERATOR }}" \ -DENABLE_CPPLINT=OFF \ -DENABLE_NCC_STYLE=OFF \ - -DENABLE_TESTS=OFF \ + -DENABLE_TESTS=ON \ -DENABLE_STRICT_DEPENDENCIES=OFF \ -DENABLE_SYSTEM_OPENCL=ON \ -DCMAKE_VERBOSE_MAKEFILE=ON \ From 7f0ae9a89b8f09dfe681973867923ce6097daf39 Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Fri, 27 Sep 2024 13:39:49 +0200 Subject: [PATCH 49/72] final flags --- cmake/developer_package/compile_flags/sanitizer.cmake | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/cmake/developer_package/compile_flags/sanitizer.cmake b/cmake/developer_package/compile_flags/sanitizer.cmake index 6c1956b84c18b5..d8385f9ed2fa0b 100644 --- a/cmake/developer_package/compile_flags/sanitizer.cmake +++ b/cmake/developer_package/compile_flags/sanitizer.cmake @@ -63,7 +63,7 @@ if(ENABLE_UB_SANITIZER) # Mute -fsanitize=enum Load of a value of an enumerated type which is not in the range of representable values for that enumerated type. # Samples cases: # load of value 4294967295, which is not a valid value for type 'const (anonymous namespace)::onnx::Field' - set(SANITIZER_COMPILER_FLAGS "${SANITIZER_COMPILER_FLAGS} -fsanitize=undefined -fno-sanitize=null -fno-sanitize=alignment -fno-sanitize=bool -fno-sanitize=enum") + set(SANITIZER_COMPILER_FLAGS "${SANITIZER_COMPILER_FLAGS} -fsanitize=undefined -fno-sanitize=null -fno-sanitize=alignment -fno-sanitize=bool -fno-sanitize=enum -fsanitize-undefined-trap-on-error") if(OV_COMPILER_IS_CLANG) set(SANITIZER_COMPILER_FLAGS "${SANITIZER_COMPILER_FLAGS} -fno-sanitize=function") endif() @@ -89,7 +89,11 @@ if(ENABLE_THREAD_SANITIZER) message(FATAL_ERROR "Thread sanitizer is not supported in Windows with MSVC compiler. Please, use clang-cl or mingw") elseif(CMAKE_COMPILER_IS_GNUCXX OR OV_COMPILER_IS_CLANG) set(SANITIZER_COMPILER_FLAGS "${SANITIZER_COMPILER_FLAGS} -fsanitize=thread") - set(SANITIZER_LINKER_FLAGS "${SANITIZER_LINKER_FLAGS} -fsanitize=thread") + if(OV_COMPILER_IS_CLANG) + set(SANITIZER_LINKER_FLAGS "${SANITIZER_LINKER_FLAGS} -ltsan") + else() + set(SANITIZER_LINKER_FLAGS "${SANITIZER_LINKER_FLAGS} -fsanitize=thread") + endif() else() message(WARNING "Unsupported CXX compiler ${CMAKE_CXX_COMPILER_ID}") endif() From c84fb76b6be562347a446aefbd8de183a30a5af5 Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Fri, 27 Sep 2024 17:13:07 +0200 Subject: [PATCH 50/72] debug info --- .github/workflows/linux_sanitizers.yml | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/linux_sanitizers.yml b/.github/workflows/linux_sanitizers.yml index 646c771cbf7d57..1d7ab628ee02dc 100644 --- a/.github/workflows/linux_sanitizers.yml +++ b/.github/workflows/linux_sanitizers.yml @@ -106,7 +106,7 @@ jobs: CCACHE_REMOTE_DIR: /mount/caches/ccache/sanitizers/${{ matrix.SANITIZER }}/${{ github.base_ref || github.ref_name }} CCACHE_DIR: /__w/openvino/openvino/ccache CCACHE_TEMPDIR: /__w/openvino/openvino/ccache_temp - CCACHE_MAXSIZE: 20G + CCACHE_MAXSIZE: 30G CCACHE_LOGFILE: /__w/openvino/openvino/openvino_build/ccache.log CCACHE_DEBUG: 1 @@ -153,7 +153,7 @@ jobs: # - name: Clean ccache stats run: ccache --zero-stats - + - name: Show ccache stats run: ccache --show-stats @@ -163,7 +163,7 @@ jobs: -G "${{ env.CMAKE_GENERATOR }}" \ -DENABLE_CPPLINT=OFF \ -DENABLE_NCC_STYLE=OFF \ - -DENABLE_TESTS=ON \ + -DENABLE_TESTS=OFF \ -DENABLE_STRICT_DEPENDENCIES=OFF \ -DENABLE_SYSTEM_OPENCL=ON \ -DCMAKE_VERBOSE_MAKEFILE=ON \ @@ -188,6 +188,12 @@ jobs: ASAN_OPTIONS: halt_on_error=0:suppressions=${{ env.ASAN_IGNORE }} LSAN_OPTIONS: suppressions=${{ env.LSAN_IGNORE }}:NEOReadDebugKeys=1:DisableDeepBind=1 + - name: Show ccache stats + run: ccache --show-stats + + - name: Cleanup ccache + run: ccache --cleanup --zero-stats --max-size=15G + - name: Show ccache stats run: ccache --show-stats From 9760f6a693ab921a68d97e17cb3ab9770416746d Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Fri, 27 Sep 2024 17:58:41 +0200 Subject: [PATCH 51/72] cleanup the ccache --- .github/workflows/linux_sanitizers.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/linux_sanitizers.yml b/.github/workflows/linux_sanitizers.yml index 1d7ab628ee02dc..4378861a4a5512 100644 --- a/.github/workflows/linux_sanitizers.yml +++ b/.github/workflows/linux_sanitizers.yml @@ -108,7 +108,7 @@ jobs: CCACHE_TEMPDIR: /__w/openvino/openvino/ccache_temp CCACHE_MAXSIZE: 30G CCACHE_LOGFILE: /__w/openvino/openvino/openvino_build/ccache.log - CCACHE_DEBUG: 1 + # CCACHE_DEBUG: 1 steps: - name: Clone OpenVINO @@ -193,6 +193,8 @@ jobs: - name: Cleanup ccache run: ccache --cleanup --zero-stats --max-size=15G + env: + CCACHE_MAXSIZE: 15G - name: Show ccache stats run: ccache --show-stats From cc31b94261fa7cb429c00c0aa38cf23f6c3d574f Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Fri, 27 Sep 2024 19:41:07 +0200 Subject: [PATCH 52/72] reverted tests --- .github/workflows/linux_sanitizers.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/linux_sanitizers.yml b/.github/workflows/linux_sanitizers.yml index 4378861a4a5512..f9ec0bb7894818 100644 --- a/.github/workflows/linux_sanitizers.yml +++ b/.github/workflows/linux_sanitizers.yml @@ -163,7 +163,7 @@ jobs: -G "${{ env.CMAKE_GENERATOR }}" \ -DENABLE_CPPLINT=OFF \ -DENABLE_NCC_STYLE=OFF \ - -DENABLE_TESTS=OFF \ + -DENABLE_TESTS=ON \ -DENABLE_STRICT_DEPENDENCIES=OFF \ -DENABLE_SYSTEM_OPENCL=ON \ -DCMAKE_VERBOSE_MAKEFILE=ON \ From 90c61ce49122ae05fc6315ce494a9cfccf0ba026 Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Mon, 30 Sep 2024 10:42:25 +0200 Subject: [PATCH 53/72] sccache instead of ccache --- .../ov_build/sanitizers/Dockerfile | 14 ++++- .github/workflows/linux_sanitizers.yml | 51 ++++++------------- 2 files changed, 29 insertions(+), 36 deletions(-) diff --git a/.github/dockerfiles/ov_build/sanitizers/Dockerfile b/.github/dockerfiles/ov_build/sanitizers/Dockerfile index 59f9fc62611c09..83089a981b2022 100644 --- a/.github/dockerfiles/ov_build/sanitizers/Dockerfile +++ b/.github/dockerfiles/ov_build/sanitizers/Dockerfile @@ -36,4 +36,16 @@ RUN chmod +x /install_build_dependencies.sh && \ # Set gcc-10 as a default compiler RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 30 && \ - update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-10 30 \ No newline at end of file + update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-10 30 + +# Install sscache +ARG SCCACHE_VERSION="v0.7.5" +ENV SCCACHE_HOME="/opt/sccache" \ + SCCACHE_PATH="/opt/sccache/sccache" + +RUN mkdir ${SCCACHE_HOME} && cd ${SCCACHE_HOME} && \ + SCCACHE_ARCHIVE="sccache-${SCCACHE_VERSION}-x86_64-unknown-linux-musl.tar.gz" && \ + curl -SLO https://github.com/mozilla/sccache/releases/download/${SCCACHE_VERSION}/${SCCACHE_ARCHIVE} && \ + tar -xzf ${SCCACHE_ARCHIVE} --strip-components=1 && rm ${SCCACHE_ARCHIVE} + +ENV PATH="$SCCACHE_HOME:$PATH" diff --git a/.github/workflows/linux_sanitizers.yml b/.github/workflows/linux_sanitizers.yml index f9ec0bb7894818..93fab7d583199c 100644 --- a/.github/workflows/linux_sanitizers.yml +++ b/.github/workflows/linux_sanitizers.yml @@ -101,14 +101,13 @@ jobs: BUILD_DIR: /__w/openvino/openvino/openvino_build LSAN_IGNORE: /__w/openvino/openvino/openvino/tests/sanitizers/lsan/suppressions.txt ASAN_IGNORE: /__w/openvino/openvino/openvino/tests/sanitizers/asan/suppressions.supp - CMAKE_CXX_COMPILER_LAUNCHER: ccache - CMAKE_C_COMPILER_LAUNCHER: ccache - CCACHE_REMOTE_DIR: /mount/caches/ccache/sanitizers/${{ matrix.SANITIZER }}/${{ github.base_ref || github.ref_name }} - CCACHE_DIR: /__w/openvino/openvino/ccache - CCACHE_TEMPDIR: /__w/openvino/openvino/ccache_temp - CCACHE_MAXSIZE: 30G - CCACHE_LOGFILE: /__w/openvino/openvino/openvino_build/ccache.log - # CCACHE_DEBUG: 1 + CMAKE_CXX_COMPILER_LAUNCHER: sccache + CMAKE_C_COMPILER_LAUNCHER: sccache + SCCACHE_IGNORE_SERVER_IO_ERROR: 1 + SCCACHE_SERVER_PORT: 35555 + SCCACHE_ERROR_LOG: /__w/openvino/sccache_log.txt + SCCACHE_LOG: warn + SCCACHE_AZURE_KEY_PREFIX: ${{ runner.os }}_${{ runner.arch }}_${{ matrix.SANITIZER }}_master steps: - name: Clone OpenVINO @@ -131,31 +130,15 @@ jobs: - name: System info uses: ./openvino/.github/actions/system_info - - - name: Setup ccache - id: ccache_restore - uses: ./openvino/.github/actions/cache - with: - # save-always: ${{ github.event_name == 'push' && 'true' || 'false' }} - # cleanup-always: ${{ github.event_name == 'push' && 'true' || 'false' }} - save-always: true - cleanup-always: true - cache-size: 10 - max-cache-size: 50 - cache-path: ${{ env.CCACHE_REMOTE_DIR }} - path: ${{ env.CCACHE_DIR }} - key: ${{ runner.os }}-${{ runner.arch }}-ccache-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-${{ runner.arch }}-ccache - + # # Build # - name: Clean ccache stats - run: ccache --zero-stats + run: ${SCCACHE_PATH} --zero-stats - name: Show ccache stats - run: ccache --show-stats + run: ${SCCACHE_PATH} --show-stats - name: CMake configure - OpenVINO run: | @@ -189,15 +172,13 @@ jobs: LSAN_OPTIONS: suppressions=${{ env.LSAN_IGNORE }}:NEOReadDebugKeys=1:DisableDeepBind=1 - name: Show ccache stats - run: ccache --show-stats + run: ${SCCACHE_PATH} --show-stats - name: Cleanup ccache - run: ccache --cleanup --zero-stats --max-size=15G - env: - CCACHE_MAXSIZE: 15G + run: ${SCCACHE_PATH} --cleanup --zero-stats - name: Show ccache stats - run: ccache --show-stats + run: ${SCCACHE_PATH} --show-stats - name: Cmake install - OpenVINO run: | @@ -222,12 +203,12 @@ jobs: # # Upload build artifacts # - - name: Upload ccache log + - name: Upload sccache log if: ${{ always() }} uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0 with: - name: ccache_log_${{ matrix.SANITIZER }} - path: ${{ env.CCACHE_LOGFILE }} + name: sccache_log_${{ matrix.SANITIZER }} + path: ${{ env.SCCACHE_ERROR_LOG }} if-no-files-found: 'error' - name: Upload openvino package From cea5621fc38c994cb35f6e5fa382ff42e1643572 Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Mon, 30 Sep 2024 10:52:42 +0200 Subject: [PATCH 54/72] change sccache prefix --- .github/workflows/linux_sanitizers.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/linux_sanitizers.yml b/.github/workflows/linux_sanitizers.yml index 93fab7d583199c..3f61b1ab020d71 100644 --- a/.github/workflows/linux_sanitizers.yml +++ b/.github/workflows/linux_sanitizers.yml @@ -107,7 +107,7 @@ jobs: SCCACHE_SERVER_PORT: 35555 SCCACHE_ERROR_LOG: /__w/openvino/sccache_log.txt SCCACHE_LOG: warn - SCCACHE_AZURE_KEY_PREFIX: ${{ runner.os }}_${{ runner.arch }}_${{ matrix.SANITIZER }}_master + SCCACHE_AZURE_KEY_PREFIX: sanitizers_lin_${{ matrix.SANITIZER }}_master steps: - name: Clone OpenVINO From 04bdd25d36ee4bc8bf1eaa0dc96c9188f62080c6 Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Mon, 30 Sep 2024 16:56:35 +0200 Subject: [PATCH 55/72] sccache size 50G --- .github/workflows/linux_sanitizers.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/linux_sanitizers.yml b/.github/workflows/linux_sanitizers.yml index 3f61b1ab020d71..8f50d74652b893 100644 --- a/.github/workflows/linux_sanitizers.yml +++ b/.github/workflows/linux_sanitizers.yml @@ -108,6 +108,7 @@ jobs: SCCACHE_ERROR_LOG: /__w/openvino/sccache_log.txt SCCACHE_LOG: warn SCCACHE_AZURE_KEY_PREFIX: sanitizers_lin_${{ matrix.SANITIZER }}_master + SCCACHE_CACHE_SIZE: 50G steps: - name: Clone OpenVINO From b0e325a1b94d1cebbb2a1fe76de5a69c12314c6c Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Mon, 30 Sep 2024 17:07:14 +0200 Subject: [PATCH 56/72] disabled cleanup cache steps --- .github/workflows/linux_sanitizers.yml | 8 ++++---- cache | 1 + src/plugins/intel_npu/thirdparty/level-zero | 1 + 3 files changed, 6 insertions(+), 4 deletions(-) create mode 160000 cache create mode 160000 src/plugins/intel_npu/thirdparty/level-zero diff --git a/.github/workflows/linux_sanitizers.yml b/.github/workflows/linux_sanitizers.yml index 8f50d74652b893..0b1cafbe58a9fd 100644 --- a/.github/workflows/linux_sanitizers.yml +++ b/.github/workflows/linux_sanitizers.yml @@ -175,11 +175,11 @@ jobs: - name: Show ccache stats run: ${SCCACHE_PATH} --show-stats - - name: Cleanup ccache - run: ${SCCACHE_PATH} --cleanup --zero-stats + # - name: Cleanup ccache + # run: ${SCCACHE_PATH} --cleanup --zero-stats - - name: Show ccache stats - run: ${SCCACHE_PATH} --show-stats + # - name: Show ccache stats + # run: ${SCCACHE_PATH} --show-stats - name: Cmake install - OpenVINO run: | diff --git a/cache b/cache new file mode 160000 index 00000000000000..0c45773b623bea --- /dev/null +++ b/cache @@ -0,0 +1 @@ +Subproject commit 0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 diff --git a/src/plugins/intel_npu/thirdparty/level-zero b/src/plugins/intel_npu/thirdparty/level-zero new file mode 160000 index 00000000000000..fd459f28041c17 --- /dev/null +++ b/src/plugins/intel_npu/thirdparty/level-zero @@ -0,0 +1 @@ +Subproject commit fd459f28041c176df23a0b7b791ff20a7689d237 From 213d33496509ea8490a20553d49ec532d4f545b3 Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Mon, 30 Sep 2024 17:13:27 +0200 Subject: [PATCH 57/72] Revert "disabled cleanup cache steps" This reverts commit b0e325a1b94d1cebbb2a1fe76de5a69c12314c6c. --- cache | 1 - src/plugins/intel_npu/thirdparty/level-zero | 1 - 2 files changed, 2 deletions(-) delete mode 160000 cache delete mode 160000 src/plugins/intel_npu/thirdparty/level-zero diff --git a/cache b/cache deleted file mode 160000 index 0c45773b623bea..00000000000000 --- a/cache +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 diff --git a/src/plugins/intel_npu/thirdparty/level-zero b/src/plugins/intel_npu/thirdparty/level-zero deleted file mode 160000 index fd459f28041c17..00000000000000 --- a/src/plugins/intel_npu/thirdparty/level-zero +++ /dev/null @@ -1 +0,0 @@ -Subproject commit fd459f28041c176df23a0b7b791ff20a7689d237 From 3cb0d9acbec4d52031a612263d2245050b539d8b Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Mon, 30 Sep 2024 22:49:01 +0200 Subject: [PATCH 58/72] SCCACHE_AZURE_CONNECTION_STRING --- .github/workflows/linux_sanitizers.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/linux_sanitizers.yml b/.github/workflows/linux_sanitizers.yml index 0b1cafbe58a9fd..03ff8c503bbfe6 100644 --- a/.github/workflows/linux_sanitizers.yml +++ b/.github/workflows/linux_sanitizers.yml @@ -79,6 +79,7 @@ jobs: image: ${{ fromJSON(needs.docker.outputs.images).ov_build.sanitizers }} volumes: - /mount:/mount + options: -e SCCACHE_AZURE_BLOB_CONTAINER -e SCCACHE_AZURE_CONNECTION_STRING strategy: max-parallel: 3 fail-fast: false @@ -175,11 +176,11 @@ jobs: - name: Show ccache stats run: ${SCCACHE_PATH} --show-stats - # - name: Cleanup ccache - # run: ${SCCACHE_PATH} --cleanup --zero-stats + - name: Cleanup ccache + run: ${SCCACHE_PATH} --cleanup --zero-stats - # - name: Show ccache stats - # run: ${SCCACHE_PATH} --show-stats + - name: Show ccache stats + run: ${SCCACHE_PATH} --show-stats - name: Cmake install - OpenVINO run: | From d5186084b5455504acdb433704e51ce3b8d1c448 Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Tue, 1 Oct 2024 10:50:34 +0200 Subject: [PATCH 59/72] diadable cache cleanup --- .github/workflows/linux_sanitizers.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/linux_sanitizers.yml b/.github/workflows/linux_sanitizers.yml index 03ff8c503bbfe6..5cc16f334a7062 100644 --- a/.github/workflows/linux_sanitizers.yml +++ b/.github/workflows/linux_sanitizers.yml @@ -176,11 +176,11 @@ jobs: - name: Show ccache stats run: ${SCCACHE_PATH} --show-stats - - name: Cleanup ccache - run: ${SCCACHE_PATH} --cleanup --zero-stats + # - name: Cleanup ccache + # run: ${SCCACHE_PATH} --cleanup --zero-stats - - name: Show ccache stats - run: ${SCCACHE_PATH} --show-stats + # - name: Show ccache stats + # run: ${SCCACHE_PATH} --show-stats - name: Cmake install - OpenVINO run: | From 36aa03f28b147b1cd5fac69b72a2061f40c07dde Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Tue, 1 Oct 2024 11:58:35 +0200 Subject: [PATCH 60/72] test old tes image --- .github/workflows/linux_sanitizers.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/linux_sanitizers.yml b/.github/workflows/linux_sanitizers.yml index 5cc16f334a7062..88929f4f039cf8 100644 --- a/.github/workflows/linux_sanitizers.yml +++ b/.github/workflows/linux_sanitizers.yml @@ -236,7 +236,7 @@ jobs: timeout-minutes: 100 runs-on: 'aks-linux-16-cores-32gb' container: - image: ${{ fromJSON(needs.docker.outputs.images).ov_build.sanitizers }} + image: 'openvinogithubactions.azurecr.io/dockerhub/ubuntu:20.04' defaults: run: shell: bash From 4927057f9245fc448ab1abd7a2655142d07dea1a Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Tue, 1 Oct 2024 13:46:02 +0200 Subject: [PATCH 61/72] remove commented code --- .github/workflows/linux_sanitizers.yml | 9 --------- cache | 1 + src/plugins/intel_npu/thirdparty/level-zero | 1 + 3 files changed, 2 insertions(+), 9 deletions(-) create mode 160000 cache create mode 160000 src/plugins/intel_npu/thirdparty/level-zero diff --git a/.github/workflows/linux_sanitizers.yml b/.github/workflows/linux_sanitizers.yml index 88929f4f039cf8..0ba34fcd5b8db0 100644 --- a/.github/workflows/linux_sanitizers.yml +++ b/.github/workflows/linux_sanitizers.yml @@ -139,9 +139,6 @@ jobs: - name: Clean ccache stats run: ${SCCACHE_PATH} --zero-stats - - name: Show ccache stats - run: ${SCCACHE_PATH} --show-stats - - name: CMake configure - OpenVINO run: | cmake \ @@ -176,12 +173,6 @@ jobs: - name: Show ccache stats run: ${SCCACHE_PATH} --show-stats - # - name: Cleanup ccache - # run: ${SCCACHE_PATH} --cleanup --zero-stats - - # - name: Show ccache stats - # run: ${SCCACHE_PATH} --show-stats - - name: Cmake install - OpenVINO run: | cmake -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} -P ${BUILD_DIR}/cmake_install.cmake --config ${{ env.CMAKE_BUILD_TYPE }} diff --git a/cache b/cache new file mode 160000 index 00000000000000..0c45773b623bea --- /dev/null +++ b/cache @@ -0,0 +1 @@ +Subproject commit 0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 diff --git a/src/plugins/intel_npu/thirdparty/level-zero b/src/plugins/intel_npu/thirdparty/level-zero new file mode 160000 index 00000000000000..fd459f28041c17 --- /dev/null +++ b/src/plugins/intel_npu/thirdparty/level-zero @@ -0,0 +1 @@ +Subproject commit fd459f28041c176df23a0b7b791ff20a7689d237 From b33f739f6b942505de790b796e53112d962ec3ae Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Tue, 1 Oct 2024 13:53:08 +0200 Subject: [PATCH 62/72] split jobs --- .github/workflows/linux_sanitizers.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/linux_sanitizers.yml b/.github/workflows/linux_sanitizers.yml index 0ba34fcd5b8db0..4650ddef52fcc2 100644 --- a/.github/workflows/linux_sanitizers.yml +++ b/.github/workflows/linux_sanitizers.yml @@ -73,7 +73,7 @@ jobs: defaults: run: shell: bash - runs-on: aks-linux-16-cores-32gb + runs-on: ${{ matrix.RUNNER}} if: ${{ github.repository_owner == 'openvinotoolkit' }} container: image: ${{ fromJSON(needs.docker.outputs.images).ov_build.sanitizers }} @@ -87,10 +87,13 @@ jobs: include: - SANITIZER: 'AddressAndLeak' SANITIZER_CMAKE_OPTION: '-DENABLE_SANITIZER=ON' + RUNNER: aks-linux-16-cores-32gb - SANITIZER: 'UndefinedBehavior' SANITIZER_CMAKE_OPTION: '-DENABLE_UB_SANITIZER=ON' + RUNNER: aks-linux-16-cores-32gb - SANITIZER: 'Thread' SANITIZER_CMAKE_OPTION: '-DENABLE_THREAD_SANITIZER=ON' + RUNNER: aks-linux-16-cores-32gb env: CMAKE_BUILD_TYPE: 'RelWithDebInfo' CMAKE_GENERATOR: 'Ninja' @@ -225,7 +228,7 @@ jobs: if: always() needs: [Docker, Build] timeout-minutes: 100 - runs-on: 'aks-linux-16-cores-32gb' + runs-on: ${{ matrix.RUNNER}} container: image: 'openvinogithubactions.azurecr.io/dockerhub/ubuntu:20.04' defaults: @@ -237,8 +240,11 @@ jobs: matrix: include: - SANITIZER: 'AddressAndLeak' + RUNNER: aks-linux-16-cores-32gb - SANITIZER: 'UndefinedBehavior' + RUNNER: aks-linux-16-cores-32gb - SANITIZER: 'Thread' + RUNNER: aks-linux-16-cores-32gb env: INSTALL_DIR: ${{ github.workspace }}/install INSTALL_TEST_DIR: ${{ github.workspace }}/install/tests From 87b25aa03900b70f13fb23bf8fb85c197f910f05 Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Tue, 1 Oct 2024 13:53:12 +0200 Subject: [PATCH 63/72] Revert "test old tes image" This reverts commit 36aa03f28b147b1cd5fac69b72a2061f40c07dde. --- .github/workflows/linux_sanitizers.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/linux_sanitizers.yml b/.github/workflows/linux_sanitizers.yml index 4650ddef52fcc2..5ec99137d2a792 100644 --- a/.github/workflows/linux_sanitizers.yml +++ b/.github/workflows/linux_sanitizers.yml @@ -230,7 +230,7 @@ jobs: timeout-minutes: 100 runs-on: ${{ matrix.RUNNER}} container: - image: 'openvinogithubactions.azurecr.io/dockerhub/ubuntu:20.04' + image: ${{ fromJSON(needs.docker.outputs.images).ov_build.sanitizers }} defaults: run: shell: bash From 88c08a942903a5f711e8026eab0f9067b2db1913 Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Tue, 1 Oct 2024 14:00:08 +0200 Subject: [PATCH 64/72] Revert "remove commented code" This reverts commit 4927057f9245fc448ab1abd7a2655142d07dea1a. --- cache | 1 - src/plugins/intel_npu/thirdparty/level-zero | 1 - 2 files changed, 2 deletions(-) delete mode 160000 cache delete mode 160000 src/plugins/intel_npu/thirdparty/level-zero diff --git a/cache b/cache deleted file mode 160000 index 0c45773b623bea..00000000000000 --- a/cache +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 diff --git a/src/plugins/intel_npu/thirdparty/level-zero b/src/plugins/intel_npu/thirdparty/level-zero deleted file mode 160000 index fd459f28041c17..00000000000000 --- a/src/plugins/intel_npu/thirdparty/level-zero +++ /dev/null @@ -1 +0,0 @@ -Subproject commit fd459f28041c176df23a0b7b791ff20a7689d237 From c1e3895108ff0c12cf478978d73cf878de3e64be Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Wed, 2 Oct 2024 15:31:00 +0200 Subject: [PATCH 65/72] install python for the tests --- .../dockerfiles/ov_build/sanitizers/Dockerfile | 17 +++++++++++++++++ .github/workflows/linux_sanitizers.yml | 3 +++ 2 files changed, 20 insertions(+) diff --git a/.github/dockerfiles/ov_build/sanitizers/Dockerfile b/.github/dockerfiles/ov_build/sanitizers/Dockerfile index 83089a981b2022..de662ab48761de 100644 --- a/.github/dockerfiles/ov_build/sanitizers/Dockerfile +++ b/.github/dockerfiles/ov_build/sanitizers/Dockerfile @@ -22,6 +22,10 @@ RUN apt-get update && \ git \ gpg-agent \ tzdata \ + # Pythons + python3.9-dev \ + python3.9-venv \ + python3.9-distutils \ # Compiler \ gcc-10 \ g++-10 \ @@ -49,3 +53,16 @@ RUN mkdir ${SCCACHE_HOME} && cd ${SCCACHE_HOME} && \ tar -xzf ${SCCACHE_ARCHIVE} --strip-components=1 && rm ${SCCACHE_ARCHIVE} ENV PATH="$SCCACHE_HOME:$PATH" + +# Setup pip +ENV PIP_VERSION="24.0" +RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \ + python3 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \ + python3.9 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \ + rm -f get-pip.py + +# Use Python 3.9 as default instead of Python 3.8 +# Using venv here 'cause other methods to switch the default Python on Ubuntu 20 break both system and wheels build +RUN python3.9 -m venv venv +ENV PATH="/venv/bin:$PATH" +ENV PIP_CACHE_DIR=/mount/caches/pip/linux/${PIP_VERSION} \ No newline at end of file diff --git a/.github/workflows/linux_sanitizers.yml b/.github/workflows/linux_sanitizers.yml index 5ec99137d2a792..6b6e4ca6a3b7d0 100644 --- a/.github/workflows/linux_sanitizers.yml +++ b/.github/workflows/linux_sanitizers.yml @@ -181,6 +181,9 @@ jobs: cmake -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} -P ${BUILD_DIR}/cmake_install.cmake --config ${{ env.CMAKE_BUILD_TYPE }} cmake -DCMAKE_INSTALL_PREFIX=${INSTALL_TEST_DIR} -DCOMPONENT=tests -P ${BUILD_DIR}/cmake_install.cmake --config ${{ env.CMAKE_BUILD_TYPE }} cmake -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} -DCOMPONENT=python_wheels -P ${BUILD_DIR}/cmake_install.cmake --config ${{ env.CMAKE_BUILD_TYPE }} + + - name: Remove unused files to free space + run: rm -rf ${BUILD_DIR}/* # # Tests From d26658076bb2fe24dfc0c1798257617449548c01 Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Wed, 2 Oct 2024 16:15:05 +0200 Subject: [PATCH 66/72] fixed descriptions --- .github/workflows/linux_sanitizers.yml | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/.github/workflows/linux_sanitizers.yml b/.github/workflows/linux_sanitizers.yml index 6b6e4ca6a3b7d0..ad9d1a5b4770e6 100644 --- a/.github/workflows/linux_sanitizers.yml +++ b/.github/workflows/linux_sanitizers.yml @@ -73,7 +73,7 @@ jobs: defaults: run: shell: bash - runs-on: ${{ matrix.RUNNER}} + runs-on: aks-linux-16-cores-32gb if: ${{ github.repository_owner == 'openvinotoolkit' }} container: image: ${{ fromJSON(needs.docker.outputs.images).ov_build.sanitizers }} @@ -87,13 +87,10 @@ jobs: include: - SANITIZER: 'AddressAndLeak' SANITIZER_CMAKE_OPTION: '-DENABLE_SANITIZER=ON' - RUNNER: aks-linux-16-cores-32gb - SANITIZER: 'UndefinedBehavior' SANITIZER_CMAKE_OPTION: '-DENABLE_UB_SANITIZER=ON' - RUNNER: aks-linux-16-cores-32gb - SANITIZER: 'Thread' SANITIZER_CMAKE_OPTION: '-DENABLE_THREAD_SANITIZER=ON' - RUNNER: aks-linux-16-cores-32gb env: CMAKE_BUILD_TYPE: 'RelWithDebInfo' CMAKE_GENERATOR: 'Ninja' @@ -139,7 +136,7 @@ jobs: # # Build # - - name: Clean ccache stats + - name: Clean sccache stats run: ${SCCACHE_PATH} --zero-stats - name: CMake configure - OpenVINO @@ -173,7 +170,7 @@ jobs: ASAN_OPTIONS: halt_on_error=0:suppressions=${{ env.ASAN_IGNORE }} LSAN_OPTIONS: suppressions=${{ env.LSAN_IGNORE }}:NEOReadDebugKeys=1:DisableDeepBind=1 - - name: Show ccache stats + - name: Show sccache stats run: ${SCCACHE_PATH} --show-stats - name: Cmake install - OpenVINO @@ -231,7 +228,7 @@ jobs: if: always() needs: [Docker, Build] timeout-minutes: 100 - runs-on: ${{ matrix.RUNNER}} + runs-on: aks-linux-16-cores-32gb container: image: ${{ fromJSON(needs.docker.outputs.images).ov_build.sanitizers }} defaults: @@ -243,11 +240,8 @@ jobs: matrix: include: - SANITIZER: 'AddressAndLeak' - RUNNER: aks-linux-16-cores-32gb - SANITIZER: 'UndefinedBehavior' - RUNNER: aks-linux-16-cores-32gb - SANITIZER: 'Thread' - RUNNER: aks-linux-16-cores-32gb env: INSTALL_DIR: ${{ github.workspace }}/install INSTALL_TEST_DIR: ${{ github.workspace }}/install/tests From 8481b39b866ccbc8f3b4076e22cacbde0c0b8be6 Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Wed, 2 Oct 2024 16:47:03 +0200 Subject: [PATCH 67/72] parallel gzip --- .github/dockerfiles/ov_build/sanitizers/Dockerfile | 2 ++ .github/workflows/linux_sanitizers.yml | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/dockerfiles/ov_build/sanitizers/Dockerfile b/.github/dockerfiles/ov_build/sanitizers/Dockerfile index de662ab48761de..fc6cd3d8eb2049 100644 --- a/.github/dockerfiles/ov_build/sanitizers/Dockerfile +++ b/.github/dockerfiles/ov_build/sanitizers/Dockerfile @@ -22,6 +22,8 @@ RUN apt-get update && \ git \ gpg-agent \ tzdata \ + # parallel gzip + pigz \ # Pythons python3.9-dev \ python3.9-venv \ diff --git a/.github/workflows/linux_sanitizers.yml b/.github/workflows/linux_sanitizers.yml index ad9d1a5b4770e6..eab970dd028576 100644 --- a/.github/workflows/linux_sanitizers.yml +++ b/.github/workflows/linux_sanitizers.yml @@ -189,11 +189,11 @@ jobs: - name: Pack Artifacts run: | pushd ${INSTALL_DIR} - tar -czvf ${BUILD_DIR}/openvino_package.tar.gz * + tar -I pigz -czvf ${BUILD_DIR}/openvino_package.tar.gz * popd pushd ${INSTALL_TEST_DIR} - tar -czvf ${BUILD_DIR}/openvino_tests.tar.gz * + tar -I pigz -czvf ${BUILD_DIR}/openvino_tests.tar.gz * popd # From 74d0b67d0d72e3f5d3b2929f8b5cf55e37ea662b Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Wed, 2 Oct 2024 16:55:09 +0200 Subject: [PATCH 68/72] setup python --- .github/workflows/linux_sanitizers.yml | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/.github/workflows/linux_sanitizers.yml b/.github/workflows/linux_sanitizers.yml index eab970dd028576..c620ea70a73748 100644 --- a/.github/workflows/linux_sanitizers.yml +++ b/.github/workflows/linux_sanitizers.yml @@ -18,6 +18,8 @@ permissions: read-all env: TARGET_BRANCH: ${{ github.base_ref || github.event.merge_group.base_ref || github.ref }} + PIP_CACHE_PATH: /mount/caches/pip/linux + PYTHON_VERSION: '3.11' jobs: Smart_CI: @@ -126,13 +128,18 @@ jobs: submodules: 'true' ref: ${{ env.TARGET_BRANCH }} - # - # Print system info - # - - name: System info uses: ./openvino/.github/actions/system_info + - name: Setup Python ${{ env.PYTHON_VERSION }} + uses: ./openvino/.github/actions/setup_python + with: + version: ${{ env.PYTHON_VERSION }} + pip-cache-path: ${{ env.PIP_CACHE_PATH }} + should-setup-pip-paths: 'true' + self-hosted-runner: 'true' + show-cache-info: 'true' + # # Build # From 0cdfc56efa25d9cccfcf5196f5bdf2d2711491ac Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Wed, 2 Oct 2024 17:20:23 +0200 Subject: [PATCH 69/72] fixed tar command --- .github/workflows/linux_sanitizers.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/linux_sanitizers.yml b/.github/workflows/linux_sanitizers.yml index c620ea70a73748..22979fd15960a6 100644 --- a/.github/workflows/linux_sanitizers.yml +++ b/.github/workflows/linux_sanitizers.yml @@ -196,11 +196,11 @@ jobs: - name: Pack Artifacts run: | pushd ${INSTALL_DIR} - tar -I pigz -czvf ${BUILD_DIR}/openvino_package.tar.gz * + tar -I pigz -cvf ${BUILD_DIR}/openvino_package.tar.gz * popd pushd ${INSTALL_TEST_DIR} - tar -I pigz -czvf ${BUILD_DIR}/openvino_tests.tar.gz * + tar -I pigz -cvf ${BUILD_DIR}/openvino_tests.tar.gz * popd # From c7307d24bc482b641c5344ce63609650b6db8895 Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Wed, 2 Oct 2024 22:15:12 +0200 Subject: [PATCH 70/72] removed fsanitize-undefined-trap-on-error --- cmake/developer_package/compile_flags/sanitizer.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/developer_package/compile_flags/sanitizer.cmake b/cmake/developer_package/compile_flags/sanitizer.cmake index d8385f9ed2fa0b..5fc24c4f862239 100644 --- a/cmake/developer_package/compile_flags/sanitizer.cmake +++ b/cmake/developer_package/compile_flags/sanitizer.cmake @@ -63,7 +63,7 @@ if(ENABLE_UB_SANITIZER) # Mute -fsanitize=enum Load of a value of an enumerated type which is not in the range of representable values for that enumerated type. # Samples cases: # load of value 4294967295, which is not a valid value for type 'const (anonymous namespace)::onnx::Field' - set(SANITIZER_COMPILER_FLAGS "${SANITIZER_COMPILER_FLAGS} -fsanitize=undefined -fno-sanitize=null -fno-sanitize=alignment -fno-sanitize=bool -fno-sanitize=enum -fsanitize-undefined-trap-on-error") + set(SANITIZER_COMPILER_FLAGS "${SANITIZER_COMPILER_FLAGS} -fsanitize=undefined -fno-sanitize=null -fno-sanitize=alignment -fno-sanitize=bool -fno-sanitize=enum") if(OV_COMPILER_IS_CLANG) set(SANITIZER_COMPILER_FLAGS "${SANITIZER_COMPILER_FLAGS} -fno-sanitize=function") endif() From 0ba8bc3d9482c346bd6d585d53fa101eb003db01 Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Thu, 3 Oct 2024 00:34:02 +0200 Subject: [PATCH 71/72] revert test deps --- .github/workflows/linux_sanitizers.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/linux_sanitizers.yml b/.github/workflows/linux_sanitizers.yml index 22979fd15960a6..8b068d09d67e0f 100644 --- a/.github/workflows/linux_sanitizers.yml +++ b/.github/workflows/linux_sanitizers.yml @@ -139,7 +139,12 @@ jobs: should-setup-pip-paths: 'true' self-hosted-runner: 'true' show-cache-info: 'true' - + + - name: Install python dependencies + run: | + # For converting model_zoo models + python3 -m pip install --force-reinstall -r ${OPENVINO_REPO}/src/frontends/onnx/tests/requirements.txt + # # Build # From 0d63b9a7b0a771f077efcb911987f713000e637c Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Thu, 3 Oct 2024 09:24:48 +0200 Subject: [PATCH 72/72] increase runner ram --- .github/workflows/linux_sanitizers.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/linux_sanitizers.yml b/.github/workflows/linux_sanitizers.yml index 8b068d09d67e0f..a8a73956dad8b3 100644 --- a/.github/workflows/linux_sanitizers.yml +++ b/.github/workflows/linux_sanitizers.yml @@ -75,7 +75,7 @@ jobs: defaults: run: shell: bash - runs-on: aks-linux-16-cores-32gb + runs-on: aks-linux-16-cores-64gb if: ${{ github.repository_owner == 'openvinotoolkit' }} container: image: ${{ fromJSON(needs.docker.outputs.images).ov_build.sanitizers }} @@ -177,7 +177,7 @@ jobs: - name: Cmake build - OpenVINO run: | - cmake --build ${BUILD_DIR} --parallel --config ${{ env.CMAKE_BUILD_TYPE }} + cmake --build ${BUILD_DIR} --parallel $(nproc) --config ${{ env.CMAKE_BUILD_TYPE }} env: ASAN_OPTIONS: halt_on_error=0:suppressions=${{ env.ASAN_IGNORE }} LSAN_OPTIONS: suppressions=${{ env.LSAN_IGNORE }}:NEOReadDebugKeys=1:DisableDeepBind=1