Skip to content

Commit

Permalink
Merge branch 'master' into xp/decompse_matmul_split_or_matmul_gather
Browse files Browse the repository at this point in the history
  • Loading branch information
xipingyan authored Aug 26, 2024
2 parents 621a4dc + db44fb6 commit 185b4ce
Show file tree
Hide file tree
Showing 748 changed files with 21,160 additions and 11,860 deletions.
2 changes: 1 addition & 1 deletion .github/dockerfiles/docker_tag
Original file line number Diff line number Diff line change
@@ -1 +1 @@
pr-25992
pr-26067
95 changes: 95 additions & 0 deletions .github/dockerfiles/ov_build/ubuntu_22_04_x64_dpcpp/Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,95 @@
FROM openvinogithubactions.azurecr.io/dockerhub/ubuntu:22.04

USER root

# APT configuration
RUN echo 'Acquire::Retries "10";' > /etc/apt/apt.conf && \
echo 'APT::Get::Assume-Yes "true";' >> /etc/apt/apt.conf && \
echo 'APT::Get::Fix-Broken "true";' >> /etc/apt/apt.conf && \
echo 'APT::Get::no-install-recommends "true";' >> /etc/apt/apt.conf

ENV DEBIAN_FRONTEND="noninteractive" \
TZ="Europe/London"

RUN apt-get update && \
apt-get install software-properties-common && \
add-apt-repository --yes --no-update ppa:git-core/ppa && \
add-apt-repository --yes --no-update ppa:deadsnakes/ppa && \
apt-get update && \
apt-get install \
wget \
curl \
git \
ca-certificates \
gpg-agent \
tzdata \
libtbb2 \
# Pythons \
python3.11-dev \
python3.11-venv \
python3.11-distutils \
default-jdk \
&& \
rm -rf /var/lib/apt/lists/*


# Install OneAPI Toolkit
RUN wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB | gpg --dearmor | tee /usr/share/keyrings/intel-oneapi-archive-keyring.gpg > /dev/null
RUN echo "deb [signed-by=/usr/share/keyrings/intel-oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main " > /etc/apt/sources.list.d/oneAPI.list

RUN apt-get update && \
apt-get install \
intel-oneapi-compiler-dpcpp-cpp=2024.2.1-1079 \
&& \
rm -rf /var/lib/apt/lists/*

# Install build dependencies
ADD install_build_dependencies.sh /install_build_dependencies.sh
RUN chmod +x /install_build_dependencies.sh && \
/install_build_dependencies.sh && \
rm -rf /var/lib/apt/lists/*

# Install sscache
ARG SCCACHE_VERSION="v0.7.5"
ENV SCCACHE_HOME="/opt/sccache" \
SCCACHE_PATH="/opt/sccache/sccache"

RUN mkdir ${SCCACHE_HOME} && cd ${SCCACHE_HOME} && \
SCCACHE_ARCHIVE="sccache-${SCCACHE_VERSION}-x86_64-unknown-linux-musl.tar.gz" && \
curl -SLO https://github.com/mozilla/sccache/releases/download/${SCCACHE_VERSION}/${SCCACHE_ARCHIVE} && \
tar -xzf ${SCCACHE_ARCHIVE} --strip-components=1 && rm ${SCCACHE_ARCHIVE}

# Setup pip
ENV PIP_VERSION="24.0"
RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \
python3 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \
python3.11 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \
rm -f get-pip.py

# Use Python 3.11 as default
# Using venv here 'cause other methods to switch the default Python on Ubuntu break both system and wheels build
RUN python3.11 -m venv venv
ENV PATH="/venv/bin:$SCCACHE_HOME:$PATH"

ENV PIP_CACHE_DIR=/mount/caches/pip/linux/${PIP_VERSION}

# OneAPI env
ENV ONEAPI_ROOT=/opt/intel/oneapi
ENV PKG_CONFIG_PATH=/opt/intel/oneapi/compiler/2024.2/lib/pkgconfig
ENV DIAGUTIL_PATH=/opt/intel/oneapi/dpcpp-ct/2024.2/etc/dpct/sys_check/sys_check.sh:/opt/intel/oneapi/debugger/2024.2/etc/debugger/sys_check/sys_check.py:/opt/intel/oneapi/compiler/2024.2/etc/compiler/sys_check/sys_check.sh
ENV MANPATH=/opt/intel/oneapi/debugger/2024.2/share/man:/opt/intel/oneapi/compiler/2024.2/share/man:
ENV GDB_INFO=/opt/intel/oneapi/debugger/2024.2/share/info/
ENV CMAKE_PREFIX_PATH=/opt/intel/oneapi/compiler/2024.2
ENV CMPLR_ROOT=/opt/intel/oneapi/compiler/2024.2
ENV INFOPATH=/opt/intel/oneapi/debugger/2024.2/share/info
ENV LIBRARY_PATH=/opt/intel/oneapi/compiler/2024.2/lib
ENV OCL_ICD_FILENAMES=/opt/intel/oneapi/compiler/2024.2/lib/libintelocl.so
ENV LD_LIBRARY_PATH=/opt/intel/oneapi/debugger/2024.2/opt/debugger/lib:/opt/intel/oneapi/compiler/2024.2/opt/compiler/lib:/opt/intel/oneapi/compiler/2024.2/lib
ENV NLSPATH=/opt/intel/oneapi/mkl/2024.2/share/locale/%l_%t/%N:/opt/intel/oneapi/compiler/2024.2/lib/compiler/locale/%l_%t/%N
ENV PATH=$PATH:/opt/intel/oneapi/dev-utilities/2024.2/bin:/opt/intel/oneapi/debugger/2024.2/opt/debugger/bin:/opt/intel/oneapi/compiler/2024.2/bin
ENV INTEL_PYTHONHOME=/opt/intel/oneapi/debugger/2024.2/opt/debugger
ENV CPATH=/opt/intel/oneapi/dpl/2022.6/include:/opt/intel/oneapi/dev-utilities/2024.2/include:

# Set Intel DPC++ as a default compiler
ENV CC=icx
ENV CXX=icpx
2 changes: 2 additions & 0 deletions .github/workflows/dev_cpu_linux_snippets_libxsmm.yml
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@ on:
- 'src/plugins/intel_cpu/src/emitters/tpp/**'
- 'src/plugins/intel_cpu/src/transformations/snippets/**'
- 'src/plugins/intel_cpu/src/transformations/tpp/**'
- 'src/plugins/intel_cpu/tests/unit/snippets_transformations/**'
- 'src/plugins/intel_cpu/tests/functional/shared_tests_instances/snippets/**'

concurrency:
group: ${{ github.event_name == 'push' && github.run_id || github.ref }}-linux-cpu-dev
Expand Down
2 changes: 2 additions & 0 deletions .github/workflows/fedora.yml
Original file line number Diff line number Diff line change
Expand Up @@ -221,6 +221,7 @@ jobs:
if-no-files-found: 'error'

RPM_Packages:
name: RPM packages
needs: [Docker, Build]
timeout-minutes: 10
defaults:
Expand Down Expand Up @@ -273,6 +274,7 @@ jobs:
python3 -c 'from openvino import Core; Core().get_property("BATCH", "SUPPORTED_PROPERTIES")'
python3 -c 'from openvino.frontend import FrontEndManager; assert len(FrontEndManager().get_available_front_ends()) == 7'
benchmark_app --help
opt_in_out --help
ovc --help
Overall_Status:
Expand Down
10 changes: 2 additions & 8 deletions .github/workflows/job_build_linux.yml
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ jobs:

# Ticket: 139627
- name: Checkout the latest OneDNN for GPU in nightly
if: ${{ inputs.event-name == 'schedule' }}
if: ${{ inputs.event-name == 'schedule' && inputs.os == 'ubuntu_20_04' }} # GPU tests are enabled only on U20
working-directory: ${{ env.OPENVINO_REPO }}/src/plugins/intel_gpu/thirdparty/onednn_gpu
run: |
git fetch origin
Expand Down Expand Up @@ -113,13 +113,7 @@ jobs:
python3 -m pip install -r ${OPENVINO_REPO}/src/bindings/python/wheel/requirements-dev.txt
# For running ONNX frontend unit tests
if [[ ${{ inputs.os }} == 'ubuntu_24_04' ]]; then
# Should be removed after https://github.com/openvinotoolkit/openvino/pull/24242 is merged
export CMAKE_GENERATOR="Unix Makefiles"
python3 -m pip install --force-reinstall --no-cache-dir -r ${OPENVINO_REPO}/src/frontends/onnx/tests/requirements.txt
else
python3 -m pip install --force-reinstall -r ${OPENVINO_REPO}/src/frontends/onnx/tests/requirements.txt
fi
python3 -m pip install -r ${OPENVINO_REPO}/src/frontends/onnx/tests/requirements.txt
# For running TensorFlow frontend unit tests
python3 -m pip install -r ${OPENVINO_REPO}/src/frontends/tensorflow/tests/requirements.txt
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/job_onnx_models_tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,7 @@ jobs:
- name: Install Python tests dependencies
run: |
# To enable pytest parallel features
python3 -m pip install pytest-xdist[psutil] pytest-forked
python3 -m pip install pytest-xdist[psutil] pytest-forked pytest-randomly
- name: ONNX Models Tests
run: python3 -m pytest --backend="CPU" --model_zoo_dir="${MODELS_SHARE_PATH}" ${INSTALL_TEST_DIR}/onnx/tests/tests_python/test_zoo_models.py -v -n auto --forked -k 'not _cuda' --model_zoo_xfail
9 changes: 9 additions & 0 deletions .github/workflows/job_pytorch_models_tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -160,6 +160,15 @@ jobs:
TEST_DEVICE: CPU
USE_SYSTEM_CACHE: False

- name: RoPE Test
if: ${{ inputs.model_scope == 'precommit' }}
run: |
export PYTHONPATH=${MODEL_HUB_TESTS_INSTALL_DIR}:$PYTHONPATH
python3 -m pytest ${MODEL_HUB_TESTS_INSTALL_DIR}/transformation_tests/test_transformations.py -m precommit --html=${INSTALL_TEST_DIR}/TEST-torch_rope_tests.html --self-contained-html -v --tb=short -n 2
env:
TEST_DEVICE: CPU
USE_SYSTEM_CACHE: False

- name: StatefulToStateless Test
if: ${{ inputs.model_scope == 'precommit' }}
run: |
Expand Down
3 changes: 2 additions & 1 deletion .github/workflows/linux_conditional_compilation.yml
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,7 @@ jobs:
BUILD_DIR: /__w/openvino/openvino/openvino_build
SELECTIVE_BUILD_STAT_DIR: /__w/openvino/openvino/selective_build_stat
MODELS_PATH: /__w/openvino/openvino/testdata
SCCACHE_AZURE_KEY_PREFIX: ubuntu22_x86_64_itt_clang_Release
SCCACHE_AZURE_KEY_PREFIX: ubuntu22_x86_64_itt_clang_Release_faster_build
if: ${{ !needs.smart_ci.outputs.skip_workflow && github.event_name != 'merge_group' }}

steps:
Expand Down Expand Up @@ -157,6 +157,7 @@ jobs:
-DCMAKE_COMPILE_WARNING_AS_ERROR=ON \
-DENABLE_PROFILING_ITT=ON \
-DSELECTIVE_BUILD=COLLECT \
-DENABLE_FASTER_BUILD=ON \
-DENABLE_DEBUG_CAPS=ON \
-DCMAKE_C_COMPILER_LAUNCHER=${{ env.CMAKE_C_COMPILER_LAUNCHER }} \
-DCMAKE_CXX_COMPILER_LAUNCHER=${{ env.CMAKE_CXX_COMPILER_LAUNCHER }} \
Expand Down
90 changes: 90 additions & 0 deletions .github/workflows/ubuntu_22_dpcpp.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
name: Linux (Ubuntu 22.04, Python 3.11, Intel DPC++ Compiler)
on:
workflow_dispatch:
pull_request:
merge_group:

concurrency:
# github.ref is not unique in post-commit
group: ${{ github.event_name == 'push' && github.run_id || github.ref }}-ubuntu-22-dpcpp
cancel-in-progress: true

permissions: read-all

jobs:
Smart_CI:
runs-on: ubuntu-latest
outputs:
affected_components: "${{ steps.smart_ci.outputs.affected_components }}"
changed_components: "${{ steps.smart_ci.outputs.changed_components }}"
skip_workflow: "${{ steps.smart_ci.outputs.skip_workflow }}"
steps:
- name: checkout action
uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
with:
sparse-checkout: .github/actions/smart-ci

- name: Get affected components
id: smart_ci
uses: ./.github/actions/smart-ci
with:
repository: ${{ github.repository }}
pr: ${{ github.event.number }}
commit_sha: ${{ github.sha }}
ref_name: ${{ github.ref_name }}
component_pattern: "category: (.*)"
repo_token: ${{ secrets.GITHUB_TOKEN }}
skip_when_only_listed_labels_set: 'docs'
skip_when_only_listed_files_changed: '*.md,*.rst,*.png,*.jpg,*.svg'

- name: Show affected components
run: |
echo "${{ toJSON(steps.smart_ci.outputs.affected_components) }}"
shell: bash

Docker:
needs: Smart_CI
runs-on: aks-linux-4-cores-16gb-docker-build
container:
image: openvinogithubactions.azurecr.io/docker_build:0.2
volumes:
- /mount:/mount
outputs:
images: "${{ steps.handle_docker.outputs.images }}"
steps:
- name: Checkout
uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7

- uses: ./.github/actions/handle_docker
id: handle_docker
with:
images: |
ov_build/ubuntu_22_04_x64_dpcpp
registry: 'openvinogithubactions.azurecr.io'
dockerfiles_root_dir: '.github/dockerfiles'
changed_components: ${{ needs.smart_ci.outputs.changed_components }}

Build:
needs: [Docker, Smart_CI]
if: "!needs.smart_ci.outputs.skip_workflow"
uses: ./.github/workflows/job_build_linux.yml
with:
runner: 'aks-linux-16-cores-32gb'
container: '{"image": "${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_22_04_x64_dpcpp }}", "volumes": ["/mount:/mount"], "options": "-e SCCACHE_AZURE_BLOB_CONTAINER -e SCCACHE_AZURE_CONNECTION_STRING"}'
affected-components: ${{ needs.smart_ci.outputs.affected_components }}
event-name: ${{ github.event_name }}
os: 'ubuntu_22_04_dpcpp'

Overall_Status:
name: ci/gha_overall_status_ubuntu_22.04_dpcpp
needs: [Smart_CI, Build]
if: ${{ always() }}
runs-on: ubuntu-latest
steps:
- name: Check status of all jobs
if: >-
${{
contains(needs.*.result, 'failure') ||
contains(needs.*.result, 'cancelled')
}}
run: exit 1
1 change: 1 addition & 0 deletions .github/workflows/windows_conditional_compilation.yml
Original file line number Diff line number Diff line change
Expand Up @@ -169,6 +169,7 @@ jobs:
-DCMAKE_COMPILE_WARNING_AS_ERROR=ON `
-DENABLE_PROFILING_ITT=ON `
-DSELECTIVE_BUILD=COLLECT `
-DENABLE_FASTER_BUILD=ON `
-DCMAKE_DISABLE_FIND_PACKAGE_PkgConfig=ON `
-DCMAKE_TOOLCHAIN_FILE=${{ env.ONECORE_TOOLCHAIN }} `
-S ${{ env.OPENVINO_REPO }} `
Expand Down
Loading

0 comments on commit 185b4ce

Please sign in to comment.