-
Notifications
You must be signed in to change notification settings - Fork 1
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Arm64 - Repo/Branch Support + Venv + Refactor (#12)
* CD (Daily release, release) and CI (build unit tests) are separated * New venv structure for multiple repo/branch support * OpenBLAS is temporarily disabled --------- Co-authored-by: mantaionut <ionut@janeasystems.com> Co-authored-by: Irem Yuksel <113098562+iremyux@users.noreply.github.com>
- Loading branch information
1 parent
0d0af71
commit e00400b
Showing
19 changed files
with
482 additions
and
173 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,31 @@ | ||
:: For now, APL is considered as already installed (script is not in use) | ||
|
||
@echo off | ||
echo Dependency APL installation is started. | ||
:: Pre check for downloads and dependencies folders | ||
if not exist "%DOWNLOADS_DIR%" mkdir %DOWNLOADS_DIR% | ||
if not exist "%DEPENDENCIES_DIR%" mkdir %DEPENDENCIES_DIR% | ||
|
||
set DOWNLOAD_URL="https://developer.arm.com/-/media/Files/downloads/hpc/arm-performance-libraries/24-04/windows/arm-performance-libraries_24.04_Windows.msi?rev=2b30cf856f7d4e6b968ce7f3698c1088&revision=2b30cf85-6f7d-4e6b-968c-e7f3698c1088" | ||
set TARGET_DIR=apl\armpl_24.04\bin | ||
set TARGET_FILE=armpl-info.exe | ||
|
||
|
||
:: Check if the file already exists in the destination directory | ||
:: TODO: smarter check mechanism can be used (e.g. call with error level) | ||
if not exist "%DEPENDENCIES_DIR%\%TARGET_DIR%\%TARGET_FILE%" ( | ||
echo Downloading arm-performance-libraries_24.04_Windows.msi... | ||
curl -L -o "%DOWNLOADS_DIR%\arm-performance-libraries_24.04_Windows.msi" %DOWNLOAD_URL% | ||
|
||
echo Installing APL 24.04... | ||
msiexec /i "%DOWNLOADS_DIR%\arm-performance-libraries_24.04_Windows.msi" /qn /norestart ACCEPT_EULA=1 INSTALLFOLDER="%DEPENDENCIES_DIR%\apl" | ||
|
||
echo Successfully installed: APL | ||
) else ( | ||
echo Skipped download and install: APL | ||
) | ||
|
||
echo "%DEPENDENCIES_DIR%\%TARGET_DIR%" >> %GITHUB_PATH% | ||
echo ARMPL_DIR=%INSTALL_DIR%\armpl_24.04 >> %GITHUB_ENV% | ||
|
||
echo Dependency APL installation is finished. |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,8 +1,7 @@ | ||
pip uninstall torch -y | ||
pip uninstall pytorch -y | ||
|
||
del %JOB_DIR%\pytorch\install\* /s /q /f | ||
|
||
cd %JOB_DIR%\pytorch | ||
python -m pip install --upgrade pip | ||
python -m pip install -r requirements.txt | ||
@echo off | ||
|
||
if exist [%PYTORCH_SOURCES_DIR%] ( | ||
cd %PYTORCH_SOURCES_DIR% | ||
git clean -fxd | ||
cd .. | ||
) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,34 @@ | ||
:: For now, sccache is considered as already installed (script is not in use) | ||
|
||
@echo off | ||
echo Dependency sccache installation is started. | ||
:: Pre check for downloads and dependencies folders | ||
if not exist "%DOWNLOADS_DIR%" mkdir %DOWNLOADS_DIR% | ||
if not exist "%DEPENDENCIES_DIR%" mkdir %DEPENDENCIES_DIR% | ||
|
||
set DOWNLOAD_URL="https://github.com/mozilla/sccache/releases/download/v0.8.1/sccache-v0.8.1-x86_64-pc-windows-msvc.zip" | ||
set TARGET_DIR=sccache | ||
set TARGET_FILE=sccache.exe | ||
|
||
:: Check if the file already exists in the destination directory | ||
:: TODO: smarter check mechanism can be used (e.g. call with error level) | ||
if not exist "%DEPENDENCIES_DIR%\%TARGET_DIR%\%TARGET_FILE%" ( | ||
echo Downloading sccache.zip... | ||
curl -L -o "%DOWNLOADS_DIR%\sccache.zip" %DOWNLOAD_URL% | ||
|
||
echo Extracting sccache.zip... | ||
tar xf "%DOWNLOADS_DIR%\sccache.zip" -C %DEPENDENCIES_DIR% | ||
cd %DEPENDENCIES_DIR% | ||
ren sccache-v0.8.1-x86_64-pc-windows-msvc sccache | ||
cd .. | ||
|
||
echo Successfully installed: sccache | ||
) else ( | ||
echo Skipped download and install: sccache | ||
) | ||
|
||
echo "%DEPENDENCIES_DIR%\%TARGET_DIR%" >> %GITHUB_PATH% | ||
echo SCCACHE_CACHE_SIZE=30G >> %GITHUB_ENV% | ||
|
||
echo Dependency sccache installation is finished. | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,14 +1,12 @@ | ||
python -m pip install --upgrade pip | ||
python -m pip install pytest pytest-shard pytest-rerunfailures pytest-flakefinder pytest-pytorch pytest-xdist expecttest hypothesis xdoctest xmlrunner | ||
|
||
mkdir %JOB_DIR%\tmp | ||
cd %JOB_DIR%\tmp | ||
:: change to source directory | ||
cd %PYTORCH_SOURCES_DIR% | ||
|
||
curl -L https://github.com/cgohlke/win_arm64.whl/releases/download/v2023.4.1/2023.4.1-experimental-cp311-win_arm64.whl.zip -o win_arm64.whl.zip | ||
powershell Expand-Archive win_arm64.whl.zip -DestinationPath . | ||
:: create virtual environment | ||
python -m venv venv | ||
echo * > venv\.gitignore | ||
call .\venv\Scripts\activate | ||
where python | ||
|
||
python -m pip uninstall numpy -y | ||
python -m pip install 2023.4.1-experimental-cp311-win_arm64.whl\numpy-1.24.2-cp311-cp311-win_arm64.whl | ||
python -m pip install 2023.4.1-experimental-cp311-win_arm64.whl\SciPy-1.10.1-cp311-cp311-win_arm64.whl | ||
|
||
del %JOB_DIR%\tmp /s /q /f | ||
:: install dependencies | ||
python -m pip install --upgrade pip | ||
pip install pytest==8.1.1 pytest-xdist==3.5.0 pytest-shard pytest-rerunfailures==13.0 pytest-flakefinder pytest-pytorch expecttest hypothesis xdoctest |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,40 +1,60 @@ | ||
@echo on | ||
:: start sccache server and reset sccache stats | ||
sccache --zero-stats | ||
|
||
:: environment variables | ||
:: TODO: OpenBLAS implementation will be applied | ||
set BLAS=APL | ||
set REL_WITH_DEB_INFO=1 | ||
set CMAKE_BUILD_TYPE=RelWithDebInfo | ||
set USE_LAPACK=1 | ||
set USE_NUMPY=OFF | ||
set CMAKE_C_COMPILER_LAUNCHER=sccache | ||
set CMAKE_CXX_COMPILER_LAUNCHER=sccache | ||
|
||
:: TODO: Dependencies | ||
set PATH=%DEPENDENCY_SCCACHE_DIR%;%PATH% | ||
set PATH=%DEPENDENCY_APL_DIR%;%PATH% | ||
|
||
:: change to source directory | ||
cd %PYTORCH_SOURCES_DIR% | ||
|
||
:: create virtual environment | ||
python -m venv venv | ||
echo * > venv\.gitignore | ||
call .\venv\Scripts\activate | ||
where python | ||
|
||
:: python install dependencies | ||
python -m pip install --upgrade pip | ||
pip install -r requirements.txt | ||
|
||
:: activate visual studio | ||
set "vswhere=%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" | ||
set visualstudio= | ||
for /f "delims=" %%v in ('"%vswhere%" -latest -property installationPath') do set "visualstudio=%%v" | ||
for /f "delims=" %%v in ('"%vswhere%" -products * -latest -property installationPath') do set "visualstudio=%%v" | ||
if "%visualstudio%" == "" ( | ||
echo Visual Studio not found! | ||
goto :eof | ||
) | ||
|
||
call "%visualstudio%\VC\Auxiliary\Build\vcvarsall.bat" arm64 | ||
:: TODO: vcvars_ver is optional but currently it is being tested with 14.36 and 14.40 | ||
call "%visualstudio%\VC\Auxiliary\Build\vcvarsall.bat" arm64 -vcvars_ver=14.40 | ||
|
||
cd %JOB_DIR%\pytorch | ||
:: show sccache stats | ||
sccache --show-stats | ||
|
||
set REL_WITH_DEB_INFO=1 | ||
set CMAKE_BUILD_TYPE=RelWithDebInfo | ||
if defined ENABLE_BUILD_WHEEL ( | ||
echo ENABLE_BUILD_WHEEL is set, wheel is generating... | ||
|
||
echo %ENABLE_OPENBLAS% | ||
python setup.py bdist_wheel | ||
|
||
if "%ENABLE_OPENBLAS%" == "1" ( | ||
set BLAS=OpenBLAS | ||
set OpenBLAS_HOME=%JOB_DIR%\openblas\install | ||
) | ||
:: rename wheel file | ||
cd %WHEEL_DIR% | ||
ren "torc*" %WHEEL_NAME% | ||
cd .. | ||
) else ( | ||
echo ENABLE_BUILD_WHEEL is not set, install command will be executed... | ||
|
||
set USE_CUDA=OFF | ||
set USE_DISTRIBUTED=OFF | ||
set USE_FAKELOWP=OFF | ||
set USE_FBGEMM=OFF | ||
set USE_MAGMA=OFF | ||
set USE_METAL=OFF | ||
set USE_MKLDNN=OFF | ||
set USE_NNPACK=OFF | ||
set USE_PYTORCH_QNNPACK=OFF | ||
set USE_QNNPACK=OFF | ||
set USE_ROCM=OFF | ||
set USE_TENSORPIPE=OFF | ||
set USE_XNNPACK=OFF | ||
|
||
python setup.py install --cmake --home=%JOB_DIR%\pytorch\install | ||
python setup.py install | ||
python setup.py install | ||
) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,3 +1,13 @@ | ||
cd %JOB_DIR%\pytorch\test | ||
:: change to source directory | ||
cd %PYTORCH_SOURCES_DIR% | ||
|
||
:: create virtual environment | ||
python -m venv venv | ||
echo * > venv\.gitignore | ||
call .\venv\Scripts\activate | ||
where python | ||
|
||
:: change to test directory | ||
cd test | ||
|
||
python run_test.py --core --verbose --save-xml --keep-going %* |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,4 +1,4 @@ | ||
@echo on | ||
@echo off | ||
|
||
if exist [%OPENBLAS_SOURCES_DIR%] ( | ||
rmdir /s /q %OPENBLAS_SOURCES_DIR% | ||
|
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,19 @@ | ||
name: Release | ||
|
||
# Controls when the workflow will run | ||
# OpenBLAS is disabled for now | ||
# Schedule doesn't accept inputs. So, this has hardcoded values for repository / branch | ||
on: | ||
schedule: | ||
- cron: "7 0 * * *" | ||
|
||
jobs: | ||
create_release: | ||
name: Create Release | ||
uses: ./.github/workflows/release.yml | ||
with: | ||
blass_type: 'apl' | ||
pytorch_repository: 'iremyux/pytorch' | ||
pytorch_branch: 'apl-tests' | ||
wheel_name: 'torch-2.4.0a-cp311-cp311-win_arm64.whl' | ||
release_prefix: 'Daily Release' |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,30 @@ | ||
# Daily Sync workflow draft for PyTorch's daily code. | ||
# Workflow file is created with dummy values for main branch (so we can use it with different branch while working) | ||
|
||
name: Daily Sync | ||
|
||
# Controls when the workflow will run | ||
on: | ||
workflow_dispatch: | ||
inputs: | ||
run_tests: | ||
description: "Dummy value" | ||
required: false | ||
default: false | ||
type: boolean | ||
|
||
# A workflow run is made up of one or more jobs that can run sequentially or in parallel | ||
jobs: | ||
# This workflow contains a single job called "build" | ||
build: | ||
# The type of runner that the job will run on | ||
runs-on: [self-hosted, Windows, ARM64, PYTORCH] | ||
|
||
# Steps represent a sequence of tasks that will be executed as part of the job | ||
steps: | ||
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it | ||
- uses: actions/checkout@v4 | ||
|
||
# Runs a single command using the runners shell | ||
- name: Run a one-line script | ||
run: echo Hello, world! |
Oops, something went wrong.