From cc67c6c8191f5e2b5ce9b63e426ffbb6cbd03a81 Mon Sep 17 00:00:00 2001 From: Huy Do Date: Wed, 8 Jan 2025 16:03:15 -0800 Subject: [PATCH 1/8] Fix the use of PyTorch pin commit in MacOS --- .ci/scripts/setup-macos.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.ci/scripts/setup-macos.sh b/.ci/scripts/setup-macos.sh index b1a8ff14b5..e0fd36969b 100755 --- a/.ci/scripts/setup-macos.sh +++ b/.ci/scripts/setup-macos.sh @@ -131,5 +131,8 @@ if [[ -z "${GITHUB_RUNNER:-}" ]]; then fi print_cmake_info -install_executorch +# We use PyTorch build from source here instead of nightly. This allows CI to test against +# the pinned commit from PyTorch +install_executorch "use-pt-pinned-commit" build_executorch_runner "${BUILD_TOOL}" +do_not_use_nightly_on_ci From 07cedd72758d4ecd9192a05a851c192e0e2b6f4d Mon Sep 17 00:00:00 2001 From: Huy Do Date: Wed, 8 Jan 2025 16:06:57 -0800 Subject: [PATCH 2/8] Fix typo --- .ci/scripts/setup-macos.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/scripts/setup-macos.sh b/.ci/scripts/setup-macos.sh index e0fd36969b..f9e43c1c8b 100755 --- a/.ci/scripts/setup-macos.sh +++ b/.ci/scripts/setup-macos.sh @@ -131,7 +131,7 @@ if [[ -z "${GITHUB_RUNNER:-}" ]]; then fi print_cmake_info -# We use PyTorch build from source here instead of nightly. This allows CI to test against +# We build PyTorch from source here instead of using nightly. This allows CI to test against # the pinned commit from PyTorch install_executorch "use-pt-pinned-commit" build_executorch_runner "${BUILD_TOOL}" From c5bb091e0c167615a989e76c3ea1a58ea93f8cae Mon Sep 17 00:00:00 2001 From: Huy Do Date: Wed, 8 Jan 2025 16:14:13 -0800 Subject: [PATCH 3/8] Bring back install_pytorch_and_domains removed by https://github.com/pytorch/executorch/pull/4826 --- .ci/scripts/setup-macos.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.ci/scripts/setup-macos.sh b/.ci/scripts/setup-macos.sh index f9e43c1c8b..395f0c1767 100755 --- a/.ci/scripts/setup-macos.sh +++ b/.ci/scripts/setup-macos.sh @@ -131,6 +131,7 @@ if [[ -z "${GITHUB_RUNNER:-}" ]]; then fi print_cmake_info +install_pytorch_and_domains # We build PyTorch from source here instead of using nightly. This allows CI to test against # the pinned commit from PyTorch install_executorch "use-pt-pinned-commit" From e520bde1fac8b62f7241512a54b36fda6b49c56b Mon Sep 17 00:00:00 2001 From: Huy Do Date: Wed, 8 Jan 2025 16:28:58 -0800 Subject: [PATCH 4/8] Bring back more changes from https://github.com/pytorch/executorch/pull/4826 --- .ci/scripts/utils.sh | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/.ci/scripts/utils.sh b/.ci/scripts/utils.sh index aa8d78da4b..f5ca2b851c 100644 --- a/.ci/scripts/utils.sh +++ b/.ci/scripts/utils.sh @@ -40,6 +40,41 @@ install_pip_dependencies() { popd || return } +install_domains() { + echo "Install torchvision and torchaudio" + pip install --no-use-pep517 --user "git+https://github.com/pytorch/audio.git@${TORCHAUDIO_VERSION}" + pip install --no-use-pep517 --user "git+https://github.com/pytorch/vision.git@${TORCHVISION_VERSION}" +} + +install_pytorch_and_domains() { + pushd .ci/docker || return + TORCH_VERSION=$(cat ci_commit_pins/pytorch.txt) + popd || return + + git clone https://github.com/pytorch/pytorch.git + + # Fetch the target commit + pushd pytorch || return + git checkout "${TORCH_VERSION}" + git submodule update --init --recursive + + # Then build and install PyTorch + python setup.py bdist_wheel + pip install "$(echo dist/*.whl)" + + # Grab the pinned audio and vision commits from PyTorch + TORCHAUDIO_VERSION=$(cat .github/ci_commit_pins/audio.txt) + export TORCHAUDIO_VERSION + TORCHVISION_VERSION=$(cat .github/ci_commit_pins/vision.txt) + export TORCHVISION_VERSION + + install_domains + + popd || return + # Print sccache stats for debugging + sccache --show-stats || true +} + install_flatc_from_source() { # NB: This function could be used to install flatbuffer from source pushd third-party/flatbuffers || return From 16084bf3c0275871f2214810c94394477dcea01f Mon Sep 17 00:00:00 2001 From: Huy Do Date: Wed, 8 Jan 2025 17:47:01 -0800 Subject: [PATCH 5/8] Could it be this flag --- .ci/scripts/utils.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.ci/scripts/utils.sh b/.ci/scripts/utils.sh index f5ca2b851c..6364660c21 100644 --- a/.ci/scripts/utils.sh +++ b/.ci/scripts/utils.sh @@ -58,6 +58,7 @@ install_pytorch_and_domains() { git checkout "${TORCH_VERSION}" git submodule update --init --recursive + export _GLIBCXX_USE_CXX11_ABI=1 # Then build and install PyTorch python setup.py bdist_wheel pip install "$(echo dist/*.whl)" From 9bf460a7bd270801735155f3df83815008a30cb6 Mon Sep 17 00:00:00 2001 From: Huy Do Date: Wed, 8 Jan 2025 17:59:38 -0800 Subject: [PATCH 6/8] Try again --- .ci/scripts/utils.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.ci/scripts/utils.sh b/.ci/scripts/utils.sh index 6364660c21..9834fc7670 100644 --- a/.ci/scripts/utils.sh +++ b/.ci/scripts/utils.sh @@ -58,7 +58,10 @@ install_pytorch_and_domains() { git checkout "${TORCH_VERSION}" git submodule update --init --recursive - export _GLIBCXX_USE_CXX11_ABI=1 + export USE_PYTORCH_METAL_EXPORT=1 + export USE_COREML_DELEGATE=1 + export USE_DISTRIBUTED=0 + # Then build and install PyTorch python setup.py bdist_wheel pip install "$(echo dist/*.whl)" From 8a1c44d9ca8098105fc6ad5f328a919fa83eaa48 Mon Sep 17 00:00:00 2001 From: Huy Do Date: Wed, 8 Jan 2025 18:48:58 -0800 Subject: [PATCH 7/8] Try this flag then --- .ci/scripts/utils.sh | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.ci/scripts/utils.sh b/.ci/scripts/utils.sh index 9834fc7670..6364660c21 100644 --- a/.ci/scripts/utils.sh +++ b/.ci/scripts/utils.sh @@ -58,10 +58,7 @@ install_pytorch_and_domains() { git checkout "${TORCH_VERSION}" git submodule update --init --recursive - export USE_PYTORCH_METAL_EXPORT=1 - export USE_COREML_DELEGATE=1 - export USE_DISTRIBUTED=0 - + export _GLIBCXX_USE_CXX11_ABI=1 # Then build and install PyTorch python setup.py bdist_wheel pip install "$(echo dist/*.whl)" From 023381eb96a22b77183aabf82e6a35bbb7520fd6 Mon Sep 17 00:00:00 2001 From: Huy Do Date: Wed, 8 Jan 2025 19:24:18 -0800 Subject: [PATCH 8/8] Another attempt --- .ci/docker/conda-env-ci.txt | 2 ++ .ci/scripts/utils.sh | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.ci/docker/conda-env-ci.txt b/.ci/docker/conda-env-ci.txt index 9ebfe654da..8f2e65dae7 100644 --- a/.ci/docker/conda-env-ci.txt +++ b/.ci/docker/conda-env-ci.txt @@ -1,2 +1,4 @@ cmake=3.22.1 ninja=1.10.2 +libuv +pkg-config diff --git a/.ci/scripts/utils.sh b/.ci/scripts/utils.sh index 6364660c21..ebed4a3150 100644 --- a/.ci/scripts/utils.sh +++ b/.ci/scripts/utils.sh @@ -58,7 +58,7 @@ install_pytorch_and_domains() { git checkout "${TORCH_VERSION}" git submodule update --init --recursive - export _GLIBCXX_USE_CXX11_ABI=1 + export USE_DISTRIBUTED=1 # Then build and install PyTorch python setup.py bdist_wheel pip install "$(echo dist/*.whl)"