From 5b5a4070c9723edece1980e97c03a91b908738a2 Mon Sep 17 00:00:00 2001 From: "Kevin R. Thornton" Date: Sun, 1 Oct 2023 13:15:16 -0700 Subject: [PATCH] Rewrite wheel building work flow. (#1196) * simpify scripts in deployment/linux_wheels/ * use new scripts in CI * validate sdist separately in CI * enable parallel CI jobs * update docs Closes #1192 --- .github/workflows/manylinux/buildwheels.sh | 52 -------------- .github/workflows/wheels.yml | 62 ++++++++++------ deployment/linux_wheels/Dockerfile | 29 -------- deployment/linux_wheels/build_and_audit.sh | 18 ----- deployment/linux_wheels/build_docker_image.sh | 1 - deployment/linux_wheels/extract_wheels.sh | 5 -- deployment/linux_wheels/install_gsl.sh | 9 --- .../linux_wheels/install_wheels_run_tests.sh | 26 ------- deployment/linux_wheels/run_wheel_workflow.sh | 3 + deployment/linux_wheels/wheel_workflow.sh | 70 +++++++++++++++++++ doc/pages/deploymenttools.md | 15 ++-- 11 files changed, 125 insertions(+), 165 deletions(-) delete mode 100644 .github/workflows/manylinux/buildwheels.sh delete mode 100644 deployment/linux_wheels/Dockerfile delete mode 100644 deployment/linux_wheels/build_and_audit.sh delete mode 100644 deployment/linux_wheels/build_docker_image.sh delete mode 100644 deployment/linux_wheels/extract_wheels.sh delete mode 100644 deployment/linux_wheels/install_gsl.sh delete mode 100644 deployment/linux_wheels/install_wheels_run_tests.sh create mode 100644 deployment/linux_wheels/run_wheel_workflow.sh create mode 100644 deployment/linux_wheels/wheel_workflow.sh diff --git a/.github/workflows/manylinux/buildwheels.sh b/.github/workflows/manylinux/buildwheels.sh deleted file mode 100644 index fbfe13e29..000000000 --- a/.github/workflows/manylinux/buildwheels.sh +++ /dev/null @@ -1,52 +0,0 @@ -#!/bin/bash - -set -e -x - -yum update -y -yum -y install cmake curl gsl-devel - -curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | bash -s -- -y -source "$HOME/.cargo/env" -# Pin the rustc toolchain to a specific version. -# Rust 1.64.0 will change the minimum glibc ABI -# to a version incompatibly with manylinux_2014, -# so we need to be careful in general. -rustup override set 1.62.1 -# Pin cbindgen -cargo install --locked cbindgen@0.24.3 - -# GSL_VERSION=2.5 -# curl -o gsl-${GSL_VERSION}.tar.gz "ftp://ftp.gnu.org/gnu/gsl/gsl-${GSL_VERSION}.tar.gz" -# tar -zxf gsl-${GSL_VERSION}.tar.gz -# cd gsl-${GSL_VERSION} -# ./configure --prefix=/usr/local -# make -j 2 -# make install -# cd .. - -# Taken from msprime/#2043 -# We're running as root in the docker container so git commands issued by -# setuptools_scm will fail without this: -git config --global --add safe.directory /project -# Fetch the full history as we'll be missing tags otherwise. -# git fetch --unshallow - -for py in cp38-cp38 cp39-cp39 cp310-cp310 cp311-cp311 -do - PYPATH=/opt/python/${py} - PYBIN=${PYPATH}/bin/python - rm -rf build/ - # Instead of letting setup.py install a newer numpy we install it here - # using the oldest supported version for ABI compatibility - ${PYBIN} -m pip install --no-cache oldest-supported-numpy build - PATH=${PYPATH}:$PATH ${PYBIN} -m build -done - -cd dist -for whl in *.whl; do - # Need to set this so that the core library - # can be found - LD_LIBRARY_PATH=fwdpy11 auditwheel repair "$whl" - rm "$whl" -done -cd .. diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index b5f9e1bc3..513f3bff8 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -10,6 +10,7 @@ on: jobs: macOS_wheel: + name: Build macOS/x86 wheels runs-on: macos-latest strategy: matrix: @@ -64,7 +65,8 @@ jobs: name: macOS-wheel-${{ matrix.python }} path: dist - manylinux2_28: + build_sdist: + name: Build source distribution runs-on: ubuntu-latest steps: - name: Cancel Previous Runs @@ -95,10 +97,29 @@ jobs: name: sdist path: dist + manylinux2_28: + name: Build and test Linux wheels + runs-on: ubuntu-latest + strategy: + matrix: + python: ["python3.8", "python3.9", "python3.10", "python3.11"] + steps: + - name: Cancel Previous Runs + uses: styfle/cancel-workflow-action@0.11.0 + with: + access_token: ${{ secrets.GITHUB_TOKEN }} + + - name: Checkout + uses: actions/checkout@v3 + with: + submodules: true + fetch-depth: 0 + - name: Build wheels in docker shell: bash run: | - docker run --rm -v `pwd`:/project -w /project quay.io/pypa/manylinux_2_28_x86_64:2022-10-02-69a0972 bash .github/workflows/manylinux/buildwheels.sh + bash deployment/linux_wheels/run_wheel_workflow.sh ${{ matrix.python }} + #docker run --rm -v `pwd`:/project -w /project quay.io/pypa/manylinux_2_28_x86_64:2022-10-02-69a0972 bash .github/workflows/manylinux/buildwheels.sh - name: Upload Wheels uses: actions/upload-artifact@v3 @@ -107,8 +128,9 @@ jobs: path: dist/wheelhouse manylinux2_28_test: + name: Build package from source dist runs-on: ubuntu-latest - needs: ['manylinux2_28'] + needs: ['build_sdist'] strategy: matrix: python: [3.8, 3.9, "3.10", "3.11"] @@ -119,10 +141,6 @@ jobs: access_token: ${{ secrets.GITHUB_TOKEN }} - name: Checkout uses: actions/checkout@v3 - - name: Download wheels - uses: actions/download-artifact@v3.0.2 - with: - name: linux-wheels - name: Download sdist uses: actions/download-artifact@v3.0.2 with: @@ -144,21 +162,22 @@ jobs: deactivate rm -rf sdist_venv - - name: Install wheel and test - run: | - python -VV - # pip install minimal dependencies - pip install --upgrade pip - pip install wheel - pip install -r requirements/wheel_building_workflow.txt - # delete the source dir to prevent pip from mistaking it for - # the package - rm -rf fwdpy11 - # Install the local wheel - pip install fwdpy11 --no-deps --no-index --pre --only-binary fwdpy11 -f . - python -c "import fwdpy11;print(fwdpy11.__version__)" + # - name: Install wheel and test + # run: | + # python -VV + # # pip install minimal dependencies + # pip install --upgrade pip + # pip install wheel + # pip install -r requirements/wheel_building_workflow.txt + # # delete the source dir to prevent pip from mistaking it for + # # the package + # rm -rf fwdpy11 + # # Install the local wheel + # pip install fwdpy11 --no-deps --no-index --pre --only-binary fwdpy11 -f . + # python -c "import fwdpy11;print(fwdpy11.__version__)" macOS_test: + name: Test macOS/x86 wheels needs: ['macOS_wheel'] runs-on: macos-latest strategy: @@ -197,8 +216,9 @@ jobs: python -c "import fwdpy11;print(fwdpy11.__version__)" upload_to_PyPI: + name: Upload to PyPI runs-on: ubuntu-latest - needs: ['macOS_test', 'manylinux2_28_test'] + needs: ['macOS_test', 'manylinux2_28', 'manylinux2_28_test'] steps: - name: Cancel Previous Runs uses: styfle/cancel-workflow-action@0.11.0 diff --git a/deployment/linux_wheels/Dockerfile b/deployment/linux_wheels/Dockerfile deleted file mode 100644 index 910d15d91..000000000 --- a/deployment/linux_wheels/Dockerfile +++ /dev/null @@ -1,29 +0,0 @@ -# This file builds an image that does a complete -# test of building and installing wheels and -# then running the test suite for multiple -# Python versions -FROM quay.io/pypa/manylinux_2_28_x86_64:2022-10-02-69a0972 - -WORKDIR /app - -COPY . /app - -RUN yum update -y && yum install -y cmake curl gsl-devel \ - && curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | bash -s -- -y \ - && source "$HOME/.cargo/env" \ - # Pin the rustc toolchain to a specific version. - # Rust 1.64.0 will change the minimum glibc ABI - # to a version incompatibly with manylinux_2014, - # so we need to be careful in general. - && rustup override set 1.62.1 \ - # Pin cbindgen - && cargo install --locked cbindgen@0.24.3 \ - && rustc --version \ - && cbindgen --version \ - # The GSL version available from yum install is too old so we manually install. - # && bash deployment/linux_wheels/install_gsl.sh \ - && bash deployment/linux_wheels/build_and_audit.sh \ - && rm -rf build - -RUN bash deployment/linux_wheels/install_wheels_run_tests.sh - diff --git a/deployment/linux_wheels/build_and_audit.sh b/deployment/linux_wheels/build_and_audit.sh deleted file mode 100644 index a4ad46bda..000000000 --- a/deployment/linux_wheels/build_and_audit.sh +++ /dev/null @@ -1,18 +0,0 @@ -rm -rf dist/ - -for py in cp38-cp38 cp39-cp39 cp310-cp310 cp311-cp311 -do - rm -rf build - PYPATH=/opt/python/${py} - PYBIN=${PYPATH}/bin/python - ${PYBIN} -m pip install --no-cache-dir -r requirements/development.txt - PATH=${PYPATH}:$PATH ${PYBIN} setup.py build_ext -i - PATH=${PYPATH}:$PATH ${PYBIN} setup.py bdist_wheel -done - -cd dist -for whl in *.whl; do - LD_LIBRARY_PATH=fwdpy11 auditwheel -v repair "$whl" - rm "$whl" -done -cd .. diff --git a/deployment/linux_wheels/build_docker_image.sh b/deployment/linux_wheels/build_docker_image.sh deleted file mode 100644 index 291cdad06..000000000 --- a/deployment/linux_wheels/build_docker_image.sh +++ /dev/null @@ -1 +0,0 @@ -docker build --rm -t fwdpy11_build_wheels -f deployment/linux_wheels/Dockerfile . diff --git a/deployment/linux_wheels/extract_wheels.sh b/deployment/linux_wheels/extract_wheels.sh deleted file mode 100644 index a5cc44192..000000000 --- a/deployment/linux_wheels/extract_wheels.sh +++ /dev/null @@ -1,5 +0,0 @@ -docker create -ti --name extract_fwdpy11_wheels fwdpy11_build_wheels -docker start extract_fwdpy11_wheels -docker exec -ti extract_fwdpy11_wheels tar cvf wheels.tar dist -docker cp extract_fwdpy11_wheels:/app/wheels.tar . -docker rm -f extract_fwdpy11_wheels diff --git a/deployment/linux_wheels/install_gsl.sh b/deployment/linux_wheels/install_gsl.sh deleted file mode 100644 index 9512248ff..000000000 --- a/deployment/linux_wheels/install_gsl.sh +++ /dev/null @@ -1,9 +0,0 @@ -GSL_VERSION=2.5 -curl -o gsl-${GSL_VERSION}.tar.gz "ftp://ftp.gnu.org/gnu/gsl/gsl-${GSL_VERSION}.tar.gz" -tar -zxf gsl-${GSL_VERSION}.tar.gz -cd gsl-${GSL_VERSION} -./configure --prefix=/usr/local -make -j 2 -make install -cd .. - diff --git a/deployment/linux_wheels/install_wheels_run_tests.sh b/deployment/linux_wheels/install_wheels_run_tests.sh deleted file mode 100644 index 8bcfe6f60..000000000 --- a/deployment/linux_wheels/install_wheels_run_tests.sh +++ /dev/null @@ -1,26 +0,0 @@ -ls dist/wheelhouse -rm -rf test_wheels -# Have to do this from a separate dir -# so that the source repo doesn't interfere. -# The repo gets seen as the fwdpy11 package. -mkdir test_wheels -cd test_wheels -# Copy tests over b/c some of the test rely on paths -cp -r ../tests . -pwd -for py in cp38-cp38 cp39-cp39 cp310-cp310 cp311-cp311 -do - PYPATH=/opt/python/${py} - PYBIN=${PYPATH}/bin/python - ${PYBIN} -m venv test-${py} - source test-${py}/bin/activate - which pip - pip install --upgrade --no-cache-dir pip wheel - pip install fwdpy11 --pre --only-binary fwdpy11 -f ../dist/wheelhouse - pip install --no-cache-dir msprime pytest pytest-xdist - python -m pytest tests -n 4 - deactivate - rm -rf test-${py} -done -cd .. -rm -rf test_wheels diff --git a/deployment/linux_wheels/run_wheel_workflow.sh b/deployment/linux_wheels/run_wheel_workflow.sh new file mode 100644 index 000000000..d30a10a20 --- /dev/null +++ b/deployment/linux_wheels/run_wheel_workflow.sh @@ -0,0 +1,3 @@ +PYTHON=$1 + +docker run --rm -v `pwd`:/project -w /project quay.io/pypa/manylinux_2_28_x86_64:2022-10-02-69a0972 /bin/sh ./deployment/linux_wheels/wheel_workflow.sh $PYTHON diff --git a/deployment/linux_wheels/wheel_workflow.sh b/deployment/linux_wheels/wheel_workflow.sh new file mode 100644 index 000000000..4d5e8aa22 --- /dev/null +++ b/deployment/linux_wheels/wheel_workflow.sh @@ -0,0 +1,70 @@ +PYTHON=$1 + +# SETUP +set -e -x +rm -rf venv + +# INSTALL SYSTEM DEPENDENCIES + +yum -y install curl gsl-devel + +# INSTALL RUST + +curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | bash -s -- -y +source "$HOME/.cargo/env" +# Pin the rustc toolchain to a specific version. +# Rust 1.64.0 will change the minimum glibc ABI +# to a version incompatible with manylinux_2014, +# so we need to be careful in general. +rustup override set 1.62.1 +# Pin cbindgen +cargo install --locked cbindgen@0.24.3 + +# Taken from msprime/#2043 +# We're running as root in the docker container so git commands issued by +# setuptools_scm will fail without this: +git config --global --add safe.directory /project + +# BUILD WHEEL INSIDE A VENV + +$(which $PYTHON) -m venv venv +source venv/bin/activate +python -m pip install --upgrade pip setuptools build oldest-supported-numpy +python -m build . +deactivate +rm -rf build venv + +cd dist +for whl in *.whl; do + # Need to set this so that the core library + # can be found + LD_LIBRARY_PATH=fwdpy11 auditwheel repair "$whl" + rm "$whl" +done +cd .. + +# INSTALL WHEEL INTO VENV FOR TESTING + +$(which $PYTHON) -m venv venv +source venv/bin/activate +python -m pip install --upgrade pip +python -m pip install wheel +python -m pip install fwdpy11 --pre --no-cache-dir --only-binary fwdpy11 --find-links dist/wheelhouse +# cd to avoid having the fwdpy11/ directory get mistaken for a package +TESTDIR=wheel_tests_$PYTHON +if [ -e $TESTDIR ] +then + rm -rf $TESTDIR +fi +mkdir $TESTDIR +cd $TESTDIR +python -c "import fwdpy11;print(fwdpy11.__version__)" +python -m pip install pytest pytest-xdist hypothesis msprime +# Copy the tests b/c some of what they do depends on paths +# and will fail don't run them from right outside that directory. +cp -r ../tests . +python -m pytest tests -n 4 +rm -rf $TESTDIR +cd .. +deactivate +rm -rf venv diff --git a/doc/pages/deploymenttools.md b/doc/pages/deploymenttools.md index f0f7c369d..26155ead7 100644 --- a/doc/pages/deploymenttools.md +++ b/doc/pages/deploymenttools.md @@ -161,19 +161,26 @@ docker run --rm -v $HOME/tmp:/app IMAGE_NAME /bin/bash -c ". /venv/bin/activate; ## Linux wheel building The directory `deployment/linux_wheels` defines a `Docker` work flow to build binary wheels on Linux. -Although this work flow *could* be used to make wheels for each new release, its main purposes are for testing and easily getting development versions to collaborators. +These scripts are what we use to build wheels for each release. +They may also be used to make wheels for development versions on a local machine. -From the `root` of the `fwdpy11` repository: +From the `root` of the `fwdpy11` repository. ```sh -bash deployment/linux_wheels/build_docker_image.sh +bash deployment/linux_wheels/run_wheel_workflow.sh VERSION ``` +`VERSION` must be a valid Python version present in the docker image. +For example, `python3.10`. + The above command will require `sudo` on some systems. On trusted machines, you may wish to add yourself to the `docker` group as described [here](https://docs.docker.com/engine/install/linux-postinstall/). This work flow builds binary wheels for Python [manylinux](https://github.com/pypa/manylinux). -After building and auditing each wheel, the wheels are installed into virtual environments and the `fwdpy11` test suite is run. + +The wheels will be found in `dist/wheelhouse` upon completion. + +**Do not** run this work flow in parallel on the same filesystem. ### Extracting the wheels