Rewrite wheel building work flow.

* simpify scripts in deployment/linux_wheels/
* use new scripts in CI
* validate sdist separately in CI
* enable parallel CI jobs
* update docs

Closes #1192

.github/workflows/wheels.yml

@@ -10,6 +10,7 @@ on:
 
 jobs:
   macOS_wheel:
+    name: Build macOS/x86 wheels
     runs-on: macos-latest
     strategy:
       matrix:
@@ -64,7 +65,8 @@ jobs:
           name: macOS-wheel-${{ matrix.python }}
           path: dist
 
-  manylinux2_28:
+  build_sdist:
+    name: Build source distribution
     runs-on: ubuntu-latest
     steps:
       - name: Cancel Previous Runs
@@ -95,10 +97,29 @@ jobs:
           name: sdist
           path: dist
 
+  manylinux2_28:
+    name: Build and test Linux wheels
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python: ["python3.8", "python3.9", "python3.10", "python3.11"]
+    steps:
+      - name: Cancel Previous Runs
+        uses: styfle/cancel-workflow-action@0.11.0
+        with:
+          access_token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+            submodules: true
+            fetch-depth: 0
+
       - name: Build wheels in docker
         shell: bash
         run: |
-          docker run --rm -v `pwd`:/project -w /project quay.io/pypa/manylinux_2_28_x86_64:2022-10-02-69a0972 bash .github/workflows/manylinux/buildwheels.sh
+          bash deployment/linux_wheels/run_wheel_workflow.sh ${{ matrix.python }}
+          #docker run --rm -v `pwd`:/project -w /project quay.io/pypa/manylinux_2_28_x86_64:2022-10-02-69a0972 bash .github/workflows/manylinux/buildwheels.sh
 
       - name: Upload Wheels
         uses: actions/upload-artifact@v3
@@ -107,8 +128,9 @@ jobs:
           path: dist/wheelhouse
 
   manylinux2_28_test:
+    name: Build package from source dist
     runs-on: ubuntu-latest
-    needs: ['manylinux2_28']
+    needs: ['build_sdist']
     strategy:
       matrix:
         python: [3.8, 3.9, "3.10", "3.11"]
@@ -119,10 +141,6 @@ jobs:
           access_token: ${{ secrets.GITHUB_TOKEN }}
       - name: Checkout
         uses: actions/checkout@v3
-      - name: Download wheels
-        uses: actions/download-artifact@v3.0.2
-        with:
-          name: linux-wheels
       - name: Download sdist
         uses: actions/download-artifact@v3.0.2
         with:
@@ -144,21 +162,22 @@ jobs:
           deactivate
           rm -rf sdist_venv
 
-      - name: Install wheel and test
-        run: |
-          python -VV
-          # pip install minimal dependencies
-          pip install --upgrade pip
-          pip install wheel
-          pip install -r requirements/wheel_building_workflow.txt
-          # delete the source dir to prevent pip from mistaking it for 
-          # the package
-          rm -rf fwdpy11
-          # Install the local wheel
-          pip install fwdpy11 --no-deps --no-index --pre --only-binary fwdpy11 -f .
-          python -c "import fwdpy11;print(fwdpy11.__version__)"
+          # - name: Install wheel and test
+          #   run: |
+          #     python -VV
+          #     # pip install minimal dependencies
+          #     pip install --upgrade pip
+          #     pip install wheel
+          #     pip install -r requirements/wheel_building_workflow.txt
+          #     # delete the source dir to prevent pip from mistaking it for 
+          #     # the package
+          #     rm -rf fwdpy11
+          #     # Install the local wheel
+          #     pip install fwdpy11 --no-deps --no-index --pre --only-binary fwdpy11 -f .
+          #     python -c "import fwdpy11;print(fwdpy11.__version__)"
 
   macOS_test:
+    name: Test macOS/x86 wheels
     needs: ['macOS_wheel']
     runs-on: macos-latest
     strategy:
@@ -197,8 +216,9 @@ jobs:
           python -c "import fwdpy11;print(fwdpy11.__version__)"
       
   upload_to_PyPI:
+    name: Upload to PyPI
     runs-on: ubuntu-latest
-    needs: ['macOS_test', 'manylinux2_28_test']
+    needs: ['macOS_test', 'manylinux2_28', 'manylinux2_28_test']
     steps:
       - name: Cancel Previous Runs
         uses: styfle/cancel-workflow-action@0.11.0

deployment/linux_wheels/run_wheel_workflow.sh

@@ -0,0 +1,3 @@
+PYTHON=$1
+
+docker run --rm -v `pwd`:/project -w /project quay.io/pypa/manylinux_2_28_x86_64:2022-10-02-69a0972 /bin/sh ./deployment/linux_wheels/wheel_workflow.sh $PYTHON

deployment/linux_wheels/wheel_workflow.sh

@@ -0,0 +1,70 @@
+PYTHON=$1
+
+# SETUP
+set -e -x
+rm -rf venv
+
+# INSTALL SYSTEM DEPENDENCIES
+
+yum -y install curl gsl-devel
+
+# INSTALL RUST
+
+curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | bash -s -- -y 
+source "$HOME/.cargo/env" 
+# Pin the rustc toolchain to a specific version.
+# Rust 1.64.0 will change the minimum glibc ABI
+# to a version incompatible with manylinux_2014,
+# so we need to be careful in general.
+rustup override set 1.62.1 
+# Pin cbindgen 
+cargo install --locked cbindgen@0.24.3 
+
+# Taken from msprime/#2043
+# We're running as root in the docker container so git commands issued by
+# setuptools_scm will fail without this:
+git config --global --add safe.directory /project
+
+# BUILD WHEEL INSIDE A VENV
+
+$(which $PYTHON) -m venv venv
+source venv/bin/activate
+python -m pip install --upgrade pip setuptools build oldest-supported-numpy
+python -m build .
+deactivate
+rm -rf build venv
+
+cd dist
+for whl in *.whl; do
+    # Need to set this so that the core library
+    # can be found
+    LD_LIBRARY_PATH=fwdpy11 auditwheel repair "$whl"
+    rm "$whl"
+done
+cd ..
+
+# INSTALL WHEEL INTO VENV FOR TESTING
+
+$(which $PYTHON) -m venv venv
+source venv/bin/activate
+python -m pip install --upgrade pip
+python -m pip install wheel
+python -m pip install fwdpy11 --pre --no-cache-dir --only-binary fwdpy11 --find-links dist/wheelhouse
+# cd to avoid having the fwdpy11/ directory get mistaken for a package
+TESTDIR=wheel_tests_$PYTHON
+if [ -e  $TESTDIR ]
+then
+    rm -rf $TESTDIR
+fi
+mkdir $TESTDIR
+cd $TESTDIR
+python -c "import fwdpy11;print(fwdpy11.__version__)"
+python -m pip install pytest pytest-xdist hypothesis msprime
+# Copy the tests b/c some of what they do depends on paths
+# and will fail don't run them from right outside that directory.
+cp -r ../tests .
+python -m pytest tests -n 4
+rm -rf $TESTDIR
+cd ..
+deactivate
+rm -rf venv

doc/pages/deploymenttools.md

@@ -161,19 +161,26 @@ docker run --rm -v $HOME/tmp:/app IMAGE_NAME /bin/bash -c ". /venv/bin/activate;
 ## Linux wheel building
 
 The directory `deployment/linux_wheels` defines a `Docker` work flow to build binary wheels on Linux.
-Although this work flow *could* be used to make wheels for each new release, its main purposes are for testing and easily getting development versions to collaborators.
+These scripts are what we use to build wheels for each release.
+They may also be used to make wheels for development versions on a local machine.
 
-From the `root` of the `fwdpy11` repository:
+From the `root` of the `fwdpy11` repository.
 
 ```sh
-bash deployment/linux_wheels/build_docker_image.sh
+bash deployment/linux_wheels/run_wheel_workflow.sh VERSION
 ```
 
+`VERSION` must be a valid Python version present in the docker image.
+For example, `python3.10`.
+
 The above command will require `sudo` on some systems.
 On trusted machines, you may wish to add yourself to the `docker` group as described [here](https://docs.docker.com/engine/install/linux-postinstall/).
 
 This work flow builds binary wheels for Python [manylinux](https://github.com/pypa/manylinux).
-After building and auditing each wheel, the wheels are installed into virtual environments and the `fwdpy11` test suite is run.
+
+The wheels will be found in `dist/wheelhouse` upon completion.
+
+**Do not** run this work flow in parallel on the same filesystem.
 
 ### Extracting the wheels