[CLIENT-2217] Bundle manylinux2014 wheels with OpenSSL 3.0 (#688)

CI/CD: Add workflow to build and install OpenSSL 3 with the manylinux2014 images

.github/workflows/build-wheels.yml

@@ -162,6 +162,7 @@ jobs:
     env:
       BUILD_IDENTIFIER: "${{ matrix.python-tag }}-${{ inputs.platform-tag }}"
       MACOS_OPENSSL_VERSION: 3
+      CUSTOM_IMAGE_NAME: ghcr.io/aerospike/manylinux2014_{0}:latest
     steps:
     - name: Create status check message
       run: echo STATUS_CHECK_MESSAGE="cibuildwheel (${{ env.BUILD_IDENTIFIER }})" >> $GITHUB_ENV
@@ -227,7 +228,7 @@ jobs:
     - name: Otherwise, enable integration tests
       if: ${{ env.RUN_INTEGRATION_TESTS_IN_CIBW == 'true' }}
       # Run with capture output disabled to check that TLS works (i.e we are using the bundled openssl)
-      run: echo "TEST_COMMAND=cd {project}/test/ && pip install -r requirements.txt && python -m pytest -vv new_tests/${{ inputs.test-file }}" >> $GITHUB_ENV
+      run: echo "TEST_COMMAND=cd {project}/test/ && pip install -r requirements.txt && python -m pytest -vvs new_tests/${{ inputs.test-file }}" >> $GITHUB_ENV
       shell: bash
 
     - name: Set unoptimize flag
@@ -238,6 +239,19 @@ jobs:
       if: ${{ inputs.include-debug-info-for-macos && startsWith(inputs.platform-tag, 'macosx') }}
       run: echo "INCLUDE_DSYM=1" >> $GITHUB_ENV
 
+    - if: ${{ startsWith(inputs.platform-tag, 'manylinux') }}
+      run: echo CIBW_MANYLINUX_X86_64_IMAGE=${{ format(env.CUSTOM_IMAGE_NAME, 'x86_64') }} >> $GITHUB_ENV
+
+    - if: ${{ startsWith(inputs.platform-tag, 'manylinux') }}
+      run: echo CIBW_MANYLINUX_AARCH64_IMAGE=${{ format(env.CUSTOM_IMAGE_NAME, 'aarch64') }} >> $GITHUB_ENV
+
+    - uses: docker/login-action@v3
+      if: ${{ startsWith(inputs.platform-tag, 'manylinux') }}
+      with:
+        registry: ghcr.io
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
+
     - name: Build wheel
       uses: pypa/cibuildwheel@v2.21.3
       env:
@@ -246,11 +260,11 @@ jobs:
         CIBW_BUILD: ${{ env.BUILD_IDENTIFIER }}
         CIBW_BUILD_FRONTEND: build
         CIBW_BEFORE_ALL_LINUX: >
-          yum install openssl-devel -y &&
-          yum install python-devel -y &&
           yum install python-setuptools -y
         # delvewheel is not enabled by default but we do need to repair the wheel
         CIBW_BEFORE_BUILD_WINDOWS: "pip install delvewheel==1.*"
+        # We want to check that our new openssl 3 install is used, not the system default
+        CIBW_REPAIR_WHEEL_COMMAND_LINUX: auditwheel repair -w {dest_dir} {wheel} && auditwheel show {dest_dir}/*
         CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: "delvewheel repair --add-path ./aerospike-client-c/vs/x64/Release -w {dest_dir} {wheel}"
         CIBW_TEST_COMMAND: ${{ env.TEST_COMMAND }}
 

.github/workflows/manylinux2014-openssl.Dockerfile

@@ -0,0 +1,24 @@
+ARG CPU_ARCH=x86_64
+FROM quay.io/pypa/manylinux2014_$CPU_ARCH
+ARG OPENSSL_VERSION
+LABEL com.aerospike.clients.openssl-version=$OPENSSL_VERSION
+
+RUN yum install -y perl-core wget
+
+ARG OPENSSL_TAR_NAME=openssl-$OPENSSL_VERSION
+RUN wget https://www.openssl.org/source/$OPENSSL_TAR_NAME.tar.gz
+RUN tar xzvf $OPENSSL_TAR_NAME.tar.gz
+WORKDIR $OPENSSL_TAR_NAME
+
+# The default folder pointed to by --prefix contains a default openssl installation
+# But we're assuming it's fine to replace the default openssl that comes with the image
+# We aren't going to use this image in production, anyways
+RUN ./Configure
+RUN make
+# These tests are expected to fail because we are using a buggy version of nm
+# https://github.com/openssl/openssl/issues/18953
+# devtoolset-11 contains a newer version of binutils 2.36, which contains a bug fix for nm
+# We don't use it though because we want to make sure the compiled openssl 3 library is compatible with manylinux2014's
+# default env
+RUN make V=1 TESTS='-test_symbol_presence*' test
+RUN make install

.github/workflows/update-manylinux-openssl-image.yml

@@ -0,0 +1,62 @@
+on:
+  schedule:
+    # * is a special character in YAML so you have to quote this string
+    - cron:  '0 17 * * 1-5'
+  workflow_dispatch:
+
+jobs:
+  main:
+    env:
+      # We want granular control over the openssl version bundled with our wheels
+      OPENSSL_VERSION: '3.0.15'
+      REGISTRY: ghcr.io
+    strategy:
+      matrix:
+        arch-and-runner-os: [
+          [x86_64, ubuntu-24.04],
+          [aarch64, aerospike_arm_runners_2]
+        ]
+      fail-fast: false
+
+    runs-on: ${{ matrix.arch-and-runner-os[1] }}
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        sparse-checkout: |
+          .github/workflows
+
+    - run: docker pull quay.io/pypa/manylinux2014_${{ matrix.arch-and-runner-os[0] }}
+
+    - uses: docker/login-action@v3
+      with:
+        registry: ${{ env.REGISTRY }}
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
+
+    - uses: docker/metadata-action@v5
+      id: meta
+      with:
+        images: ${{ env.REGISTRY }}/aerospike/manylinux2014_${{ matrix.arch-and-runner-os[0] }}
+        flavor: latest=true
+
+    - name: Set up Docker Buildx so we can cache our Docker image layers
+      uses: docker/setup-buildx-action@v3
+
+    - name: Build and push
+      uses: docker/build-push-action@v6
+      with:
+        # Don't want to use default Git context or else it will clone the whole Python client repo again
+        context: .github/workflows
+        file: .github/workflows/manylinux2014-openssl.Dockerfile
+        build-args: |
+          OPENSSL_VERSION=${{ env.OPENSSL_VERSION }}
+          CPU_ARCH=${{ matrix.arch-and-runner-os[0] }}
+        # setup-buildx-action configures Docker to use the docker-container build driver
+        # This driver doesn't publish an image locally by default
+        # so we have to manually enable it
+        push: true
+        tags: ${{ steps.meta.outputs.tags }}
+        labels: ${{ steps.meta.outputs.labels }}
+        # Also cache intermediate layers to make development faster
+        cache-from: type=gha
+        cache-to: type=gha,mode=max