Update base image to use Python 3.12 (#295)

* copy changes from #285 I'm copying the changes because that PR is far behind `main`. We will probably wait for the base image change to happen before merging this update work though * upgrade packages until project builds and tests run * fix tests * disable mutation testing * fix: Fix pytest-cov performance Solution from nedbat/coveragepy#1665 (comment) Thanks @trent-codecov for the suggestion.
codecov · Apr 29, 2024 · 02be4f5 · 02be4f5
1 parent 0790e36
commit 02be4f5
Show file tree

Hide file tree

Showing 10 changed files with 85 additions and 74 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -84,41 +84,43 @@ jobs:
       push_rolling: true
       repo: ${{ vars.CODECOV_IMAGE_V2 || 'codecov/self-hosted-worker' }}
 
-  mutation:
-    name: Run Mutations
-    needs: [build]
-    runs-on: ubuntu-latest
-    env:
-      AR_REPO: ${{ vars.CODECOV_IMAGE_V2 || 'codecov/self-hosted-worker' }}
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-      - name: Cache App
-        id: cache-app
-        uses: actions/cache@v3
-        env:
-          cache-name: cache-app
-        with:
-          path: |
-            app.tar
-          key: ${{ runner.os }}-${{ env.cache-name }}-${{ github.run_id }}
-      - name: Load built image
-        run: |
-          docker load --input app.tar
-      - name: Install docker compose
-        run: |
-          sudo curl -SL https://github.com/docker/compose/releases/download/v2.20.0/docker-compose-linux-x86_64 -o /usr/local/bin/docker-compose
-          sudo chmod +x /usr/local/bin/docker-compose
-      - name: Bring test env up
-        run: |
-          make test_env.up
-      - name: Prepare for tests
-        run: |
-          make test_env.prepare
-      - name: Run mutations
-        run: |
-          make test_env.run_mutation
+  # Disable mutations because they currently don't work with Pythin 3.12
+  # A dependency of mutmut doens't support it (see https://github.com/ponyorm/pony/issues/703)
+  # mutation:
+  #   name: Run Mutations
+  #   needs: [build]
+  #   runs-on: ubuntu-latest
+  #   env:
+  #     AR_REPO: ${{ vars.CODECOV_IMAGE_V2 || 'codecov/self-hosted-worker' }}
+  #   steps:
+  #     - name: Checkout
+  #       uses: actions/checkout@v4
+  #       with:
+  #         fetch-depth: 0
+  #     - name: Cache App
+  #       id: cache-app
+  #       uses: actions/cache@v3
+  #       env:
+  #         cache-name: cache-app
+  #       with:
+  #         path: |
+  #           app.tar
+  #         key: ${{ runner.os }}-${{ env.cache-name }}-${{ github.run_id }}
+  #     - name: Load built image
+  #       run: |
+  #         docker load --input app.tar
+  #     - name: Install docker compose
+  #       run: |
+  #         sudo curl -SL https://github.com/docker/compose/releases/download/v2.20.0/docker-compose-linux-x86_64 -o /usr/local/bin/docker-compose
+  #         sudo chmod +x /usr/local/bin/docker-compose
+  #     - name: Bring test env up
+  #       run: |
+  #         make test_env.up
+  #     - name: Prepare for tests
+  #       run: |
+  #         make test_env.prepare
+  #     - name: Run mutations
+  #       run: |
+  #         make test_env.run_mutation
 
 
diff --git a/Makefile b/Makefile
@@ -48,13 +48,13 @@ lint:
 	make lint.run
 
 test:
-	python -m pytest --cov=./ --junitxml=junit.xml
+	COVERAGE_CORE=sysmon python -m pytest --cov=./ --junitxml=junit.xml
 
 test.unit:
-	python -m pytest --cov=./ -m "not integration" --cov-report=xml:unit.coverage.xml --junitxml=unit.junit.xml
+	COVERAGE_CORE=sysmon python -m pytest --cov=./ -m "not integration" --cov-report=xml:unit.coverage.xml --junitxml=unit.junit.xml
 
 test.integration:
-	python -m pytest --cov=./ -m "integration" --cov-report=xml:integration.coverage.xml --junitxml=integration.junit.xml
+	COVERAGE_CORE=sysmon python -m pytest --cov=./ -m "integration" --cov-report=xml:integration.coverage.xml --junitxml=integration.junit.xml
 
 
 update-requirements:

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -12,6 +12,9 @@ services:
       - ./docker/test_codecov_config.yml:/config/codecov.yml
     environment:
       - SETUP__TIMESERIES__ENABLED=${TIMESERIES_ENABLED-true}
+      # Improves pytest-cov performance in python 3.12
+      # https://github.com/nedbat/coveragepy/issues/1665#issuecomment-1937075835
+      - COVERAGE_CORE=sysmon
     env_file:
       - .testenv
     command:

diff --git a/docker/Dockerfile.requirements b/docker/Dockerfile.requirements
@@ -1,6 +1,5 @@
 # syntax=docker/dockerfile:1.4
-ARG PYTHON_IMAGE=python:3.10.13-slim-bookworm
-
+ARG PYTHON_IMAGE=python:3.12.2-slim-bookworm
 # BUILD STAGE
 FROM $PYTHON_IMAGE as build
 
@@ -15,6 +14,8 @@ RUN apt-get install -y \
     build-essential \
     libffi-dev \
     libpq-dev \
+    libxml2-dev \
+    libxslt-dev \
     curl
 
 # Install Rust
@@ -35,6 +36,8 @@ FROM $PYTHON_IMAGE
 RUN apt-get update
 RUN apt-get install -y \
     libpq-dev \
+    libxml2-dev \
+    libxslt-dev \
     make
 
 WORKDIR /pip-packages/

diff --git a/enterprise/package.sh b/enterprise/package.sh
@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 # Simple wrapper around pyinstaller
 
 set -e

diff --git a/requirements.in b/requirements.in
@@ -1,7 +1,7 @@
 https://github.com/codecov/shared/archive/5df0e67bbe5093750a4bb48871e825b694d8f338.tar.gz#egg=shared
 https://github.com/codecov/opentelem-python/archive/refs/tags/v0.0.4a1.tar.gz#egg=codecovopentelem
 https://github.com/codecov/test-results-parser/archive/5515e960d5d38881036e9127f86320efca649f13.tar.gz#egg=test-results-parser
-boto3
+boto3>=1.34
 celery>=5.3.6
 click
 codecov-ribs
@@ -41,8 +41,8 @@ stripe
 https://github.com/codecov/timestring/archive/d37ceacc5954dff3b5bd2f887936a98a668dda42.tar.gz#egg=timestring
 urllib3>=1.26.18
 vcrpy
-opentelemetry-instrumentation-celery>=0.41b0
-opentelemetry-sdk>=1.20.0
+opentelemetry-instrumentation-celery>=0.45b0
+opentelemetry-sdk>=1.24.0
 django-admin
 google-cloud-pubsub
 openai
diff --git a/requirements.txt b/requirements.txt
@@ -2,7 +2,7 @@
 # This file is autogenerated by pip-compile with Python 3.10
 # by the following command:
 #
-#    pip-compile requirements.in
+#    pip-compile
 #
 amqp==5.2.0
     # via kombu
@@ -20,17 +20,15 @@ asgiref==3.7.2
     # via django
 async-timeout==4.0.3
     # via redis
-attrs==20.3.0
-    # via pytest
 backoff==1.6.0
     # via analytics-python
 billiard==4.2.0
     # via celery
-boto3==1.17.26
+boto3==1.34.73
     # via
     #   -r requirements.in
     #   shared
-botocore==1.20.26
+botocore==1.34.73
     # via
     #   boto3
     #   s3transfer
@@ -81,7 +79,7 @@ codecovopentelem @ https://github.com/codecov/opentelem-python/archive/refs/tags
     # via -r requirements.in
 colour==0.1.5
     # via shared
-coverage==5.5
+coverage[toml]==7.5.0
     # via
     #   -r requirements.in
     #   codecovopentelem
@@ -129,7 +127,7 @@ faker==8.8.2
     # via factory-boy
 filelock==3.12.4
     # via virtualenv
-freezegun==1.2.2
+freezegun==1.5.0
     # via pytest-freezegun
 google-api-core[grpc]==2.11.1
     # via
@@ -163,7 +161,7 @@ googleapis-common-protos[grpc]==1.59.1
     #   grpcio-status
 grpc-google-iam-v1==0.12.6
     # via google-cloud-pubsub
-grpcio==1.58.0
+grpcio==1.62.1
     # via
     #   google-api-core
     #   google-cloud-pubsub
@@ -210,7 +208,7 @@ jmespath==0.10.0
     #   botocore
 kombu==5.3.5
     # via celery
-lxml==4.9.1
+lxml==5.1.1
     # via -r requirements.in
 markupsafe==2.1.3
     # via jinja2
@@ -232,28 +230,28 @@ oauthlib==3.1.0
     # via shared
 openai==1.2.4
     # via -r requirements.in
-opentelemetry-api==1.20.0
+opentelemetry-api==1.24.0
     # via
     #   opentelemetry-instrumentation
     #   opentelemetry-instrumentation-celery
     #   opentelemetry-sdk
-opentelemetry-instrumentation==0.41b0
+opentelemetry-instrumentation==0.45b0
     # via opentelemetry-instrumentation-celery
-opentelemetry-instrumentation-celery==0.41b0
+opentelemetry-instrumentation-celery==0.45b0
     # via -r requirements.in
-opentelemetry-sdk==1.20.0
+opentelemetry-sdk==1.24.0
     # via
     #   -r requirements.in
     #   codecovopentelem
-opentelemetry-semantic-conventions==0.41b0
+opentelemetry-semantic-conventions==0.45b0
     # via
     #   opentelemetry-instrumentation-celery
     #   opentelemetry-sdk
 packaging==20.9
     # via pytest
 platformdirs==3.11.0
     # via virtualenv
-pluggy==0.13.1
+pluggy==1.5.0
     # via pytest
 pre-commit==3.4.0
     # via -r requirements.in
@@ -296,7 +294,7 @@ pyparsing==2.4.7
     # via
     #   httplib2
     #   packaging
-pytest==7.2.0
+pytest==8.1.1
     # via
     #   -r requirements.in
     #   pytest-asyncio
@@ -309,7 +307,7 @@ pytest-asyncio==0.14.0
     # via -r requirements.in
 pytest-celery==0.0.0
     # via -r requirements.in
-pytest-cov==2.11.1
+pytest-cov==5.0.0
     # via -r requirements.in
 pytest-django==4.7.0
     # via -r requirements.in
@@ -319,7 +317,7 @@ pytest-mock==1.13.0
     # via -r requirements.in
 pytest-sqlalchemy==0.2.1
     # via -r requirements.in
-python-dateutil==2.8.2
+python-dateutil==2.9.0.post0
     # via
     #   -r requirements.in
     #   analytics-python
@@ -363,10 +361,12 @@ requests==2.31.0
 respx==0.20.2
     # via -r requirements.in
 rfc3986[idna2008]==1.4.0
-    # via httpx
+    # via
+    #   httpx
+    #   rfc3986
 rsa==4.7.2
     # via google-auth
-s3transfer==0.3.4
+s3transfer==0.10.1
     # via boto3
 screen==1.0.1
     # via excel-base
@@ -376,7 +376,7 @@ sentry-sdk==1.40.0
     #   shared
 shared @ https://github.com/codecov/shared/archive/5df0e67bbe5093750a4bb48871e825b694d8f338.tar.gz
     # via -r requirements.in
-six==1.15.0
+six==1.16.0
     # via
     #   analytics-python
     #   click-repl
@@ -406,7 +406,7 @@ statsd==3.3.0
     # via
     #   -r requirements.in
     #   shared
-stripe==2.56.0
+stripe==8.9.0
     # via -r requirements.in
 test-results-parser @ https://github.com/codecov/test-results-parser/archive/5515e960d5d38881036e9127f86320efca649f13.tar.gz
     # via -r requirements.in
@@ -419,7 +419,9 @@ timestring @ https://github.com/codecov/timestring/archive/d37ceacc5954dff3b5bd2
 tlslite-ng==0.8.0b1
     # via shared
 tomli==2.0.1
-    # via pytest
+    # via
+    #   coverage
+    #   pytest
 tqdm==4.66.1
     # via openai
 typing==3.7.4.3
@@ -432,6 +434,7 @@ typing-extensions==4.6.3
     #   pydantic
     #   pydantic-core
     #   shared
+    #   stripe
 tzdata==2024.1
     # via celery
 tzlocal==5.2
@@ -456,7 +459,7 @@ virtualenv==20.24.5
     # via pre-commit
 wcwidth==0.2.5
     # via prompt-toolkit
-wrapt==1.12.1
+wrapt==1.16.0
     # via
     #   deprecated
     #   opentelemetry-instrumentation

diff --git a/services/lock_manager.py b/services/lock_manager.py
@@ -87,7 +87,7 @@ def locked(self, lock_type: LockType, retry_num=0):
                 )
         except LockError:
             max_retry = 200 * 3**retry_num
-            countdown = min(random.randint(max_retry / 2, max_retry), 60 * 60 * 5)
+            countdown = min(random.randint(max_retry // 2, max_retry), 60 * 60 * 5)
 
             log.warning(
                 "Unable to acquire lock",

diff --git a/tasks/tests/unit/test_save_commit_measurements.py b/tasks/tests/unit/test_save_commit_measurements.py
@@ -23,8 +23,8 @@ def test_save_commit_measurements_success(self, dbsession, mocker):
         assert task.run_impl(
             dbsession, commitid=commit.commitid, repoid=commit.repoid
         ) == {"successful": True}
-        assert save_commit_measurements_mock.called_with(
-            commitid=commit.commitid, dataset_names=None
+        save_commit_measurements_mock.assert_called_with(
+            commit=commit, dataset_names=None
         )
 
     def test_save_commit_measurements_no_commit(self, dbsession):

diff --git a/worker.sh b/worker.sh
@@ -6,11 +6,11 @@ if [ -n "$PROMETHEUS_MULTIPROC_DIR" ]; then
 fi
 
 queues=""
-if [[ "$CODECOV_WORKER_QUEUES" ]]; then
+if [ "$CODECOV_WORKER_QUEUES" ]; then
   queues="--queue $CODECOV_WORKER_QUEUES"
 fi
 
-if [[ "$RUN_ENV" == "ENTERPRISE" ]] || [[ "$RUN_ENV" == "DEV" ]]; then
+if [ "$RUN_ENV" = "ENTERPRISE" ] || [ "$RUN_ENV" = "DEV" ]; then
     python manage.py migrate
     python manage.py migrate --database "timeseries"
 fi