python.yml

name: Python tests

on:
    push:
        branches: ["main"]
        paths:
            - python/**
            - glide-core/src/**
            - submodules/**
            - utils/cluster_manager.py
            - .github/workflows/python.yml
            - .github/workflows/build-python-wrapper/action.yml
            - .github/workflows/install-shared-dependencies/action.yml
            - .github/workflows/test-benchmark/action.yml
            - .github/workflows/lint-rust/action.yml
            - .github/workflows/install-valkey/action.yml
            - .github/json_matrices/build-matrix.json
            - .github/json_matrices/engine-matrix.json
            - .github/workflows/start-self-hosted-runner/action.yml

    pull_request:
        paths:
            - python/**
            - glide-core/src/**
            - submodules/**
            - utils/cluster_manager.py
            - .github/workflows/python.yml
            - .github/workflows/build-python-wrapper/action.yml
            - .github/workflows/install-shared-dependencies/action.yml
            - .github/workflows/test-benchmark/action.yml
            - .github/workflows/lint-rust/action.yml
            - .github/workflows/install-valkey/action.yml
            - .github/json_matrices/build-matrix.json
            - .github/json_matrices/engine-matrix.json
            - .github/workflows/start-self-hosted-runner/action.yml
    workflow_dispatch:

concurrency:
    group: python-${{ github.head_ref || github.ref }}
    cancel-in-progress: true

permissions:
    contents: read
    # Allows the GITHUB_TOKEN to make an API call to generate an OIDC token.
    id-token: write

jobs:
    load-engine-matrix:
        runs-on: ubuntu-latest
        outputs:
          matrix: ${{ steps.load-engine-matrix.outputs.matrix }}
        steps:
            - name: Checkout
              uses: actions/checkout@v4
        
            - name: Load the engine matrix
              id: load-engine-matrix
              shell: bash
              run: echo "matrix=$(jq -c . < .github/json_matrices/engine-matrix.json)" >> $GITHUB_OUTPUT
        
    test:
        runs-on: ${{ matrix.host.RUNNER }}
        needs: load-engine-matrix
        timeout-minutes: 35
        strategy:
            fail-fast: false
            matrix:
                engine: ${{ fromJson(needs.load-engine-matrix.outputs.matrix) }}
                python:
                  # - "3.8"
                  # - "3.9"
                  # - "3.10"
                  # - "3.11"
                    - "3.12"
                host:
                    - {
                        OS: ubuntu,
                        RUNNER: ubuntu-latest,
                        TARGET: x86_64-unknown-linux-gnu
                    }
                    # - {
                    #     OS: macos,
                    #     RUNNER: macos-latest,
                    #     TARGET: aarch64-apple-darwin
                    # }

        steps:
            - uses: actions/checkout@v4
              with:
                  submodules: recursive

            - name: Set up Python
              uses: actions/setup-python@v4
              with:
                  python-version: ${{ matrix.python }}

            - name: Install dependencies
              working-directory: ./python
              run: |
                  python -m pip install --upgrade pip
                  pip install mypy-protobuf

            - name: Build Python wrapper
              uses: ./.github/workflows/build-python-wrapper
              with:
                  os: ${{ matrix.host.OS }}
                  target: ${{ matrix.host.TARGET }}
                  github-token: ${{ secrets.GITHUB_TOKEN }}
                  engine-version: ${{ matrix.engine.version }}

            - name: Type check with mypy
              working-directory: ./python
              run: |
                  # The type check should run inside the virtual env to get
                  # all installed dependencies and build files
                  source .env/bin/activate
                  pip install mypy types-protobuf
                  # Install the benchmark requirements
                  pip install -r ../benchmarks/python/requirements.txt
                  python -m mypy ..

            - name: Test with pytest
              working-directory: ./python
              run: |
                  source .env/bin/activate
                  cd python/tests/
                  pytest --asyncio-mode=auto --html=pytest_report.html --self-contained-html

            - uses: ./.github/workflows/test-benchmark
              with:
                  language-flag: -python

            - name: Upload test reports
              if: always()
              continue-on-error: true
              uses: actions/upload-artifact@v4
              with:
                  name: test-report-python-${{ matrix.python }}-${{ matrix.engine.type }}-${{ matrix.engine.version }}-${{ matrix.host.RUNNER }}
                  path: |
                      python/python/tests/pytest_report.html
                      utils/clusters/**
                      benchmarks/results/**

    test-pubsub:
        runs-on: ${{ matrix.host.RUNNER }}
        needs: load-engine-matrix
        timeout-minutes: 35
        strategy:
            fail-fast: false
            matrix:
                engine: ${{ fromJson(needs.load-engine-matrix.outputs.matrix) }}
                python:
                  # - "3.8"
                  # - "3.9"
                  # - "3.10"
                  # - "3.11"
                    - "3.12"
                host:
                    - {
                        OS: ubuntu,
                        RUNNER: ubuntu-latest,
                        TARGET: x86_64-unknown-linux-gnu
                     }
                    # - {
                    #     OS: macos,
                    #     RUNNER: macos-latest,
                    #     TARGET: aarch64-apple-darwin
                    # }

        steps:
            - uses: actions/checkout@v4
              with:
                  submodules: recursive
        
            - name: Set up Python
              uses: actions/setup-python@v4
              with:
                  python-version: ${{ matrix.python }}
        
            - name: Build Python wrapper
              uses: ./.github/workflows/build-python-wrapper
              with:
                  os: ${{ matrix.host.OS }}
                  target: ${{ matrix.host.TARGET }}
                  github-token: ${{ secrets.GITHUB_TOKEN }}
                  engine-version: ${{ matrix.engine.version }}
        
            - name: Test pubsub with pytest
              working-directory: ./python
              run: |
                  source .env/bin/activate
                  cd python/tests/
                  pytest --asyncio-mode=auto -k test_pubsub --html=pytest_report.html --self-contained-html
        
            - name: Upload test reports
              if: always()
              continue-on-error: true
              uses: actions/upload-artifact@v4
              with:
                  name: pubsub-test-report-python-${{ matrix.python }}-${{ matrix.engine.type }}-${{ matrix.engine.version }}-${{ matrix.host.RUNNER }}
                  path: |
                      python/python/tests/pytest_report.html

    lint:
        runs-on: ubuntu-latest
        timeout-minutes: 15
        steps:
            - uses: actions/checkout@v4
              with:
                  submodules: recursive

            - name: lint rust
              uses: ./.github/workflows/lint-rust
              with:
                  cargo-toml-folder: ./python

            - name: Install dependencies
              if: always()
              working-directory: ./python
              run: |
                  python -m pip install --upgrade pip
                  pip install flake8 isort black

            - name: Lint python with isort
              if: always()
              working-directory: ./python
              run: |
                  isort . --profile black --check --diff

            - name: Lint python with flake8
              if: always()
              working-directory: ./python
              run: |
                  # stop the build if there are Python syntax errors or undefined names
                  flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics --extend-ignore=E230 --exclude=python/glide/protobuf,.env/*
                  # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
                  flake8 . --count --exit-zero --max-complexity=12 --max-line-length=127 --statistics --extend-ignore=E230 --exclude=python/glide/protobuf,.env/*

            - name: Lint python with black
              if: always()
              working-directory: ./python
              run: |
                  black --check --diff .

    build-amazonlinux-latest:
        runs-on: ubuntu-latest
        container: amazonlinux:latest
        timeout-minutes: 15
        steps:
            - name: Install git
              run: |
                  yum -y remove git
                  yum -y remove git-*
                  yum -y install https://packages.endpointdev.com/rhel/7/os/x86_64/endpoint-repo.x86_64.rpm
                  yum install -y git
                  git --version

            - uses: actions/checkout@v4

            - name: Checkout submodules
              run: |
                  git config --global --add safe.directory "$GITHUB_WORKSPACE"
                  git submodule update --init --recursive

            - name: Install python
              run: |
                  yum install -y python3

            - name: Build Python wrapper
              uses: ./.github/workflows/build-python-wrapper
              with:
                  os: "amazon-linux"
                  target: "x86_64-unknown-linux-gnu"
                  github-token: ${{ secrets.GITHUB_TOKEN }}
                  engine-version: "7.2.5"

            - name: Test compatibility with pytest
              working-directory: ./python
              run: |
                  source .env/bin/activate
                  pytest --asyncio-mode=auto -m smoke_test --html=pytest_report.html --self-contained-html

            - name: Upload test reports
              if: always()
              continue-on-error: true
              uses: actions/upload-artifact@v4
              with:
                  name: smoke-test-report-amazon-linux
                  path: |
                      python/python/tests/pytest_report.html
    
    start-self-hosted-runner:
      if: github.event.pull_request.head.repo.owner.login == 'valkey-io'
      runs-on: ubuntu-latest
      environment: AWS_ACTIONS
      steps:
          - name: Checkout
            uses: actions/checkout@v4

          - name: Start self hosted EC2 runner
            uses: ./.github/workflows/start-self-hosted-runner
            with:
                role-to-assume: ${{ secrets.ROLE_TO_ASSUME }}
                aws-region: ${{ secrets.AWS_REGION }}
                ec2-instance-id: ${{ secrets.AWS_EC2_INSTANCE_ID }}

    test-modules:
      needs: [start-self-hosted-runner, load-engine-matrix]
      name: Running Module Tests
      runs-on: ${{ matrix.host.RUNNER }}
      timeout-minutes: 35
      strategy:
        fail-fast: false
        matrix:
                engine: ${{ fromJson(needs.load-engine-matrix.outputs.matrix) }}
                python:
                    - "3.12"
                host:
                    - {
                        OS: "ubuntu",
                        NAMED_OS: "linux",
                        RUNNER: ["self-hosted", "Linux", "ARM64"],
                        TARGET: "aarch64-unknown-linux-gnu",
                     }
        
      steps:
          - name: Setup self-hosted runner access
            if: ${{ contains(matrix.host.RUNNER, 'self-hosted') }}
            run: sudo chown -R $USER:$USER /home/ubuntu/actions-runner/_work/valkey-glide
          
          - uses: actions/checkout@v4
            with:
              submodules: recursive
          
          - name: Setup Python for self-hosted Ubuntu runners
            run: |
                  sudo apt update -y
                  sudo apt upgrade -y
                  sudo apt install python3 python3-venv python3-pip -y
  
          - name: Build Python wrapper
            uses: ./.github/workflows/build-python-wrapper
            with:
                    os: ${{ matrix.host.OS }}
                    target: ${{ matrix.host.TARGET }}
                    github-token: ${{ secrets.GITHUB_TOKEN }}
                    engine-version: ${{ matrix.engine.version }}

          - name: Test with pytest
            working-directory: ./python
            run: |
                    source .env/bin/activate
                    cd python/tests/
                    pytest --asyncio-mode=auto --tls --cluster-endpoints=${{ secrets.MEMDB_MODULES_ENDPOINT }} -k server_modules --html=pytest_report.html --self-contained-html

          - name: Upload test reports
            if: always()
            continue-on-error: true
            uses: actions/upload-artifact@v4
            with:
                name: smoke-test-report-amazon-linux
                path: |
                    python/python/tests/pytest_report.html