Skip to content

Updated attribution files for scheduled-ort #5337

Updated attribution files for scheduled-ort

Updated attribution files for scheduled-ort #5337

Workflow file for this run

name: Python tests
on:
push:
branches: ["main"]
paths:
- python/**
- glide-core/src/**
- submodules/**
- utils/cluster_manager.py
- .github/workflows/python.yml
- .github/workflows/build-python-wrapper/action.yml
- .github/workflows/install-shared-dependencies/action.yml
- .github/workflows/test-benchmark/action.yml
- .github/workflows/lint-rust/action.yml
- .github/workflows/install-valkey/action.yml
- .github/json_matrices/build-matrix.json
- .github/json_matrices/engine-matrix.json
- .github/workflows/start-self-hosted-runner/action.yml
pull_request:
paths:
- python/**
- glide-core/src/**
- submodules/**
- utils/cluster_manager.py
- .github/workflows/python.yml
- .github/workflows/build-python-wrapper/action.yml
- .github/workflows/install-shared-dependencies/action.yml
- .github/workflows/test-benchmark/action.yml
- .github/workflows/lint-rust/action.yml
- .github/workflows/install-valkey/action.yml
- .github/json_matrices/build-matrix.json
- .github/json_matrices/engine-matrix.json
- .github/workflows/start-self-hosted-runner/action.yml
workflow_dispatch:
concurrency:
group: python-${{ github.head_ref || github.ref }}
cancel-in-progress: true
permissions:
contents: read
# Allows the GITHUB_TOKEN to make an API call to generate an OIDC token.
id-token: write
jobs:
load-engine-matrix:
runs-on: ubuntu-latest
outputs:
matrix: ${{ steps.load-engine-matrix.outputs.matrix }}
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Load the engine matrix
id: load-engine-matrix
shell: bash
run: echo "matrix=$(jq -c . < .github/json_matrices/engine-matrix.json)" >> $GITHUB_OUTPUT
test:
runs-on: ${{ matrix.host.RUNNER }}
needs: load-engine-matrix
timeout-minutes: 35
strategy:
fail-fast: false
matrix:
engine: ${{ fromJson(needs.load-engine-matrix.outputs.matrix) }}
python:
# - "3.8"
# - "3.9"
# - "3.10"
# - "3.11"
- "3.12"
host:
- {
OS: ubuntu,
RUNNER: ubuntu-latest,
TARGET: x86_64-unknown-linux-gnu
}
# - {
# OS: macos,
# RUNNER: macos-latest,
# TARGET: aarch64-apple-darwin
# }
steps:
- uses: actions/checkout@v4
with:
submodules: recursive
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python }}
- name: Install dependencies
working-directory: ./python
run: |
python -m pip install --upgrade pip
pip install mypy-protobuf
- name: Build Python wrapper
uses: ./.github/workflows/build-python-wrapper
with:
os: ${{ matrix.host.OS }}
target: ${{ matrix.host.TARGET }}
github-token: ${{ secrets.GITHUB_TOKEN }}
engine-version: ${{ matrix.engine.version }}
- name: Type check with mypy
working-directory: ./python
run: |
# The type check should run inside the virtual env to get
# all installed dependencies and build files
source .env/bin/activate
pip install mypy types-protobuf
# Install the benchmark requirements
pip install -r ../benchmarks/python/requirements.txt
python -m mypy ..
- name: Test with pytest
working-directory: ./python
run: |
source .env/bin/activate
cd python/tests/
pytest --asyncio-mode=auto --html=pytest_report.html --self-contained-html
- uses: ./.github/workflows/test-benchmark
with:
language-flag: -python
- name: Upload test reports
if: always()
continue-on-error: true
uses: actions/upload-artifact@v4
with:
name: test-report-python-${{ matrix.python }}-${{ matrix.engine.type }}-${{ matrix.engine.version }}-${{ matrix.host.RUNNER }}
path: |
python/python/tests/pytest_report.html
utils/clusters/**
benchmarks/results/**
test-pubsub:
runs-on: ${{ matrix.host.RUNNER }}
needs: load-engine-matrix
timeout-minutes: 35
strategy:
fail-fast: false
matrix:
engine: ${{ fromJson(needs.load-engine-matrix.outputs.matrix) }}
python:
# - "3.8"
# - "3.9"
# - "3.10"
# - "3.11"
- "3.12"
host:
- {
OS: ubuntu,
RUNNER: ubuntu-latest,
TARGET: x86_64-unknown-linux-gnu
}
# - {
# OS: macos,
# RUNNER: macos-latest,
# TARGET: aarch64-apple-darwin
# }
steps:
- uses: actions/checkout@v4
with:
submodules: recursive
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python }}
- name: Build Python wrapper
uses: ./.github/workflows/build-python-wrapper
with:
os: ${{ matrix.host.OS }}
target: ${{ matrix.host.TARGET }}
github-token: ${{ secrets.GITHUB_TOKEN }}
engine-version: ${{ matrix.engine.version }}
- name: Test pubsub with pytest
working-directory: ./python
run: |
source .env/bin/activate
cd python/tests/
pytest --asyncio-mode=auto -k test_pubsub --html=pytest_report.html --self-contained-html
- name: Upload test reports
if: always()
continue-on-error: true
uses: actions/upload-artifact@v4
with:
name: pubsub-test-report-python-${{ matrix.python }}-${{ matrix.engine.type }}-${{ matrix.engine.version }}-${{ matrix.host.RUNNER }}
path: |
python/python/tests/pytest_report.html
lint:
runs-on: ubuntu-latest
timeout-minutes: 15
steps:
- uses: actions/checkout@v4
with:
submodules: recursive
- name: lint rust
uses: ./.github/workflows/lint-rust
with:
cargo-toml-folder: ./python
- name: Install dependencies
if: always()
working-directory: ./python
run: |
python -m pip install --upgrade pip
pip install flake8 isort black
- name: Lint python with isort
if: always()
working-directory: ./python
run: |
isort . --profile black --check --diff
- name: Lint python with flake8
if: always()
working-directory: ./python
run: |
# stop the build if there are Python syntax errors or undefined names
flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics --extend-ignore=E230 --exclude=python/glide/protobuf,.env/*
# exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
flake8 . --count --exit-zero --max-complexity=12 --max-line-length=127 --statistics --extend-ignore=E230 --exclude=python/glide/protobuf,.env/*
- name: Lint python with black
if: always()
working-directory: ./python
run: |
black --check --diff .
build-amazonlinux-latest:
runs-on: ubuntu-latest
container: amazonlinux:latest
timeout-minutes: 15
steps:
- name: Install git
run: |
yum -y remove git
yum -y remove git-*
yum -y install https://packages.endpointdev.com/rhel/7/os/x86_64/endpoint-repo.x86_64.rpm
yum install -y git
git --version
- uses: actions/checkout@v4
- name: Checkout submodules
run: |
git config --global --add safe.directory "$GITHUB_WORKSPACE"
git submodule update --init --recursive
- name: Install python
run: |
yum install -y python3
- name: Build Python wrapper
uses: ./.github/workflows/build-python-wrapper
with:
os: "amazon-linux"
target: "x86_64-unknown-linux-gnu"
github-token: ${{ secrets.GITHUB_TOKEN }}
engine-version: "7.2.5"
- name: Test compatibility with pytest
working-directory: ./python
run: |
source .env/bin/activate
pytest --asyncio-mode=auto -m smoke_test --html=pytest_report.html --self-contained-html
- name: Upload test reports
if: always()
continue-on-error: true
uses: actions/upload-artifact@v4
with:
name: smoke-test-report-amazon-linux
path: |
python/python/tests/pytest_report.html
start-self-hosted-runner:
if: github.event.pull_request.head.repo.owner.login == 'valkey-io'
runs-on: ubuntu-latest
environment: AWS_ACTIONS
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Start self hosted EC2 runner
uses: ./.github/workflows/start-self-hosted-runner
with:
role-to-assume: ${{ secrets.ROLE_TO_ASSUME }}
aws-region: ${{ secrets.AWS_REGION }}
ec2-instance-id: ${{ secrets.AWS_EC2_INSTANCE_ID }}
test-modules:
needs: [start-self-hosted-runner, load-engine-matrix]
name: Running Module Tests
runs-on: ${{ matrix.host.RUNNER }}
timeout-minutes: 35
strategy:
fail-fast: false
matrix:
engine: ${{ fromJson(needs.load-engine-matrix.outputs.matrix) }}
python:
- "3.12"
host:
- {
OS: "ubuntu",
NAMED_OS: "linux",
RUNNER: ["self-hosted", "Linux", "ARM64"],
TARGET: "aarch64-unknown-linux-gnu",
}
steps:
- name: Setup self-hosted runner access
if: ${{ contains(matrix.host.RUNNER, 'self-hosted') }}
run: sudo chown -R $USER:$USER /home/ubuntu/actions-runner/_work/valkey-glide
- uses: actions/checkout@v4
with:
submodules: recursive
- name: Setup Python for self-hosted Ubuntu runners
run: |
sudo apt update -y
sudo apt upgrade -y
sudo apt install python3 python3-venv python3-pip -y
- name: Build Python wrapper
uses: ./.github/workflows/build-python-wrapper
with:
os: ${{ matrix.host.OS }}
target: ${{ matrix.host.TARGET }}
github-token: ${{ secrets.GITHUB_TOKEN }}
engine-version: ${{ matrix.engine.version }}
- name: Test with pytest
working-directory: ./python
run: |
source .env/bin/activate
cd python/tests/
pytest --asyncio-mode=auto --tls --cluster-endpoints=${{ secrets.MEMDB_MODULES_ENDPOINT }} -k server_modules --html=pytest_report.html --self-contained-html
- name: Upload test reports
if: always()
continue-on-error: true
uses: actions/upload-artifact@v4
with:
name: modules-test-report-${{ matrix.host.TARGET }}-python-${{ matrix.python }}-server-${{ matrix.engine.version }}
path: |
python/python/tests/pytest_report.html