Skip to content
Total Perspective Vortex

Merge pull request #1320 from usegalaxy-eu/bgruening-patch-10 #695

Sign in to view logs

Merge pull request #1320 from usegalaxy-eu/bgruening-patch-10

Merge pull request #1320 from usegalaxy-eu/bgruening-patch-10 #695

Workflow file for this run

.github/workflows/tpv.yml at 84a00dc
---
name: Total Perspective Vortex
'on':
pull_request:
paths:
- 'files/galaxy/tpv/**'
push:
branches:
- master
paths:
- 'files/galaxy/tpv/**'
jobs:
lint:
name: Total Perspective Vortex linter
runs-on: ubuntu-latest
steps:
- name: Check out the codebase.
uses: actions/checkout@v2
with:
path: 'infrastructure-playbook'
- name: Workaround Ansible vault password not being available to GitHub.
working-directory: 'infrastructure-playbook'
run: |
rm -f group_vars/htcondor/vault.yml
rm -f group_vars/htcondor-secondary/vault.yml
- name: Update git submodules.
working-directory: 'infrastructure-playbook'
run: |
git submodule update --init --recursive --remote --checkout
- name: Set up Python 3.
uses: actions/setup-python@v4
with:
python-version: '3.11'
cache: 'pip'
# Install Ansible.
- name: Cache Ansible.
id: cache-ansible
uses: actions/cache@v3
with:
path: /opt/hostedtoolcache/Python/*/*/lib/python*/site-packages/ansible*
key: ${{ hashFiles('infrastructure-playbook/requirements.txt') }}
- name: Install Ansible (fast when an existing installation was already cached).
working-directory: 'infrastructure-playbook'
run: |
# Install an Ansible version compatible with the version of
# ansible-core specified in requirements.txt for the
# infrastructure-playbook repo.
ANSIBLE_CORE_REQ=$(perl -pe 's/\\\n/ /' requirements.txt | grep ansible-core)
pip3 install --use-feature=fast-deps ansible "$ANSIBLE_CORE_REQ"
- name: Save Ansible cache.
uses: actions/cache/save@v3
if: ${{ steps.cache-ansible.outputs.cache-hit != 'true' }}
with:
path: /opt/hostedtoolcache/Python/*/*/lib/python*/site-packages/ansible*
key: ${{ hashFiles('infrastructure-playbook/requirements.txt') }}
# Total Perspective Vortex needs the Galaxy logic, which should be
# installed automatically when running `pip3 install
# total-perspective-vortex[cli]` (the `galaxy-app` package).
# However:
# - `galaxy-app` package on PyPI is outdated (see issue #15999 on
# the Galaxy repo: https://github.com/galaxyproject/galaxy/issues/15999)
# - Ideally the version of Galaxy should exactly match the one running on
# usegalaxy.eu.
# Therefore, we clone Galaxy and add it to the PYTHONPATH.
- name: Get Galaxy repo and commit id.
working-directory: 'infrastructure-playbook'
run: |
# Get the Galaxy repository URL and commit from Ansible variables.
export TMP_FILE=`mktemp`
openssl rand -base64 24 > .vault_password
ansible localhost --connection local \
--inventory hosts --module-name copy \
--args "content={{hostvars['sn06.galaxyproject.eu']}} dest=${TMP_FILE}" \
> /dev/null
export GALAXY_COMMIT_ID=$(cat ${TMP_FILE} | jq -r .galaxy_commit_id)
export GALAXY_REPO=$(cat ${TMP_FILE} | jq -r .galaxy_repo)
rm ${TMP_FILE}
echo $GALAXY_COMMIT_ID > ../galaxy_commit_id
echo $GALAXY_REPO > ../galaxy_repo
- name: Cache Galaxy
id: cache-galaxy
uses: actions/cache@v3
with:
path: galaxy
key: ${{ hashFiles('galaxy_repo') }}-${{ hashFiles('galaxy_commit_id') }}
- name: Clone Galaxy
if: ${{ steps.cache-galaxy.outputs.cache-hit != 'true' }}
run: |
export GALAXY_COMMIT_ID="$(cat ./galaxy_commit_id)"
export GALAXY_REPO="$(cat ./galaxy_repo)"
# git clone -b $GALAXY_COMMIT_ID --single-branch --depth=1 $GALAXY_REPO galaxy # does not work with commit hashes
git clone $GALAXY_REPO galaxy && cd galaxy && git checkout $GALAXY_COMMIT_ID
- name: Save Galaxy cache.
uses: actions/cache/save@v3
if: ${{ steps.cache-galaxy.outputs.cache-hit != 'true' }}
with:
path: galaxy
key: ${{ hashFiles('galaxy_repo') }}-${{ hashFiles('galaxy_commit_id') }}
- name: Install Galaxy requirements.
working-directory: 'galaxy'
run: pip install -r requirements.txt
# Install the Total Perspective Vortex version that should be running on
# usegalaxy.eu
- name: Install Total Perspective Vortex.
working-directory: 'galaxy'
run: |
TPV_REQ=$(perl -pe 's/\\\n/ /' lib/galaxy/dependencies/conditional-requirements.txt | grep total-perspective-vortex)
pip3 install --upgrade "$TPV_REQ"
- name: Install port of Ansible filters for Jinja (required for the next step).
run: |
pip3 install jinja2-ansible-filters
- name: Create mounts vars file.
working-directory: 'infrastructure-playbook/mounts'
run: |
make dest/all.yml
- name: Create a playbook to template the TPV files.
shell: python -u {0}
working-directory: 'infrastructure-playbook'
run: |
import glob
import importlib.util
import sys
# import tpv.py
spec = importlib.util.spec_from_file_location('tpv_ci', '.github/workflows/tpv.py')
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
make_playbook = module.make_playbook
tpv_path = "files/galaxy/tpv/"
templates = tuple(
(file, '.'.join(file.split('.')[:-1])) # remove j2 extension
for file in glob.glob(f"{tpv_path}/*.yaml.j2") + glob.glob(f"{tpv_path}/*.yml.j2")
)
playbook = make_playbook('sn06.yml', templates=templates)
with open('../playbook_path', 'w') as file:
file.write(str(playbook))
- name: Render TPV configuration files.
run: |
PLAYBOOK="$(cat playbook_path)"
BASENAME="$(basename $PLAYBOOK)"
DIRNAME="$(dirname $PLAYBOOK)"
cd "$DIRNAME"
shopt -s nullglob
ansible-playbook --connection=local "$BASENAME"
- name: Run Total Perspective Vortex linter.
run: |
export PYTHONPATH=$(realpath ./galaxy/lib)
PLAYBOOK="$(cat playbook_path)"
BASENAME="$(basename $PLAYBOOK)"
DIRNAME="$(dirname $PLAYBOOK)"
cd "$DIRNAME"
shopt -s nullglob
for file in files/galaxy/tpv/*.{yml,yaml}; do
echo Running TPV linter on "$file"...
tpv lint $file || exit 1
done
dry-run:
name: Total Perspective Vortex dry-run
runs-on: ubuntu-latest
steps:
- name: Check out the codebase.
uses: actions/checkout@v3
with:
fetch-depth: ${{ github.event_name == 'pull_request' && 2 || 0 }}
path: 'infrastructure-playbook'
- name: Workaround Ansible vault password not being available to GitHub.
working-directory: 'infrastructure-playbook'
run: |
rm -f group_vars/htcondor/vault.yml
rm -f group_vars/htcondor-secondary/vault.yml
- name: Update git submodules.
working-directory: 'infrastructure-playbook'
run: |
git submodule update --init --recursive --remote --checkout
- name: Set up Python 3.
uses: actions/setup-python@v4
with:
python-version: '3.11'
cache: 'pip'
# Install Ansible.
- name: Cache Ansible.
id: cache-ansible
uses: actions/cache@v3
with:
path: /opt/hostedtoolcache/Python/*/*/lib/python*/site-packages/ansible*
key: ${{ hashFiles('infrastructure-playbook/requirements.txt') }}
- name: Install Ansible.
working-directory: 'infrastructure-playbook'
run: |
# Install an Ansible version compatible with the version of
# ansible-core specified in requirements.txt for the
# infrastructure-playbook repo.
ANSIBLE_CORE_REQ=$(perl -pe 's/\\\n/ /' requirements.txt | grep ansible-core)
pip3 install --use-feature=fast-deps ansible "$ANSIBLE_CORE_REQ"
- name: Save Ansible cache.
uses: actions/cache/save@v3
if: ${{ steps.cache-ansible.outputs.cache-hit != 'true' }}
with:
path: /opt/hostedtoolcache/Python/*/*/lib/python*/site-packages/ansible*
key: ${{ hashFiles('infrastructure-playbook/requirements.txt') }}
# Total Perspective Vortex needs the Galaxy logic, which should be
# installed automatically when running `pip3 install
# total-perspective-vortex[cli]` (the `galaxy-app` package).
# However:
# - `galaxy-app` package on PyPI is outdated (see issue #15999 on
# the Galaxy repo: https://github.com/galaxyproject/galaxy/issues/15999)
# - Ideally the version of Galaxy should exactly match the one running on
# usegalaxy.eu.
# Therefore, we clone Galaxy and add it to the PYTHONPATH.
- name: Get Galaxy repo and commit id.
id: commits-galaxy
working-directory: 'infrastructure-playbook'
run: |
# Get the Galaxy repository URL and commit from Ansible variables.
export TMP_FILE=`mktemp`
openssl rand -base64 24 > .vault_password
ansible localhost --connection local \
--inventory hosts --module-name copy \
--args "content={{hostvars['sn06.galaxyproject.eu']}} dest=${TMP_FILE}" \
> /dev/null
export GALAXY_COMMIT_ID=$(cat ${TMP_FILE} | jq -r .galaxy_commit_id)
export GALAXY_REPO=$(cat ${TMP_FILE} | jq -r .galaxy_repo)
rm ${TMP_FILE}
echo "commit=$GALAXY_COMMIT_ID" >> $GITHUB_OUTPUT
echo "repo=$GALAXY_REPO" >> $GITHUB_OUTPUT
- name: Cache Galaxy
id: cache-galaxy
uses: actions/cache@v3
with:
path: galaxy
key: ${{ steps.commits-galaxy.outputs.repo }}-${{ steps.commits-galaxy.outputs.commit }}
- name: Clone Galaxy
if: ${{ steps.cache-galaxy.outputs.cache-hit != 'true' }}
run: |
# git clone -b ${{ steps.commits-galaxy.outputs.commit }} \
# --single-branch \
# --depth=1 ${{ steps.commits-galaxy.outputs.repo }} \
# galaxy # does not work with commit hashes
git clone ${{ steps.commits-galaxy.outputs.repo }} galaxy && cd galaxy && git checkout ${{ steps.commits-galaxy.outputs.commit }}
- name: Save Galaxy cache.
uses: actions/cache/save@v3
if: ${{ steps.cache-galaxy.outputs.cache-hit != 'true' }}
with:
path: galaxy
key: ${{ steps.commits-galaxy.outputs.repo }}-${{ steps.commits-galaxy.outputs.commit }}
- name: Install Galaxy requirements.
working-directory: 'galaxy'
run: pip install -r requirements.txt
# Install the Total Perspective Vortex version that should be running on
# usegalaxy.eu
- name: Install Total Perspective Vortex.
working-directory: 'galaxy'
run: |
TPV_REQ=$(perl -pe 's/\\\n/ /' lib/galaxy/dependencies/conditional-requirements.txt | grep total-perspective-vortex)
pip3 install --upgrade "$TPV_REQ"
- name: Install port of Ansible filters for Jinja.
run: |
pip3 install jinja2-ansible-filters
- name: Create mounts vars file.
working-directory: 'infrastructure-playbook/mounts'
run: |
make dest/all.yml
- name: Get commit ids before/after push or pull request.
id: commits-infrastructure-playbook
working-directory: 'infrastructure-playbook'
run: |
set -Eeo pipefail
if ${{ github.event_name == 'pull_request' }}; then
echo "before=$(git rev-parse HEAD^1)" >> $GITHUB_OUTPUT
echo "after=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT
else
echo "before=${{ github.event.before }}" >> $GITHUB_OUTPUT
echo "after=${{ github.event.after }}" >> $GITHUB_OUTPUT
fi
- name: Create playbooks to template the TPV files.
id: playbooks
shell: python -u {0}
working-directory: 'infrastructure-playbook'
run: |
import glob
import importlib.util
import os
import subprocess
import sys
# import make_playbook from tpv.py
spec = importlib.util.spec_from_file_location('tpv_ci', '.github/workflows/tpv.py')
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
make_playbook = module.make_playbook
tpv_path = 'files/galaxy/tpv/'
templates = tuple(
# TPV configuration files
(file, '.'.join(file.split('.')[:-1])) # remove j2 extension
for file in glob.glob(f'{tpv_path}/*.yaml.j2') + glob.glob(f'{tpv_path}/*.yml.j2')
) + (
# job_conf.yml
('templates/galaxy/config/job_conf.yml.j2', 'templates/galaxy/config/job_conf.yml'),
)
# template files based on the new version
playbook = make_playbook('sn06.yml', templates=templates)
with open(os.environ["GITHUB_OUTPUT"], "a") as file:
file.write(f"new={playbook}\n")
# template files based on the old version
subprocess.run(
["git", "checkout", "${{ steps.commits-infrastructure-playbook.outputs.before }}"]
)
playbook = make_playbook('sn06.yml', templates=templates)
with open(os.environ["GITHUB_OUTPUT"], "a") as file:
file.write(f"old={playbook}\n")
- name: Render TPV configuration files.
run: |
shopt -s nullglob
PLAYBOOK="${{ steps.playbooks.outputs.old }}"
BASENAME="$(basename $PLAYBOOK)"
DIRNAME="$(dirname $PLAYBOOK)"
cd "$DIRNAME"
ansible-playbook --connection=local "$BASENAME"
PLAYBOOK="${{ steps.playbooks.outputs.new }}"
BASENAME="$(basename $PLAYBOOK)"
DIRNAME="$(dirname $PLAYBOOK)"
cd "$DIRNAME"
ansible-playbook --connection=local "$BASENAME"
- name: Change paths of TPV configuration files in job_conf.yml
shell: python -u {0}
run: |
import os
from pathlib import Path
from urllib.parse import urlparse
import yaml
repository = Path("${{ steps.playbooks.outputs.new }}").parent
job_conf_path = Path('templates/galaxy/config/job_conf.yml')
tpv_path = Path('files/galaxy/tpv/')
job_conf = yaml.safe_load(open(repository / job_conf_path, 'r'))
tpv_config_files = job_conf['execution']['environments']['tpv_dispatcher']['tpv_config_files']
for i, file in enumerate(tpv_config_files):
if urlparse(file).scheme in {'file', ''}:
tpv_config_files[i] = str(repository / tpv_path / Path(file).name)
job_conf['execution']['environments']['tpv_dispatcher']['tpv_config_files'] = tpv_config_files
yaml.dump(job_conf, open(repository / job_conf_path, 'w'))
- name: Detect tools that have changed.
id: tools-changed
shell: python -u {0}
run: |
import glob
import hashlib
import os
from pathlib import Path
import yaml
playbooks = {
'old': Path("${{ steps.playbooks.outputs.old }}"),
'new': Path("${{ steps.playbooks.outputs.new }}"),
}
os.chdir(playbooks['new'].parent)
tpv_path = Path('files/galaxy/tpv/')
changed = set()
for file in (Path(path) for path in glob.glob(f"{tpv_path}/*.yml") + glob.glob(f"{tpv_path}/*.yaml")):
comparison = {}
for key, playbook in playbooks.items():
comparison[key] = yaml.safe_load(open(playbooks[key].parent / file, 'r')).get("tools", {})
comparison[key] = {
key: hashlib.sha256(yaml.dump(value, sort_keys=True).encode('utf-8')).hexdigest()
for key, value in comparison[key].items()
}
changed |= set(comparison['new']) - set(comparison['old'])
changed |= {key for key in set(comparison['new']) & set(comparison['old']) if comparison['new'][key] != comparison['old'][key]}
with open(os.environ["GITHUB_OUTPUT"], 'a') as file:
file.write(f"changed={' '.join(changed)}\n")
- name: Run Total Perspective Vortex dry-run.
env:
TOOLS: ${{ steps.tools-changed.outputs.changed }}
PLAYBOOK: ${{ steps.playbooks.outputs.new }}
run: |
set -Eeo pipefail
shopt -s nullglob
export PYTHONPATH=$(realpath ./galaxy/lib)
BASENAME="$(basename $PLAYBOOK)"
DIRNAME="$(dirname $PLAYBOOK)"
cd "$DIRNAME"
IFS=" " read -a tools <<< "$TOOLS"
for tool in "${tools[@]}"; do
echo Running TPV dry-run for "$tool..."
tpv dry-run --job-conf templates/galaxy/config/job_conf.yml --tool "$tool"
done