Update pypi-repo #1043
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Update pypi-repo | |
| on: | |
| schedule: | |
| - cron: "0 1 * * *" | |
| # update pypi-repo "0 1 * * *" | |
| # publish pypi-repo to cvmfs "30 21 * * *(CST/CDT)" 30 1/2 * * * | |
| # c8-cvmfs docker build on "0 3 * * *" | |
| # sl7-cvmfs docker build on "0 4 * * *" | |
| # nighlty release build on "0 5 * * *" | |
| # publish to cvmfs on "0 1 * * *(CST/CDT)" 0 6/7 * * * | |
| # package CI build on "0 7 * * *" | |
| workflow_dispatch: | |
| jobs: | |
| update_pypi_repo: | |
| name: update_pypi_repo | |
| runs-on: ubuntu-latest | |
| container: | |
| image: ghcr.io/dune-daq/alma9-slim-externals:v2.0 | |
| defaults: | |
| run: | |
| shell: bash | |
| steps: | |
| - name: Checkout daq-release | |
| uses: actions/checkout@v3 | |
| with: | |
| repository: DUNE-DAQ/daq-release | |
| path: daq-release | |
| - name: install pip2pi | |
| run: | | |
| mkdir -p $GITHUB_WORKSPACE/workdir | |
| cd $GITHUB_WORKSPACE/workdir | |
| source /cvmfs/dunedaq.opensciencegrid.org/spack/externals/ext-v2.0/spack-installation/share/spack/setup-env.sh | |
| spack load python@3.10.10 | |
| python -m venv pypi-repo-venv | |
| source pypi-repo-venv/bin/activate | |
| echo "activated pyvenv" | |
| pip install --upgrade pip | |
| pip install https://github.com/dingp/pip2pi/archive/1.1.2.tar.gz | |
| - name: parse release_manifest and install all pkgs | |
| run: | | |
| cd $GITHUB_WORKSPACE/workdir | |
| source /cvmfs/dunedaq.opensciencegrid.org/spack/externals/ext-v2.0/spack-installation/share/spack/setup-env.sh | |
| spack load python@3.10.10 | |
| spack load go | |
| source pypi-repo-venv/bin/activate | |
| manifest_file="$GITHUB_WORKSPACE/daq-release/configs/fddaq/fddaq-develop/release.yaml" | |
| repo_path="/cvmfs/dunedaq.opensciencegrid.org/pypi-repo" | |
| $GITHUB_WORKSPACE/daq-release/scripts/create-pypi-repo.sh $manifest_file $repo_path | |
| du -sh $repo_path | |
| rsync -rvplt $repo_path $GITHUB_WORKSPACE | |
| cd $GITHUB_WORKSPACE | |
| tar zcvf pypi-repo.tar.gz pypi-repo | |
| du -sh pypi-repo.tar.gz | |
| - name: upload pypi-repo | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: pypi-repo | |
| path: ${{ github.workspace }}/pypi-repo.tar.gz | |
| - name: generaete pyvenv_requirements.txt and test installing dbt-pyvenv | |
| run: | | |
| cd $GITHUB_WORKSPACE/workdir | |
| python3 $GITHUB_WORKSPACE/daq-release/scripts/spack/make-release-repo.py -o $GITHUB_WORKSPACE --pyvenv-requirements -i $GITHUB_WORKSPACE/daq-release/configs/fddaq/fddaq-develop/release.yaml | |
| source /cvmfs/dunedaq.opensciencegrid.org/spack/externals/ext-v2.0/spack-installation/share/spack/setup-env.sh | |
| spack load python@3.10.10 | |
| spack load go | |
| python -m venv dbt-pyvenv | |
| source dbt-pyvenv/bin/activate | |
| python -m pip install -r $GITHUB_WORKSPACE/pyvenv_requirements.txt | |
| - name: generaete pip freeze | |
| run: | | |
| cd $GITHUB_WORKSPACE/workdir | |
| source /cvmfs/dunedaq.opensciencegrid.org/spack/externals/ext-v2.0/spack-installation/share/spack/setup-env.sh | |
| spack load python@3.10.10 | |
| source dbt-pyvenv/bin/activate | |
| pip freeze > $GITHUB_WORKSPACE/pip-freeze.txt | |
| - name: upload pyvenv_requirements.txt | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: pyvenv_requirements | |
| path: ${{ github.workspace }}/pyvenv_requirements.txt | |
| - name: upload pip-freeze.txt | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: pip_freeze | |
| path: ${{ github.workspace }}/pip-freeze.txt | |
| update_image: | |
| name: update_pypi_image | |
| runs-on: ubuntu-latest | |
| needs: update_pypi_repo | |
| permissions: | |
| packages: write | |
| contents: read | |
| steps: | |
| - name: Log in to the Container registry | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Extract metadata (tags, labels) for Docker | |
| id: meta | |
| uses: docker/metadata-action@v5 | |
| with: | |
| images: | | |
| ghcr.io/dune-daq/pypi-repo | |
| tags: | | |
| type=raw,value=latest | |
| type=schedule,pattern={{date 'YYYYMMDD'}} | |
| - name: Download pypi repo tarball artifact | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: pypi-repo | |
| path: ${{ github.workspace }}/docker-build | |
| - name: prepare cvmfs mirror pypi-repo | |
| run: | | |
| cd ${{ github.workspace }}/docker-build | |
| tar xvf pypi-repo.tar.gz | |
| rm -f pypi-repo.tar.gz | |
| nightly_tag="$(date +%y-%m-%d)" | |
| echo "FROM quay.io/centos/centos:stream8"> Dockerfile | |
| echo 'MAINTAINER John Freeman "jcfree@fnal.gov"' >> Dockerfile | |
| echo "ENV REFRESHED_AT $(date +%y-%m-%d)" >> Dockerfile | |
| echo "ADD pypi-repo /cvmfs/dunedaq.opensciencegrid.org/pypi-repo" >> Dockerfile | |
| echo 'ENTRYPOINT ["/bin/bash"]' >> Dockerfile | |
| - name: Build and push Docker images | |
| uses: docker/build-push-action@v5 | |
| with: | |
| context: ${{ github.workspace }}/docker-build | |
| push: true | |
| tags: ${{ steps.meta.outputs.tags }} | |
| labels: ${{ steps.meta.outputs.labels }} |