Alma9 build v5 frozen release #10
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Alma9 Build frozen release | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| release: | |
| description: 'targeted release (e.g. v4.1.0)' | |
| required: true | |
| build-number: | |
| description: 'frozen release build number, e.g. 1' | |
| required: true | |
| jobs: | |
| build_the_frozen_release_spack: | |
| name: build_frozen_release_spack | |
| #runs-on: ubuntu-latest | |
| runs-on: isc01 | |
| #runs-on: np04-build-alma9 | |
| container: | |
| image: ghcr.io/dune-daq/alma9-slim-externals:v2.0 | |
| defaults: | |
| run: | |
| shell: bash | |
| steps: | |
| - name: Checkout daq-release | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: DUNE-DAQ/daq-release | |
| path: daq-release | |
| - name: setup directories and install spack for the dunedaq release | |
| run: | | |
| export SPACK_VERSION=0.20.0 | |
| export GCC_VERSION=12.1.0 | |
| export EXT_VERSION=v2.0 | |
| export SPACK_EXTERNALS=/cvmfs/dunedaq.opensciencegrid.org/spack/externals/ext-${EXT_VERSION}/spack-$SPACK_VERSION-gcc-$GCC_VERSION | |
| export DAQ_RELEASE_CONF=dunedaq-${{ github.event.inputs.release }} | |
| export DAQ_RELEASE=dunedaq-${{ github.event.inputs.release }}-a9 | |
| export BUILD_NUMBER=${{ github.event.inputs.build-number }} | |
| export SPACK_RELEASE_DIR=/cvmfs/dunedaq.opensciencegrid.org/spack/releases/$DAQ_RELEASE | |
| export SPACK_RELEASE=$SPACK_RELEASE_DIR/spack-$SPACK_VERSION-gcc-$GCC_VERSION-b${BUILD_NUMBER} | |
| mkdir -p ${SPACK_RELEASE} | |
| cd ${SPACK_RELEASE} | |
| wget https://github.com/spack/spack/archive/refs/tags/v${SPACK_VERSION}.tar.gz | |
| tar xf v${SPACK_VERSION}.tar.gz | |
| ln -s spack-${SPACK_VERSION} spack-installation | |
| rm -f v${SPACK_VERSION}.tar.gz | |
| export DAQ_RELEASE_DIR=$GITHUB_WORKSPACE/daq-release | |
| cd $DAQ_RELEASE_DIR | |
| python3 scripts/spack/make-release-repo.py -u \ | |
| -i configs/dunedaq/${DAQ_RELEASE_CONF}/release.yaml \ | |
| -t spack-repos/dunedaq-repo-template \ | |
| -r ${DAQ_RELEASE} \ | |
| -o ${SPACK_RELEASE}/spack-${SPACK_VERSION} | |
| cd $SPACK_RELEASE | |
| source spack-${SPACK_VERSION}/share/spack/setup-env.sh | |
| echo "*********** spack arch ************ " | |
| spack arch | |
| cat <<EOT > $SPACK_ROOT/etc/spack/defaults/repos.yaml | |
| repos: | |
| - ${SPACK_RELEASE}/spack-${SPACK_VERSION}/spack-repo | |
| - ${SPACK_EXTERNALS}/spack-${SPACK_VERSION}/spack-repo-externals | |
| - \$spack/var/spack/repos/builtin | |
| EOT | |
| spack repo list | |
| cat <<EOT >> $SPACK_ROOT/etc/spack/defaults/upstreams.yaml | |
| upstreams: | |
| spack-externals: | |
| install_tree: ${SPACK_EXTERNALS}/spack-${SPACK_VERSION}/opt/spack | |
| EOT | |
| cp $SPACK_EXTERNALS/spack-${SPACK_VERSION}/etc/spack/defaults/linux/compilers.yaml \ | |
| $SPACK_RELEASE/spack-${SPACK_VERSION}/etc/spack/defaults/linux/ | |
| # verify the compilers have been added | |
| spack compiler list | |
| \cp $SPACK_EXTERNALS/spack-${SPACK_VERSION}/etc/spack/defaults/config.yaml $SPACK_RELEASE/spack-${SPACK_VERSION}/etc/spack/defaults/config.yaml | |
| \cp $SPACK_EXTERNALS/spack-${SPACK_VERSION}/etc/spack/defaults/modules.yaml $SPACK_RELEASE/spack-${SPACK_VERSION}/etc/spack/defaults/modules.yaml | |
| sed -i 's/host_compatible: true/host_compatible: false/g' $SPACK_ROOT/etc/spack/defaults/concretizer.yaml | |
| spack spec --reuse dunedaq@${DAQ_RELEASE}%gcc@12.1.0 build_type=RelWithDebInfo arch=linux-almalinux9-x86_64 | tee $SPACK_RELEASE/spec_dunedaq_log.txt | |
| cd $SPACK_RELEASE_DIR/../ | |
| tar zcf spack-${DAQ_RELEASE}.tar.gz ${DAQ_RELEASE} | |
| mkdir -p $GITHUB_WORKSPACE/frozen-spack-dunedaq | |
| rm -rf $GITHUB_WORKSPACE/frozen-spack-dunedaq/* | |
| mv spack-${DAQ_RELEASE}.tar.gz $GITHUB_WORKSPACE/frozen-spack-dunedaq | |
| - name: upload empty spack tarball for dunedaq | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: frozen_spack_dunedaq | |
| path: ${{ github.workspace }}/frozen-spack-dunedaq | |
| - name: install spack for the dunedaq release | |
| run: | | |
| export SPACK_VERSION=0.20.0 | |
| export GCC_VERSION=12.1.0 | |
| export EXT_VERSION=v2.0 | |
| export SPACK_EXTERNALS=/cvmfs/dunedaq.opensciencegrid.org/spack/externals/ext-${EXT_VERSION}/spack-$SPACK_VERSION-gcc-$GCC_VERSION | |
| export DAQ_RELEASE=dunedaq-${{ github.event.inputs.release }}-a9 | |
| export BUILD_NUMBER=${{ github.event.inputs.build-number }} | |
| export SPACK_RELEASE_DIR=/cvmfs/dunedaq.opensciencegrid.org/spack/releases/$DAQ_RELEASE | |
| export SPACK_RELEASE=$SPACK_RELEASE_DIR/spack-$SPACK_VERSION-gcc-$GCC_VERSION-b${BUILD_NUMBER} | |
| cd $SPACK_RELEASE | |
| source spack-${SPACK_VERSION}/share/spack/setup-env.sh | |
| spack install --reuse dunedaq@${DAQ_RELEASE}%gcc@12.1.0 dbe@v1.0.1%gcc@12.1.0 build_type=RelWithDebInfo arch=linux-almalinux9-x86_64 | |
| spack clean -a | |
| cd $SPACK_RELEASE_DIR/../ | |
| tar zcf ${DAQ_RELEASE}.tar.gz ${DAQ_RELEASE} | |
| mkdir -p $GITHUB_WORKSPACE/frozen-releases | |
| rm -rf $GITHUB_WORKSPACE/frozen-releases/* | |
| mv ${DAQ_RELEASE}.tar.gz $GITHUB_WORKSPACE/frozen-releases | |
| - name: upload spack frozen release tarball for dunedaq | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: frozens_dunedaq | |
| path: ${{ github.workspace }}/frozen-releases | |
| - name: setup directories and install spack for the fddaq release | |
| run: | | |
| export DET=fd | |
| export SPACK_VERSION=0.20.0 | |
| export GCC_VERSION=12.1.0 | |
| export EXT_VERSION=v2.0 | |
| export SPACK_EXTERNALS=/cvmfs/dunedaq.opensciencegrid.org/spack/externals/ext-${EXT_VERSION}/spack-$SPACK_VERSION-gcc-$GCC_VERSION | |
| export DAQ_BASE_RELEASE=dunedaq-${{ github.event.inputs.release }}-a9 | |
| export DAQ_RELEASE_CONF=${DET}daq-${{ github.event.inputs.release }} | |
| export DAQ_RELEASE=${DET}daq-${{ github.event.inputs.release }}-a9 | |
| export BUILD_NUMBER=${{ github.event.inputs.build-number }} | |
| export SPACK_RELEASE_DIR=/cvmfs/dunedaq.opensciencegrid.org/spack/releases/$DAQ_RELEASE | |
| export SPACK_RELEASE=$SPACK_RELEASE_DIR/spack-$SPACK_VERSION-gcc-$GCC_VERSION-b${BUILD_NUMBER} | |
| export SPACK_BASE_RELEASE_DIR=/cvmfs/dunedaq.opensciencegrid.org/spack/releases/$DAQ_BASE_RELEASE | |
| export SPACK_BASE_RELEASE=$SPACK_BASE_RELEASE_DIR/spack-$SPACK_VERSION-gcc-$GCC_VERSION-b${BUILD_NUMBER} | |
| mkdir -p ${SPACK_RELEASE} | |
| cd ${SPACK_RELEASE} | |
| wget https://github.com/spack/spack/archive/refs/tags/v${SPACK_VERSION}.tar.gz | |
| tar xf v${SPACK_VERSION}.tar.gz | |
| ln -s spack-${SPACK_VERSION} spack-installation | |
| rm -f v${SPACK_VERSION}.tar.gz | |
| export DAQ_RELEASE_DIR=$GITHUB_WORKSPACE/daq-release | |
| cd $DAQ_RELEASE_DIR | |
| python3 scripts/spack/make-release-repo.py -u \ | |
| -i configs/${DET}daq/${DAQ_RELEASE_CONF}/release.yaml \ | |
| -t spack-repos/${DET}daq-repo-template \ | |
| -r ${DAQ_RELEASE} \ | |
| --base-release ${DAQ_BASE_RELEASE} \ | |
| -o ${SPACK_RELEASE}/spack-${SPACK_VERSION} | |
| cd $SPACK_RELEASE | |
| source spack-${SPACK_VERSION}/share/spack/setup-env.sh | |
| cat <<EOT > $SPACK_ROOT/etc/spack/defaults/repos.yaml | |
| repos: | |
| - ${SPACK_RELEASE}/spack-${SPACK_VERSION}/spack-repo | |
| - ${SPACK_BASE_RELEASE}/spack-${SPACK_VERSION}/spack-repo | |
| - ${SPACK_EXTERNALS}/spack-${SPACK_VERSION}/spack-repo-externals | |
| - \$spack/var/spack/repos/builtin | |
| EOT | |
| spack repo list | |
| cat <<EOT >> $SPACK_ROOT/etc/spack/defaults/upstreams.yaml | |
| upstreams: | |
| ${DAQ_BASE_RELEASE}: | |
| install_tree: ${SPACK_BASE_RELEASE}/spack-${SPACK_VERSION}/opt/spack | |
| spack-externals: | |
| install_tree: ${SPACK_EXTERNALS}/spack-${SPACK_VERSION}/opt/spack | |
| EOT | |
| cp $SPACK_EXTERNALS/spack-${SPACK_VERSION}/etc/spack/defaults/linux/compilers.yaml \ | |
| $SPACK_RELEASE/spack-${SPACK_VERSION}/etc/spack/defaults/linux/ | |
| spack compiler list | |
| \cp $SPACK_EXTERNALS/spack-${SPACK_VERSION}/etc/spack/defaults/config.yaml $SPACK_RELEASE/spack-${SPACK_VERSION}/etc/spack/defaults/config.yaml | |
| \cp $SPACK_EXTERNALS/spack-${SPACK_VERSION}/etc/spack/defaults/modules.yaml $SPACK_RELEASE/spack-${SPACK_VERSION}/etc/spack/defaults/modules.yaml | |
| sed -i 's/host_compatible: true/host_compatible: false/g' $SPACK_ROOT/etc/spack/defaults/concretizer.yaml | |
| spack spec --reuse ${DET}daq@${DAQ_RELEASE}%gcc@12.1.0 build_type=RelWithDebInfo arch=linux-almalinux9-x86_64 | tee $SPACK_RELEASE/spec_${DET}daq_log.txt | |
| spack install --reuse ${DET}daq@${DAQ_RELEASE}%gcc@12.1.0 build_type=RelWithDebInfo arch=linux-almalinux9-x86_64 | |
| # Generate pyvenv_requirements.txt | |
| spack load ${DET}daq@${DAQ_RELEASE} | |
| /usr/bin/python3 $DAQ_RELEASE_DIR/scripts/spack/make-release-repo.py \ | |
| -o ${SPACK_RELEASE}/../ \ | |
| --pyvenv-requirements \ | |
| -i $DAQ_RELEASE_DIR/configs/${DET}daq/${DAQ_RELEASE_CONF}/release.yaml | |
| python -m venv --prompt dbt ${SPACK_RELEASE}/../.venv | |
| source ${SPACK_RELEASE}/../.venv/bin/activate | |
| python -m pip install -r ${SPACK_RELEASE}/../pyvenv_requirements.txt | |
| spack clean -a | |
| pushd ${SPACK_RELEASE_DIR} | |
| cp $DAQ_RELEASE_DIR/configs/${DET}daq/${DET}daq-develop/dbt-build-order.cmake . | |
| tar zcf venv.tar.gz .venv/ | |
| popd | |
| cd $SPACK_RELEASE_DIR/../ | |
| tar zcf ${DAQ_RELEASE}.tar.gz ${DAQ_RELEASE} | |
| mkdir -p $GITHUB_WORKSPACE/frozens-${DET}daq | |
| rm -rf $GITHUB_WORKSPACE/frozens-${DET}daq/* | |
| mv ${DAQ_RELEASE}.tar.gz $GITHUB_WORKSPACE/frozens-${DET}daq | |
| - name: upload spack frozen release tarball for fddaq | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: frozens_fddaq | |
| path: ${{ github.workspace }}/frozens-fddaq | |
| update_image: | |
| name: update_spack_image_rc | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| include: | |
| - image: "ghcr.io/dune-daq/alma9-slim-externals" | |
| os_name: "al9" | |
| needs: build_the_frozen_release_spack | |
| environment: dockerhub | |
| permissions: | |
| packages: write | |
| contents: read | |
| steps: | |
| - name: clean docker-build | |
| run: | | |
| mkdir -p ${GITHUB_WORKSPACE}/docker-build | |
| rm -rf ${GITHUB_WORKSPACE}/docker-build/* | |
| - name: Download spack frozen release tarball artifact for dunedaq | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: frozens_dunedaq | |
| path: ${{ github.workspace }}/docker-build | |
| - name: Download spack frozen release tarball artifact for fddaq | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: frozens_fddaq | |
| path: ${{ github.workspace }}/docker-build | |
| - name: prepare cvmfs mirror spack-frozens | |
| run: | | |
| cd ${{ github.workspace }}/docker-build | |
| mkdir -p releases | |
| export EXT_VERSION=v2.0 | |
| image_name="${{ matrix.image }}:${EXT_VERSION}" | |
| cd releases | |
| dunedaq_tag=dunedaq-${{ github.event.inputs.release }}-a9 | |
| tar xvf ../${dunedaq_tag}.tar.gz | |
| rm -rf ../${dunedaq_tag}.tar.gz | |
| fddaq_tag=fddaq-${{ github.event.inputs.release }}-a9 | |
| tar xvf ../${fddaq_tag}.tar.gz | |
| rm -rf ../${fddaq_tag}.tar.gz | |
| cd ${fddaq_tag}; ln -s spack-*-gcc-* default; cd .. | |
| cd .. | |
| echo "FROM ${image_name}"> Dockerfile | |
| echo 'MAINTAINER Pengfei Ding "dingpf@fnal.gov"' >> Dockerfile | |
| echo "ENV REFRESHED_FOR ${fddaq_tag}" >> Dockerfile | |
| echo "COPY --from=ghcr.io/dune-daq/pypi-repo:latest /cvmfs/dunedaq.opensciencegrid.org/pypi-repo /cvmfs/dunedaq.opensciencegrid.org/pypi-repo" >> Dockerfile | |
| echo "ADD releases /cvmfs/dunedaq.opensciencegrid.org/spack/releases" >> Dockerfile | |
| echo 'ENTRYPOINT ["/bin/bash"]' >> Dockerfile | |
| - name: Log in to the Container registry | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Extract metadata (tags, labels) for Docker | |
| id: meta | |
| uses: docker/metadata-action@v5 | |
| with: | |
| images: | | |
| ${{ matrix.image }} | |
| tags: | | |
| type=raw,value=spack-dev-frozen-split | |
| - name: Build and push Docker images | |
| uses: docker/build-push-action@v5 | |
| with: | |
| context: ${{ github.workspace }}/docker-build | |
| push: true | |
| tags: ${{ steps.meta.outputs.tags }} | |
| labels: ${{ steps.meta.outputs.labels }} | |
| generate_dbt_setup_release_env: | |
| name: generate_dbt_setup_release_env_${{ matrix.os_name }} | |
| runs-on: ubuntu-latest | |
| needs: update_image | |
| container: | |
| image: "ghcr.io/dune-daq/alma9-slim-externals:spack-dev-frozen-split" | |
| defaults: | |
| run: | |
| shell: bash | |
| steps: | |
| - name: create dbt-setup-release-env.sh and daq_app_rte.sh for fddaq | |
| run: | | |
| DET=fd | |
| daq_tag=${DET}daq-${{ github.event.inputs.release }}-a9 | |
| source /cvmfs/dunedaq.opensciencegrid.org/setup_dunedaq.sh | |
| setup_dbt latest|| true | |
| dbt-setup-release ${daq_tag} | |
| declare -x > ${GITHUB_WORKSPACE}/${DET}daq-dbt-setup-release-env.sh | |
| declare -f >> ${GITHUB_WORKSPACE}/${DET}daq-dbt-setup-release-env.sh | |
| egrep "declare -x (PATH|.*_SHARE|CET_PLUGIN_PATH|DUNEDAQ_SHARE_PATH|LD_LIBRARY_PATH|LIBRARY_PATH|PYTHONPATH)=" ${GITHUB_WORKSPACE}/${DET}daq-dbt-setup-release-env.sh > ${GITHUB_WORKSPACE}/${DET}daq_app_rte.sh | |
| - name: upload fddaq-dbt-setup-release-env.sh | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: fddaq-dbt_setup_release_env | |
| path: ${{ github.workspace }}/fddaq-dbt-setup-release-env.sh | |
| - name: upload fddaq_app_rte.sh | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: fddaq_app_rte | |
| path: ${{ github.workspace }}/fddaq_app_rte.sh |