Merge pull request #42 from VERITAS-Observatory/v2.0-optimisation-dev3

v2.0-dev3 bug fixes

.github/workflows/build-images.yml

@@ -26,7 +26,7 @@ jobs:
       packages: write
     strategy:
       matrix:
-        type: ['corsika', 'corsika-noopt', 'corsika-ax2', 'groptics', 'care', 'mergevbf']
+        type: ['corsika', 'groptics', 'care', 'mergevbf']
 
     steps:
       - name: Checkout repository

.github/workflows/build-optimized-corsika.yml

@@ -26,7 +26,7 @@ jobs:
       packages: write
     strategy:
       matrix:
-        type: ['noopt', 'avx2', 'avx512', 'avx', 'sse4']
+        type: ['noopt', 'avx2', 'avx512f', 'avx', 'sse4']
 
     steps:
       - name: Checkout repository

CITATION.cff

@@ -4,9 +4,9 @@ cff-version: 1.2.0
 message: If you use this software, please cite it using these metadata.
 title: "VTS-SimPipe: simulation tools for the VERITAS observatory"
 doi: 10.5281/zenodo.10541349
-version: 1.0.0
+version: 2.0.0
 type: software
-date-released: 2024-01-22
+date-released: 2024-03-03
 keywords:
 - "gamma-ray astronomy"
 - "astronomy software"

README.md

@@ -3,6 +3,7 @@
 [![DOI](https://zenodo.org/badge/738007615.svg)](https://zenodo.org/doi/10.5281/zenodo.10541349)
 [![License](https://img.shields.io/badge/License-BSD_3--Clause-blue.svg)](https://opensource.org/licenses/BSD-3-Clause)
 [![build-images](https://github.com/VERITAS-Observatory/VTS-SimPipe/actions/workflows/build-images.yml/badge.svg)](https://github.com/VERITAS-Observatory/VTS-SimPipe/actions/workflows/build-images.yml)
+[![build-optimized-corsika](https://github.com/VERITAS-Observatory/VTS-SimPipe/actions/workflows/build-optimized-corsika.yml/badge.svg)](https://github.com/VERITAS-Observatory/VTS-SimPipe/actions/workflows/build-optimized-corsika.yml)
 
 [VERITAS](https://veritas.sao.arizona.edu/) is a ground-based gamma-ray observatory located at the Fred Lawrence Whipple Observatory in southern Arizona, USA.
 It explores the gamma-ray sky in the energy range from 100 GeV to 30 TeV.
@@ -22,7 +23,7 @@ This work is built on a large effort from many people, especially:
 - Nepomuk Otte for the [CARE](https://github.com/nepomukotte/CARE) package
 - Raul Prado for an initial pipeline implementation for DESY (see [here](https://github.com/RaulRPrado/MC-DESY/tree/master))
 - Tony Lin for a Docker implementation of the pipeline (see [here](https://github.com/VERITAS-Observatory/Build_SimDockerImage/tree/master))
-- Luisa Arrabito and Orel Gueta on providing the optimized CORSIKA code and help with compilation
+- Luisa Arrabito and Orel Gueta on providing the optimized CORSIKA code and help with compilation issues.
 
 ## Quick startup
 
@@ -42,9 +43,7 @@ cd scripts && ./pull.sh
 # see example in config/config_ATM61_template.dat
 # prepare production
 cd scripts
-./prepare_all_production_steps.sh \
-   ../config/config_ATM61_template.dat
-   ../config/CORSIKA/input_template.dat
+./prepare_all_production_steps.sh ../config/config_ATM61_template.dat
 # on DESY: log into the DAG submission node
 ./prepare_DAG_jobs.sh ../config/config_ATM61_template.dat
 ./submit_DAG_jobs.sh ../config/config_ATM61_template.dat <directory with DAG files>
@@ -81,9 +80,9 @@ Images can be downloaded from the package registry of this repository.
 Requires the tar package with the CORSIKA tar software to be available in the main directory of `VTSSimPipe`.
 Note that the CI on github will build three different containers for CORSIKA:
 
-1. [vts-simpipe-corsika](https://github.com/VERITAS-Observatory/VTS-SimPipe/pkgs/container/vtsimpipe-corsika) based on [docker/Dockerfile-corsika](docker/Dockerfile-corsika) with the standard CORSIKA software (as used in VERITAS for productions in the past)
-2. [vts-simpipe-corsika-noopt](https://github.com/VERITAS-Observatory/VTS-SimPipe/pkgs/container/vtsimpipe-corsika-noopt) based on [docker/Dockerfile-corsika-noopt](docker/Dockerfile-corsika-noopt) using CORSIKA 7.7500 with minor updates to the Bernlohr package (this is the package used for the generation and propagation of Cherenkov photons).
-3. [vts-simpipe-corsika-ax2](https://github.com/VERITAS-Observatory/VTS-SimPipe/pkgs/container/vtsimpipe-corsika-ax2) based on [docker/Dockerfile-corsika-ax2](docker/Dockerfile-corsika-ax2) using CORSIKA 7.7500 with minor updates to the Bernlohr package (this is the package used for the generation and propagation of Cherenkov photons). A patch is applied to the Cherenkov photon code to allow to use vector instructions and improve runtime performance.
+1. [vts-simpipe-corsika](https://github.com/VERITAS-Observatory/VTS-SimPipe/pkgs/container/vtsimpipe-corsika) based on [docker/Dockerfile-corsika](docker/Dockerfile-corsika) with the standard CORSIKA software (as used in VERITAS for productions in the past); configuration and compilation using the `coconut` tools.
+2. [vts-simpipe-corsika-noopt](https://github.com/VERITAS-Observatory/VTS-SimPipe/pkgs/container/vtsimpipe-corsika-noopt) based on [docker/Dockerfile-corsika-noopt](docker/Dockerfile-corsika-noopt) using CORSIKA 7.7500 used compile coptions as outlined in the Docker file (in contrast to 1., uses the `O3` flags, but it does not use the vectorization code of 3.)
+3. [vts-simpipe-corsika-ax2](https://github.com/VERITAS-Observatory/VTS-SimPipe/pkgs/container/vtsimpipe-corsika-ax2) based on [docker/Dockerfile-corsika-ax2](docker/Dockerfile-corsika-ax2) using CORSIKA 7.7500 with minor updates to the Bernlohr package (this is the package used for the generation and propagation of Cherenkov photons). A patch is applied to the Cherenkov photon code to allow to use vector instructions and improve runtime performance, see discussions in L. Arrabito et al, *Optimizing Cherenkov photons generation and propagation in CORSIKA for CTA Monte-Carlo simulations*, [arXiv.2006.14927](https://arxiv.org/abs/2006.14927)
 
 To build the CORSIKA container (similar for all):
 
@@ -296,3 +295,22 @@ SLANT 1
 ### corsikaIOreader parameters
 
 - apply pre-efficiency cut of 50%
+
+## Optimization of CORSIKA
+
+The CTA simulation pipeline uses vector-optimization of the Cherenkov photon generation and propagation code in CORSIKA with notable performance improvements, see the discussion in L. Arrabito et al, *Optimizing Cherenkov photons generation and propagation in CORSIKA for CTA Monte-Carlo simulations*, [arXiv.2006.14927](https://arxiv.org/abs/2006.14927).
+
+VTS-SimPipe is able to use the same optimized CORSIKA, thanks to for Luisa Arrabito for providing it. Below a comparison of the runtime of the optimized and non-optimized CORSIKA for the same simulation setup:
+
+| Type | CORSIKA version | IACT/ATMO version | Remarks | Run Time [s] / event | Ratio to VTS-SimPipe |
+| -------- | -------- | -------- | -------- | -------- | -------- |
+| VTS-SimPipe | 7.7500 | 1.66 (2023-02-03) | default coconut build (see remarks below); [container](https://github.com/VERITAS-Observatory/VTS-SimPipe/pkgs/container/vtsimpipe-corsika/177291090?tag=1.1.0), [docker](https://github.com/VERITAS-Observatory/VTS-SimPipe/blob/main/docker/Dockerfile-corsika); SLANT option; no VIEWCONE | 33.6 | 1. |
+| VTS-noopt | 7.7500 | 1.66 (2023-02-03) | no specific optimization (O3; see remarks below); [container](https://github.com/VERITAS-Observatory/VTS-SimPipe/pkgs/container/vtsimpipe-corsika-noopt/182679178?tag=20240223-115829) [docker (noopt arg)](https://github.com/VERITAS-Observatory/VTS-SimPipe/blob/main/docker/Dockerfile-corsika-optimized) | 17.2 | 2.0 |
+| VTS-avx2 | 7.7500 | 1.66 (2023-02-03) | avx2 [container](https://github.com/VERITAS-Observatory/VTS-SimPipe/pkgs/container/vtsimpipe-corsika-avx2/182679269?tag=20240223-115831), [docker (avx2 arg)](https://github.com/VERITAS-Observatory/VTS-SimPipe/blob/main/docker/Dockerfile-corsika-optimized) | 14.42 | 2.3 |
+| VTS-sse4 | 7.7500 | 1.66 (2023-02-03) | sse4 [container](https://github.com/VERITAS-Observatory/VTS-SimPipe/pkgs/container/vtsimpipe-corsika-sse4/182679278?tag=20240223-115831), [docker (sse4 arg)](https://github.com/VERITAS-Observatory/VTS-SimPipe/blob/main/docker/Dockerfile-corsika-optimized) | 12.5 | 2.7 |
+| VTS-avx512f | 7.7500 | 1.66 (2023-02-03) | avx51f [container](https://github.com/VERITAS-Observatory/VTS-SimPipe/pkgs/container/vtsimpipe-corsika-avx512/182679206?tag=20240223-115833), [docker (avx512 arg)](https://github.com/VERITAS-Observatory/VTS-SimPipe/blob/main/docker/Dockerfile-corsika-optimized) | 17.37 | 1.9 |
+
+### Remarks
+
+- default Coconut C compile flags (VTS-SimPipe): `cc -DHAVE_CONFIG_H -I. -I../include  -DMAX_IO_BUFFER=200000000 -DCORSIKA_VERSION=77500   -g -D_FILE_OFFSET_BITS=64 -MT libiact_a-eventio.o -MD -MP -MF .deps/libiact_a-eventio.Tpo -c -o libiact_a-eventio.o `test -f 'eventio.c'` (**no optimisation at all?**)
+- VTS-noopt compile flags: `cc -DHAVE_CONFIG_H -I. -I../include  -DMAX_IO_BUFFER=200000000 -DCORSIKA_VERSION=77500   -std=c99 -O3 -MT libiact_a-eventio.o -MD -MP -MF .deps/libiact_a-eventio.Tpo -c -o libiact_a-eventio.o `test -f 'eventio.c'

docker/Dockerfile-care

@@ -38,6 +38,9 @@ RUN mkdir CARE && tar -xzf v$CARE_VERSION.tar.gz -C CARE --strip-components=1 &&
 FROM rootproject/root:6.24.06-centos7
 LABEL maintainer.name="VERITAS Collaboration"
 LABEL maintainer.email="gernot.maier@desy.de"
+LABEL org.opencontainers.image.source="https://github.com/VERITAS-Observatory/VTS-SimPipe"
+LABEL org.opencontainers.image.description="CARE Camera simulations as used for the VERITAS simulation pipeline."
+LABEL org.opencontainers.image.licenses="BSD-3-Clause"
 WORKDIR /workdir
 COPY --from=build_image /workdir/VBF /workdir/VBF
 COPY --from=build_image /workdir/CARE /workdir/CARE

docker/Dockerfile-corsika

@@ -23,6 +23,9 @@ WORKDIR /workdir/
 FROM almalinux:9.3-minimal
 LABEL maintainer.name="VERITAS Collaboration"
 LABEL maintainer.email="gernot.maier@desy.de"
+LABEL org.opencontainers.image.source="https://github.com/VERITAS-Observatory/VTS-SimPipe"
+LABEL org.opencontainers.image.description="CORSIKA 7.7500 (non-optimized version) as used for the VERITAS simulation pipeline."
+LABEL org.opencontainers.image.licenses="BSD-3-Clause"
 WORKDIR /workdir
 ENV DEBIAN_FRONTEND=noninteractive
 ARG CORSIKA=corsika-77500

docker/Dockerfile-corsika-optimized

@@ -28,34 +28,23 @@ WORKDIR /workdir/corsika-77500
 RUN ./coconut < /dev/null
 ENV GCC_PATH="/usr/bin/"
 RUN make clean && rm -f */*.a lib/*/*.a && \
-    if [ "$AVX_FLAG" == "avx512" ]; then \
+    if [ "$AVX_FLAG" != "noopt" ]; then \
         make \
         CC="${GCC_PATH}/gcc -static" \
         CXX="${GCC_PATH}/g++ -static" \
         CPP="${GCC_PATH}/gcc -static -E" \
-        F77="${GCC_PATH}/gfortran" \
-#        F77="${GCC_PATH}/gfortran -static-libgfortran" \
-        CFLAGS="-DCERENKOPT -DVLIBM -std=c99 -O3 -mavx512f -ffp-contract=off -DVECTOR_SIZE=8 -DVECTOR_LENGTH=8" \
-        CXXFLAGS="-DCERENKOPT -DVLIBM -std=c99 -O3 -mavx512f -ffp-contract=off -DVECTOR_SIZE=8 -DVECTOR_LENGTH=8" \
-        FFLAGS="-ffixed-line-length-132 -fno-automatic -frecord-marker=4 -std=legacy -DCERENKOPT -DVLIBM -std=c99 -O3 -mavx512f -ffp-contract=off -DVECTOR_SIZE=8 -DVECTOR_LENGTH=8" \
-        install; \
-    elif [ "$AVX_FLAG" != "noopt" ]; then \
-        make \
-        CC="${GCC_PATH}/gcc -static" \
-        CXX="${GCC_PATH}/g++ -static" \
-        CPP="${GCC_PATH}/gcc -static -E" \
-#        F77="${GCC_PATH}/gfortran -L/lib64/ -static-libgfortran" \
         F77="${GCC_PATH}/gfortran -L/lib64/" \
         CFLAGS="-DCERENKOPT -DVLIBM -std=c99 -O3 -m${AVX_FLAG} -DVECTOR_SIZE=8 -DVECTOR_LENGTH=8" \
         CXXFLAGS="-DCERENKOPT -DVLIBM -std=c99 -O3 -m${AVX_FLAG} -DVECTOR_SIZE=8 -DVECTOR_LENGTH=8" \
-        FFLAGS="-ffixed-line-length-132 -fno-automatic -frecord-marker=4 -std=legacy -DCERENKOPT -DVLIBM -std=c99 -O3 -m${AVX_FLAG} -ffp-contract=off -DVECTOR_SIZE=8 -DVECTOR_LENGTH=8" \
+        # Note: different to CTA - no change in avx options for Fortran
+        # FFLAGS="-ffixed-line-length-132 -fno-automatic -frecord-marker=4 -std=legacy -DCERENKOPT -DVLIBM -std=c99 -O3 -m${AVX_FLAG} -DVECTOR_SIZE=8 -DVECTOR_LENGTH=8" \
+        FFLAGS="-ffixed-line-length-132 -fno-automatic -frecord-marker=4 -std=legacy -std=c99 -O3 -mavx2" \
         install; \
     else \
         make \
         CC="${GCC_PATH}/gcc -static" \
         CXX="${GCC_PATH}/g++ -static" \
         CPP="${GCC_PATH}/gcc -static -E" \
-#        F77="${GCC_PATH}/gfortran -static-libgfortran" \
         F77="${GCC_PATH}/gfortran" \
         CFLAGS="-std=c99 -O3 -mavx2" \
         CXXFLAGS="-std=c99 -O3 -mavx2" \
@@ -72,6 +61,9 @@ WORKDIR /workdir/
 FROM almalinux:9.3-minimal
 LABEL maintainer.name="VERITAS Collaboration"
 LABEL maintainer.email="gernot.maier@desy.de"
+LABEL org.opencontainers.image.source="https://github.com/VERITAS-Observatory/VTS-SimPipe"
+LABEL org.opencontainers.image.description="CORSIKA 7.7500 (optimized version) as used for the VERITAS simulation pipeline."
+LABEL org.opencontainers.image.licenses="BSD-3-Clause"
 WORKDIR /workdir
 ENV DEBIAN_FRONTEND=noninteractive
 ARG CORSIKA=corsika-77500

docker/Dockerfile-groptics

@@ -28,6 +28,9 @@ RUN mkdir GrOptics && tar -xzf v$GROPTICS_VERSION.tar.gz -C GrOptics --strip-com
 FROM rootproject/root:6.24.06-centos7
 LABEL maintainer.name="VERITAS Collaboration"
 LABEL maintainer.email="gernot.maier@desy.de"
+LABEL org.opencontainers.image.source="https://github.com/VERITAS-Observatory/VTS-SimPipe"
+LABEL org.opencontainers.image.description="GrOptics as used for the VERITAS simulation pipeline."
+LABEL org.opencontainers.image.licenses="BSD-3-Clause"
 WORKDIR /workdir
 COPY --from=build_image /workdir/corsikaIOreader/corsikaIOreader /workdir/corsikaIOreader/corsikaIOreader
 COPY --from=build_image /workdir/GrOptics /workdir/GrOptics

docker/Dockerfile-mergevbf

@@ -29,6 +29,9 @@ RUN mkdir EventDisplay_v4 && tar -xzf v490.5.tar.gz -C EventDisplay_v4 --strip-c
 FROM rootproject/root:6.24.06-centos7
 LABEL maintainer.name="VERITAS Collaboration"
 LABEL maintainer.email="gernot.maier@desy.de"
+LABEL org.opencontainers.image.source="https://github.com/VERITAS-Observatory/VTS-SimPipe"
+LABEL org.opencontainers.image.description="MergeVBF  as used for the VERITAS simulation pipeline."
+LABEL org.opencontainers.image.licenses="BSD-3-Clause"
 WORKDIR /workdir
 COPY --from=build_image /workdir/VBF /workdir/VBF
 COPY --from=build_image /workdir/EventDisplay_v4 /workdir/EventDisplay_v4