From e13d20e5b6c3df99188f6564c415976a3438d316 Mon Sep 17 00:00:00 2001 From: Sergey Kosukhin Date: Mon, 23 Dec 2024 20:52:32 +0100 Subject: [PATCH] Refactor CMake and CI scripting (#316) Move from Gnu Make to CMake/Ninja as the build system and update continuous integration. See the PR history for details. --------- Co-authored-by: Chiel van Heerwaarden Co-authored-by: Robert Pincus Co-authored-by: Alexander Soklev --- .cmake-format.py | 25 +++ .git-blame-ignore-revs | 2 + .github/workflows/check-api.yml | 11 +- .github/workflows/containerized-ci.yml | 88 +++----- .github/workflows/continuous-integration.yml | 193 ++++++++++-------- .github/workflows/self-hosted-ci.yml | 126 ------------ .github/workflows/style.yml | 72 +++++++ .gitignore | 6 + .gitlab/common.yml | 38 ++-- .gitlab/levante.yml | 27 +-- .gitlab/lumi.yml | 7 +- CMakeLists.txt | 79 +++++++ Makefile | 28 --- build/Makefile | 68 ------ cmake/CheckFortranNeedsCBool.cmake | 72 +++++++ cmake/CheckPython3Package.cmake | 43 ++++ cmake/FindNetCDF_Fortran.cmake | 19 ++ doc/jekyll_site/how-tos/build-and-test.md | 30 +-- environment-dev.yml | 2 + environment-noplots.yml | 5 +- examples/CMakeLists.txt | 31 +++ examples/all-sky/CMakeLists.txt | 75 +++++++ examples/all-sky/Makefile | 68 ------ examples/all-sky/all_tests.sh | 9 - examples/rfmip-clear-sky/CMakeLists.txt | 59 ++++++ examples/rfmip-clear-sky/Makefile | 75 ------- rrtmgp-frontend/CMakeLists.txt | 24 +++ rrtmgp-frontend/Make.depends | 47 ----- rrtmgp-kernels/CMakeLists.txt | 38 ++++ rrtmgp-kernels/Make.depends | 10 - rte-frontend/CMakeLists.txt | 18 ++ rte-frontend/Make.depends | 51 ----- rte-kernels/CMakeLists.txt | 51 +++++ rte-kernels/Make.depends | 27 --- {rte-frontend => rte-kernels}/mo_rte_kind.F90 | 0 setup.sh | 23 +++ tests/CMakeLists.txt | 114 +++++++++++ tests/Makefile | 97 --------- tests/all_tests.sh | 10 - tests/validation-plots.py | 22 +- 40 files changed, 970 insertions(+), 820 deletions(-) create mode 100644 .cmake-format.py create mode 100644 .git-blame-ignore-revs delete mode 100644 .github/workflows/self-hosted-ci.yml create mode 100644 .github/workflows/style.yml create mode 100644 CMakeLists.txt delete mode 100644 Makefile delete mode 100644 build/Makefile create mode 100644 cmake/CheckFortranNeedsCBool.cmake create mode 100644 cmake/CheckPython3Package.cmake create mode 100644 cmake/FindNetCDF_Fortran.cmake create mode 100644 examples/CMakeLists.txt create mode 100644 examples/all-sky/CMakeLists.txt delete mode 100644 examples/all-sky/Makefile delete mode 100644 examples/all-sky/all_tests.sh create mode 100644 examples/rfmip-clear-sky/CMakeLists.txt delete mode 100644 examples/rfmip-clear-sky/Makefile create mode 100644 rrtmgp-frontend/CMakeLists.txt delete mode 100644 rrtmgp-frontend/Make.depends create mode 100644 rrtmgp-kernels/CMakeLists.txt delete mode 100644 rrtmgp-kernels/Make.depends create mode 100644 rte-frontend/CMakeLists.txt delete mode 100644 rte-frontend/Make.depends create mode 100644 rte-kernels/CMakeLists.txt delete mode 100644 rte-kernels/Make.depends rename {rte-frontend => rte-kernels}/mo_rte_kind.F90 (100%) create mode 100755 setup.sh create mode 100644 tests/CMakeLists.txt delete mode 100644 tests/Makefile delete mode 100644 tests/all_tests.sh mode change 100644 => 100755 tests/validation-plots.py diff --git a/.cmake-format.py b/.cmake-format.py new file mode 100644 index 000000000..f6ee981d9 --- /dev/null +++ b/.cmake-format.py @@ -0,0 +1,25 @@ +with section("parse"): + additional_commands = {"check_python3_package": {"pargs": 1, "kwargs": {"CODE": 1}}} + +with section("format"): + dangle_parens = True + max_lines_hwrap = 0 + keyword_case = "upper" + autosort = True + +with section("lint"): + # The formatter sometimes fails to fit the code into the line limit (C0301) and can + # disagree with the linter regarding the indentation (C0307): + disabled_codes = ["C0301", "C0307"] + # Names of local variables must be in lowercase but sometimes we need to + # override standard CMake variables: + local_var_pattern = "CMAKE_[0-9A-Z_]+|[a-z][0-9a-z_]+" + # The standard names of the languages in CMake are C and Fortran. Names of + # private variables must be in lowercase but can have substings "C" and + # "Fortran": + private_var_pattern = ( + "([a-z_][0-9a-z_]*_)?(C|Fortran)(_[a-z_][0-9a-z_]*)?|[a-z_][0-9a-z_]+" + ) + # The standard name of the language in CMake is Fortran. Names of public + # variables must be in uppercase but can have substring "Fortran": + public_var_pattern = "([A-Z][0-9A-Z_]*_)?Fortran(_[A-Z][0-9A-Z_]*)?|[A-Z][0-9A-Z_]+" diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs new file mode 100644 index 000000000..593947014 --- /dev/null +++ b/.git-blame-ignore-revs @@ -0,0 +1,2 @@ +# Formatted entire CMake code base with cmake-format +45b43632309cff022326472a8be0fdd5efc8f5c8 diff --git a/.github/workflows/check-api.yml b/.github/workflows/check-api.yml index b3681e666..4b50a0355 100644 --- a/.github/workflows/check-api.yml +++ b/.github/workflows/check-api.yml @@ -14,11 +14,11 @@ jobs: API: runs-on: ubuntu-22.04 env: - # Core variables: FC: gfortran-12 - FCFLAGS: "-ffree-line-length-none -m64 -std=f2008 -march=native -fbounds-check -fmodule-private -fimplicit-none -finit-real=nan -g -DRTE_USE_CBOOL" - RRTMGP_ROOT: ${{ github.workspace }} + FFLAGS: "-m64 -std=f2008 -march=native -fbounds-check -fmodule-private -fimplicit-none -finit-real=nan" RTE_KERNELS: extern + CMAKE_BUILD_PARALLEL_LEVEL: 8 + VERBOSE: steps: # # Check out repository under $GITHUB_WORKSPACE @@ -31,4 +31,7 @@ jobs: - name: Build libraries run: | $FC --version - make -j4 libs \ No newline at end of file + cmake -S . -B build \ + -DCMAKE_BUILD_TYPE=RelWithDebInfo \ + -DKERNEL_MODE=$RTE_KERNELS + cmake --build build diff --git a/.github/workflows/containerized-ci.yml b/.github/workflows/containerized-ci.yml index de94e8417..aa91da7f3 100644 --- a/.github/workflows/containerized-ci.yml +++ b/.github/workflows/containerized-ci.yml @@ -21,21 +21,26 @@ jobs: include: # Set flags for Intel Fortran Compiler Classic - fortran-compiler: ifort - fcflags: -m64 -g -traceback -heap-arrays -assume realloc_lhs -extend-source 132 -check bounds,uninit,pointers,stack -stand f08 -diag-disable=10448 + fcflags: -m64 -traceback -heap-arrays -assume realloc_lhs -extend-source 132 -check bounds,uninit,pointers,stack -stand f08 -diag-disable=10448 + build-type: RelWithDebInfo # Set flags for Intel Fortran Compiler - fortran-compiler: ifx rte-kernels: default fcflags: -debug -traceback -O0 -heap-arrays -assume realloc_lhs -extend-source 132 -stand f08 + build-type: None - fortran-compiler: ifx rte-kernels: accel fcflags: -debug -traceback -O0 -heap-arrays -assume realloc_lhs -extend-source 132 -stand f08 -fiopenmp -fopenmp-targets=spir64 + build-type: None # Set flags for NVIDIA Fortran compiler - fortran-compiler: nvfortran rte-kernels: default fcflags: -Mallocatable=03 -Mstandard -Mbounds -Mchkptr -Kieee -Mchkstk + build-type: None - fortran-compiler: nvfortran rte-kernels: accel fcflags: -Mallocatable=03 -Mstandard -Mbounds -Mchkptr -Kieee -Mchkstk -acc + build-type: None # Set container images - fortran-compiler: ifort image: ghcr.io/earth-system-radiation/rte-rrtmgp-ci:oneapi @@ -46,33 +51,30 @@ jobs: container: image: ${{ matrix.image }} env: - # Core variables: FC: ${{ matrix.fortran-compiler }} - FCFLAGS: ${{ matrix.fcflags }} -DRTE_USE_${{ matrix.fpmodel}} - # Make variables: - NFHOME: /opt/netcdf-fortran - RRTMGP_ROOT: ${{ github.workspace }} - RRTMGP_DATA: ${{ github.workspace }}/rrtmgp-data - RTE_KERNELS: ${{ matrix.rte-kernels }} - RUN_CMD: + FFLAGS: ${{ matrix.fcflags }} + NetCDF_Fortran_ROOT: /opt/netcdf-fortran + CMAKE_BUILD_PARALLEL_LEVEL: 8 + VERBOSE: + # TODO: add missing test dependencies and run them in parallel: + # CTEST_PARALLEL_LEVEL: 8 + CTEST_OUTPUT_ON_FAILURE: 1 # https://github.com/earth-system-radiation/rte-rrtmgp/issues/194 OMP_TARGET_OFFLOAD: DISABLED - FAILURE_THRESHOLD: 7.e-4 steps: # - # Checks-out repository under $GITHUB_WORKSPACE + # Check out repository under $GITHUB_WORKSPACE # - - uses: actions/checkout@v4 + - name: Check out code + uses: actions/checkout@v4 # - # Check out data + # Install required tools # - - name: Check out data - uses: actions/checkout@v4 - with: - repository: earth-system-radiation/rrtmgp-data - path: rrtmgp-data - ref: v1.8.2 + - name: Install Required Tools + run: | + apt-get update + apt-get install -y git cmake ninja-build # # Build libraries, examples and tests (expect success) # @@ -80,52 +82,26 @@ jobs: id: build-success if: matrix.fortran-compiler != 'ifx' || matrix.rte-kernels != 'accel' run: | - $FC --version - make -j4 libs - # - # Build libraries, examples and tests (expect failure) - # - - name: Build libraries, examples and tests (expect failure) - if: steps.build-success.outcome == 'skipped' - shell: bash - run: | - $FC --version - make -j4 libs 2> >(tee make.err >&2) && { - echo "Unexpected success" - exit 1 - } || { - grep make.err -e 'Internal compiler error' && { - echo "Expected failure" - } || { - echo "Unexpected failure" - exit 1 - } - } + cmake -S . -B build -G "Ninja" \ + -DCMAKE_BUILD_TYPE=${{ matrix.build-type }} \ + -DRTE_ENABLE_SP=`test 'x${{ matrix.fpmodel }}' = xSP && echo ON || echo OFF` \ + -DKERNEL_MODE=${{ matrix.rte-kernels }} \ + -DBUILD_TESTING=ON + cmake --build build # # Run examples and tests # - name: Run examples and tests + working-directory: build if: steps.build-success.outcome != 'skipped' - run: make -j4 tests - # - # Relax failure thresholds for single precision - # - - name: Relax failure threshold for single precision - if: matrix.fpmodel == 'SP' && steps.build-success.outcome != 'skipped' - run: echo "FAILURE_THRESHOLD=3.5e-1" >> $GITHUB_ENV - # - # Compare the results - # - - name: Compare the results - if: steps.build-success.outcome != 'skipped' - run: make -j4 check + run: ctest # # Generate validation plots # - name: Generate validation plots if: matrix.fortran-compiler == 'ifort' && matrix.rte-kernels == 'default' && matrix.fpmodel == 'DP' - working-directory: tests - run: python validation-plots.py + run: | + cmake --build build --target validation-plots # # Upload validation plots # @@ -134,4 +110,4 @@ jobs: uses: actions/upload-artifact@v4 with: name: valdiation-plot - path: tests/validation-figures.pdf + path: build/tests/validation-figures.pdf diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml index 57bd196e8..84d1829d8 100644 --- a/.github/workflows/continuous-integration.yml +++ b/.github/workflows/continuous-integration.yml @@ -1,12 +1,13 @@ name: Continuous Integration + on: push: branches: - - main - - develop + - main + - develop pull_request: branches-ignore: - - documentation + - documentation workflow_dispatch: defaults: @@ -16,88 +17,116 @@ defaults: jobs: CI: - runs-on: ubuntu-22.04 strategy: fail-fast: false matrix: - fortran-compiler: [gfortran-10, gfortran-11, gfortran-12] + os: [ubuntu-24.04, macos-13, macos-latest, windows-2022] + gfortran-version: [12, 13, 14] + gfortran-from: [system, conda] fpmodel: [DP, SP] + exclude: + - os: ubuntu-24.04 + gfortran-from: conda + - os: macos-13 + gfortran-from: system + - os: macos-13 + gfortran-version: 14 + gfortran-from: conda + - os: macos-latest + gfortran-from: system + - os: macos-latest + gfortran-version: 14 + gfortran-from: conda + - os: windows-2022 + include: + - os: ubuntu-24.04 + gfortran-version: 13 + gfortran-from: conda + fpmodel: DP + - os: windows-2022 + gfortran-version: 13 + gfortran-from: conda + fpmodel: DP + - os: windows-2022 + gfortran-version: 13 + gfortran-from: conda + fpmodel: SP + - os: windows-2022 + gfortran-version: 14 + gfortran-from: conda + fpmodel: DP + - os: windows-2022 + gfortran-version: 14 + gfortran-from: conda + fpmodel: SP env: - # Core variables: - FC: ${{ matrix.fortran-compiler }} - FCFLAGS: "-ffree-line-length-none -m64 -std=f2008 -march=native -fbounds-check -fmodule-private -fimplicit-none -finit-real=nan -g -DRTE_USE_CBOOL -DRTE_USE_${{ matrix.fpmodel }}" - # Make variables: - FCINCLUDE: -I/usr/include - RRTMGP_ROOT: ${{ github.workspace }} - RRTMGP_DATA: ${{ github.workspace }}/rrtmgp-data - RUN_CMD: - FAILURE_THRESHOLD: 7.e-4 + FC: gfortran-${{ matrix.gfortran-version }} + FFLAGS: "-ffree-line-length-none -m64 -std=f2008 -march=native -fbounds-check -fmodule-private -fimplicit-none -finit-real=nan" + CMAKE_BUILD_PARALLEL_LEVEL: 8 + VERBOSE: + # TODO: add missing test dependencies and run them in parallel: + # CTEST_PARALLEL_LEVEL: 8 + CTEST_OUTPUT_ON_FAILURE: 1 + runs-on: ${{ matrix.os }} steps: - # - # Relax failure thresholds for single precision - # - - name: Relax failure threshold for single precision - if: matrix.fpmodel == 'SP' - run: echo "FAILURE_THRESHOLD=3.5e-1" >> $GITHUB_ENV - # - # Check out repository under $GITHUB_WORKSPACE - # - - name: Check out code - uses: actions/checkout@v4 - # - # Check out data - # - - name: Check out data - uses: actions/checkout@v4 - with: - repository: earth-system-radiation/rrtmgp-data - path: rrtmgp-data - ref: v1.8.2 - # - # Synchronize the package index - # - - name: Synchronize the package index - run: sudo apt-get update - # - # Install NetCDF-Fortran (compatible with all compilers) - # - - name: Install NetCDF-Fortran - run: sudo apt-get install libnetcdff-dev - # - # Cache Conda packages - # - - name: Cache Conda packages - uses: actions/cache@v4 - with: - path: ~/conda_pkgs_dir - key: conda-pkgs - # - # Set up Conda - # - - name: Set up Conda - uses: conda-incubator/setup-miniconda@v3 - with: - miniforge-version: latest - activate-environment: rte_rrtmgp_test - environment-file: environment-noplots.yml - python-version: 3.11 - auto-activate-base: false - # Use the cache properly: - use-only-tar-bz2: true - # - # Build libraries, examples and tests - # - - name: Build libraries - run: | - $FC --version - make -j4 libs - # - # Run examples and tests - # - - name: Build and run examples and tests - run: make -j4 tests - # - # Compare the results - # - - name: Compare the results - run: make -j4 check + # + # Check out repository under $GITHUB_WORKSPACE + # + - name: Check out code + uses: actions/checkout@v4 + # + # Cache Conda packages + # + - name: Cache Conda packages + uses: actions/cache@v4 + with: + path: ~/conda_pkgs_dir + key: conda-pkgs-${{ matrix.os }} + # + # Set up Conda + # + - name: Set up Conda + uses: conda-incubator/setup-miniconda@v3 + with: + miniforge-version: latest + activate-environment: rte_rrtmgp_test + environment-file: environment-noplots.yml + python-version: 3.11 + auto-activate-base: false + # Use the cache properly: + use-only-tar-bz2: false + # + # Install compiler + # + - name: Install compiler + if: matrix.gfortran-from == 'conda' + run: | + conda install -c conda-forge gfortran=${{ matrix.gfortran-version }} -y + echo "FC=gfortran" >> $GITHUB_ENV + # + # Install dependencies + # + - name: Install dependencies + run: conda install -c conda-forge netcdf-fortran ninja -y + # + # Adjust toolchain + # + - name: Adjust toolchain + if: matrix.os == 'windows-2022' + run: echo "FC=${FC}.exe" >> $GITHUB_ENV + # + # Build libraries, examples, and tests + # + - name: Build libraries and tests + run: | + cmake -S . -B build -G "Ninja" \ + -DCMAKE_BUILD_TYPE=RelWithDebInfo \ + -DRTE_ENABLE_SP=`test 'x${{ matrix.fpmodel }}' = xSP && echo ON || echo OFF` \ + -DBUILD_TESTING=ON + cmake --build build + # + # Run examples, tests and checks + # + - name: Run examples, tests and checks + working-directory: build + run: ctest diff --git a/.github/workflows/self-hosted-ci.yml b/.github/workflows/self-hosted-ci.yml deleted file mode 100644 index d9ad8838a..000000000 --- a/.github/workflows/self-hosted-ci.yml +++ /dev/null @@ -1,126 +0,0 @@ -name: Self-hosted CI -on: - workflow_dispatch: - -defaults: - run: - shell: bash - -# A workaround for the old runtime: -env: - ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true - -jobs: - CI: - if: github.repository == 'earth-system-radiation/rte-rrtmgp' - runs-on: - labels: cscs-ci - strategy: - fail-fast: false - matrix: - config-name: [nvidia-gpu-openacc, cce-cpu-icon-production, cce-gpu-openmp] - fpmodel: [DP, SP] - include: - - config-name: nvidia-gpu-openacc - rte-kernels: accel - compiler-modules: "PrgEnv-nvidia nvidia craype-accel-nvidia60 cdt-cuda/21.09 !cray-libsci_acc" - # Generic accelerator flag - fcflags: "-O3 -acc -Mallocatable=03 -gopt" - - config-name: cce-cpu-icon-production - rte-kernels: default - compiler-modules: "PrgEnv-cray" - # Production flags for Icon model - fcflags: "-hadd_paren -r am -Ktrap=divz,ovf,inv -hflex_mp=intolerant -hfp1 -hnoacc -O1,cache0" - - config-name: cce-gpu-openmp - rte-kernels: accel - compiler-modules: "PrgEnv-cray craype-accel-nvidia60 cdt-cuda/22.05 cudatoolkit/11.2.0_3.39-2.1__gf93aa1c" - # OpenMP flags from Nichols Romero (Argonne) - fcflags: "-hnoacc -homp -O0" - env: - # Core variables: - FC: ftn - FCFLAGS: ${{ matrix.fcflags }} -DRTE_USE_${{ matrix.fpmodel }} - # Make variables: - RRTMGP_ROOT: ${{ github.workspace }} - RRTMGP_DATA: ${{ github.workspace }}/rrtmgp-data - RTE_KERNELS: ${{ matrix.rte-kernels }} - RUN_CMD: "srun -C gpu -A d56 -p cscsci -t 15:00" - FAILURE_THRESHOLD: 7.e-4 - steps: - # - # Checks-out repository under $GITHUB_WORKSPACE - # - - uses: actions/checkout@v3 - # - # Check out data - # - - name: Check out data - uses: actions/checkout@v3 - with: - repository: earth-system-radiation/rrtmgp-data - path: rrtmgp-data - ref: v1.8.2 - # - # Finalize build environment - # - - name: Finalize build environment - run: | - # There are significant limitations on what can go into ${GITHUB_ENV}, - # therefore, we use ${BASH_ENV} but only when necessary: - BASH_ENV="${GITHUB_WORKSPACE}/.bash" - echo "source '${GITHUB_WORKSPACE}/.github/workflows/module_switcher'" >> "${BASH_ENV}" - echo "switch_for_module daint-gpu ${{ matrix.compiler-modules }} cray-netcdf cray-hdf5" >> "${BASH_ENV}" - # Use custom Python environment: - # The environment can be re-generated as follows: - # module load cray-python - # python3 -m venv /scratch/snx3000/rpincus/rte-rrtmgp-python - # /scratch/snx3000/rpincus/rte-rrtmgp-python/bin/pip3 install --upgrade pip - # /scratch/snx3000/rpincus/rte-rrtmgp-python/bin/pip3 install dask[array] netCDF4 numpy xarray - echo 'PATH="/scratch/snx3000/rpincus/rte-rrtmgp-python/bin:${PATH}"' >> "${BASH_ENV}" - # Make bash run the above on startup: - echo "BASH_ENV=${BASH_ENV}" >> "${GITHUB_ENV}" - # Compiler needs more temporary space than normally available: - tmpdir='${{ github.workspace }}/tmp' - mkdir "${tmpdir}" && echo "TMPDIR=${tmpdir}" >> "${GITHUB_ENV}" - # We use the "non-default products" for the tests - # (see https://support.hpe.com/hpesc/public/docDisplay?docId=a00113984en_us&page=Modify_Linking_Behavior_to_Use_Non-default_Libraries.html): - echo 'LD_LIBRARY_PATH="${CRAY_LD_LIBRARY_PATH}:${LD_LIBRARY_PATH}"' >> "${BASH_ENV}" - # SLURM jobs, user home directories and HDF5 file locking are - # incompatible on Daint: - echo 'HDF5_USE_FILE_LOCKING=FALSE' >> "${GITHUB_ENV}" - # - # Build libraries, examples and tests - # - - name: Build libraries - run: | - $FC --version - make -j8 libs - # - # Run examples and tests (expect success) - # - - name: Build and run examples and tests (expect success) - id: run-success - if: matrix.config-name != 'cce-gpu-openmp' - run: make -j8 tests - # - # Run examples and tests (expect failure) - # - - name: Build and run examples and tests (expect failure) - if: steps.run-success.outcome == 'skipped' - run: | - make -j8 tests && { - echo "Unexpected success" - exit 1 - } || echo "Expected failure" - # - # Relax failure thresholds for single precision - # - - name: Relax failure threshold for single precision - if: matrix.fpmodel == 'SP' && steps.run-success.outcome != 'skipped' - run: echo "FAILURE_THRESHOLD=3.5e-1" >> $GITHUB_ENV - # - # Compare the results - # - - name: Compare the results - if: steps.run-success.outcome != 'skipped' - run: make -j8 check diff --git a/.github/workflows/style.yml b/.github/workflows/style.yml new file mode 100644 index 000000000..a9f60a699 --- /dev/null +++ b/.github/workflows/style.yml @@ -0,0 +1,72 @@ +name: Check Style +on: + push: + branches: + - main + - develop + pull_request: + branches-ignore: + - documentation + workflow_dispatch: + +env: + FIND_CMAKE_FILES_CMD: "find '${{ github.workspace }}' -name 'CMakeLists.txt' -o -name '*.cmake'" + +jobs: + Format: + runs-on: ubuntu-22.04 + env: + DEFAULT: '\033[0m' + RED: '\033[0;31m' + GREEN: '\033[0;32m' + FORMAT_PATCH: '${{ github.workspace }}/format.patch' + steps: + - name: Check out code + uses: actions/checkout@v4 + - name: Install Python + uses: actions/setup-python@v5 + with: + python-version: '>=3.8' + - name: Install required tools + run: python -m pip install cmake-format + - name: Format CMake + run: cmake-format -i $(eval "${FIND_CMAKE_FILES_CMD}") + - name: Check if patching is required + id: patch-required + run: | + git -C '${{ github.workspace }}' diff --patch-with-raw > "${FORMAT_PATCH}" + test -s "${FORMAT_PATCH}" && { + printf "${RED}The source code does not meet the format requirements. \ + Please, apply the patch (see artifacts).${DEFAULT}\n" + + printf "${RED}Note that the result of the formatting might depend \ + on the versions of the formatting tools. In this project, whatever \ + formatting this CI job produces if the correct one. If it expects \ + you to reformat parts of the source code that you did not modify, do \ + so in a separate commit, which must not be squashed, and list the \ + commit in the '.git-blame-ignore-revs' file.${DEFAULT}\n" + + exit 1 + } || { + printf "${GREEN}The source code meets the format requirements.${DEFAULT}\n" + rm -rf "${FORMAT_PATCH}" + } + - name: Upload the patch file + if: always() && steps.patch-required.outcome == 'failure' + uses: actions/upload-artifact@v4 + with: + name: format-patch + path: ${{ env.FORMAT_PATCH }} + Lint: + runs-on: ubuntu-22.04 + steps: + - name: Check out code + uses: actions/checkout@v4 + - name: Install Python + uses: actions/setup-python@v5 + with: + python-version: '>=3.8' + - name: Install required tools + run: python -m pip install cmake-format + - name: Lint CMake + run: cmake-lint $(eval "${FIND_CMAKE_FILES_CMD}") diff --git a/.gitignore b/.gitignore index 2ae1d6a51..e32360bf6 100644 --- a/.gitignore +++ b/.gitignore @@ -33,8 +33,14 @@ examples/*/*/*.nc *.html *.gif +# Python virtual environments +.venv*/ + # gh-pages directory public +# build +build/ + # Ruby stuff **/Gemfile.lock diff --git a/.gitlab/common.yml b/.gitlab/common.yml index b8c180085..707122f1c 100644 --- a/.gitlab/common.yml +++ b/.gitlab/common.yml @@ -1,37 +1,33 @@ .dp: variables: - FPMODEL: DP + RTE_ENABLE_SP: OFF FAILURE_THRESHOLD: "7.e-4" .sp: variables: - FPMODEL: SP + RTE_ENABLE_SP: ON FAILURE_THRESHOLD: "3.5e-1" .common: variables: - # Make variables: - MAKEFLAGS: -j8 - RRTMGP_ROOT: ${CI_PROJECT_DIR} - RRTMGP_DATA: ${CI_PROJECT_DIR}/rrtmgp-data - # Convenience variables: - RRTMGP_DATA_REPO: https://github.com/earth-system-radiation/rrtmgp-data.git - RRTMGP_DATA_TAG: v1.8.2 + CMAKE_BUILD_PARALLEL_LEVEL: 8 + VERBOSE: + # TODO: add missing test dependencies and run them in parallel: + # CTEST_PARALLEL_LEVEL: 8 + CTEST_OUTPUT_ON_FAILURE: 1 script: # # Build libraries, examples and tests # - - ${FC} ${VERSION_FCFLAGS} - - make libs + - | + cmake -S . -B build \ + -DCMAKE_BUILD_TYPE=None \ + -DRTE_ENABLE_SP=$RTE_ENABLE_SP \ + -DKERNEL_MODE=$KERNEL_MODE \ + -DBUILD_TESTING=ON \ + -DFAILURE_THRESHOLD=$FAILURE_THRESHOLD + - cmake --build build # - # Check out data + # Run examples, tests and checks # - - git clone --depth 1 ${RRTMGP_DATA_TAG:+--branch "${RRTMGP_DATA_TAG}"} "${RRTMGP_DATA_REPO}" "${RRTMGP_DATA}" - # - # Run examples and tests - # - - make tests - # - # Compare the results - # - - make check + - ctest --test-dir build diff --git a/.gitlab/levante.yml b/.gitlab/levante.yml index 2a0a1a435..677cf4039 100644 --- a/.gitlab/levante.yml +++ b/.gitlab/levante.yml @@ -29,22 +29,13 @@ variables: .nvhpc: variables: - # Core variables: - # FC: /sw/spack-levante/nvhpc-22.5-v4oky3/Linux_x86_64/22.5/compilers/bin/nvfortran FC: /sw/spack-levante/nvhpc-24.9-p7iohv/Linux_x86_64/24.9/compilers/bin/nvfortran - # Convenience variables: - VERSION_FCFLAGS: --version - NFHOME: /sw/spack-levante/netcdf-fortran-4.5.4-syv4qr - NCHOME: /sw/spack-levante/netcdf-c-4.9.0-gc7kgj + NetCDF_Fortran_ROOT: /sw/spack-levante/netcdf-fortran-4.6.1-4wu5wt .nag: variables: - # Core variables: FC: /sw/spack-levante/nag-7.1-lqjbej/bin/nagfor - # Convenience variables: - VERSION_FCFLAGS: -V - NFHOME: /sw/spack-levante/netcdf-fortran-4.5.3-5di6qe - NCHOME: /sw/spack-levante/netcdf-c-4.8.1-vbnli5 + NetCDF_Fortran_ROOT: /sw/spack-levante/netcdf-fortran-4.5.3-5di6qe .common-levante: extends: .common @@ -52,15 +43,11 @@ variables: PYHOME: /sw/spack-levante/mambaforge-22.9.0-2-Linux-x86_64-kptncg # Suppress an irrelevant but annoying error message: PROJ_LIB: ${PYHOME}/share/proj - # Make variables: - FCINCLUDE: -I${NFHOME}/include - LDFLAGS: -L${NFHOME}/lib -L${NCHOME}/lib before_script: - module purge - module load git # Extend the existing environment variables: - export PATH="${PYHOME}/bin:${PATH}" - - export LD_LIBRARY_PATH="${NFHOME}/lib:${NCHOME}/lib:${LD_LIBRARY_PATH-}" # Some tests require a large stack: - ulimit -s unlimited @@ -71,8 +58,8 @@ variables: - .common-levante variables: # Compiler flags used for ICON model: - FCFLAGS: -g -O2 -Mrecursive -Mallocatable=03 -Mstack_arrays -Minfo=accel,inline -acc=gpu,verystrict -gpu=cc80,cuda11.8 -DRTE_USE_${FPMODEL} - RTE_KERNELS: accel + FFLAGS: -g -O2 -Mrecursive -Mallocatable=03 -Mstack_arrays -Minfo=accel,inline -acc=gpu,verystrict -gpu=cc80,cuda12.6 + KERNEL_MODE: accel .nag-cpu: extends: @@ -81,17 +68,17 @@ variables: - .common-levante variables: # Compiler flags used for ICON model: - FCFLAGS: -Wc=/sw/spack-levante/gcc-11.2.0-bcn7mb/bin/gcc -f2008 -colour -w=uep -g -gline -O0 -float-store -nan -Wc,-g -Wc,-pipe -Wc,--param,max-vartrack-size=200000000 -Wc,-mno-fma -C=all -DRTE_USE_CBOOL -DRTE_USE_${FPMODEL} + FFLAGS: -Wc=/sw/spack-levante/gcc-11.2.0-bcn7mb/bin/gcc -f2008 -colour -w=uep -g -gline -O0 -float-store -nan -Wc,-g -Wc,-pipe -Wc,--param,max-vartrack-size=200000000 -Wc,-mno-fma -C=all .nag-cpu-default: extends: .nag-cpu variables: - RTE_KERNELS: default + KERNEL_MODE: default .nag-cpu-accel: extends: .nag-cpu variables: - RTE_KERNELS: accel + KERNEL_MODE: accel nvhpc-gpu-openacc-DP: extends: diff --git a/.gitlab/lumi.yml b/.gitlab/lumi.yml index 9e2c1ad4c..4f351f2e2 100644 --- a/.gitlab/lumi.yml +++ b/.gitlab/lumi.yml @@ -38,10 +38,7 @@ variables: .cce: variables: - # Core variables: FC: ftn - # Convenience variables: - VERSION_FCFLAGS: -V COMPILER_MODULES: PrgEnv-cray cce/16.0.1 craype-x86-milan # @@ -91,8 +88,8 @@ setup-python: - .common-lumi variables: # Compiler flags used for ICON model: - FCFLAGS: -hacc -hadd_paren -Ktrap=divz,ovf,inv -hflex_mp=intolerant -hfp1 -g -DRTE_USE_${FPMODEL} - RTE_KERNELS: accel + FFLAGS: -hacc -hadd_paren -Ktrap=divz,ovf,inv -hflex_mp=intolerant -hfp1 -g + KERNEL_MODE: accel # Convenience variables: EXTRA_COMPILER_MODULES: craype-accel-amd-gfx90a rocm diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 000000000..29513fd1a --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,79 @@ +cmake_minimum_required(VERSION 3.18) + +project( + rte-rrtmgp + VERSION 1.8 + LANGUAGES Fortran +) + +option(BUILD_TESTING "Build tests" OFF) + +option(RTE_ENABLE_SP "Enable single-precision floating-point model" OFF) + +set(PREFERRED_KERNEL_MODES "default" "accel" "extern") +set(KERNEL_MODE + "default" + CACHE STRING "Select the kernel mode: ${PREFERRED_KERNEL_MODES}" +) +set_property(CACHE KERNEL_MODE PROPERTY STRINGS ${PREFERRED_KERNEL_MODES}) + +list(APPEND CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake) + +add_compile_options( + $<$:-ffree-line-length-none> +) + +set(CMAKE_Fortran_MODULE_DIRECTORY ${PROJECT_BINARY_DIR}/modules) + +add_subdirectory(rte-kernels) +add_subdirectory(rte-frontend) +add_subdirectory(rrtmgp-kernels) +add_subdirectory(rrtmgp-frontend) + +include(CTest) +if(BUILD_TESTING) + find_package(Python3 REQUIRED COMPONENTS Interpreter) + include(CheckPython3Package) + check_python3_package(numpy) + check_python3_package( + "netCDF4 or h5netcdf+scipy" + CODE "try: + import netCDF4 +except: + import h5netcdf + import scipy +" + ) + check_python3_package( + "xarray>=0.12.2" + CODE "import xarray +exit(tuple(map(int, xarray.__version__.split('.'))) < (0, 12, 2))" + ) + check_python3_package(dask.array) + + find_package(NetCDF_Fortran REQUIRED) + + if(NOT RRTMGP_DATA) + set(RRTMGP_DATA + "${PROJECT_BINARY_DIR}/rrtmgp-data" + CACHE PATH "Path to the RRTMGP data" FORCE + ) + set(RRTMGP_DATA_VERSION "v1.8.2") + include(FetchContent) + message(CHECK_START "Fetching RRTMGP data ${RRTMGP_DATA_VERSION}") + FetchContent_Declare( + rrtmgp-data + GIT_REPOSITORY https://github.com/earth-system-radiation/rrtmgp-data.git + GIT_TAG ${RRTMGP_DATA_VERSION} + SOURCE_DIR ${RRTMGP_DATA} + ) + FetchContent_MakeAvailable(rrtmgp-data) + message(CHECK_PASS "done") + endif() + + add_subdirectory(examples) + add_subdirectory(tests) +else() + # Allow for 'make test' even if the tests are disabled: + enable_testing() +endif() diff --git a/Makefile b/Makefile deleted file mode 100644 index 96c1e2edd..000000000 --- a/Makefile +++ /dev/null @@ -1,28 +0,0 @@ -# -# Top-level Makefile -# -.PHONY: libs tests check -all: libs tests check - -libs: - $(MAKE) -C build $@ - -tests: - $(MAKE) -C tests $@ - $(MAKE) -C examples/rfmip-clear-sky $@ - $(MAKE) -C examples/all-sky $@ - -check: - $(MAKE) -C tests $@ - $(MAKE) -C examples/rfmip-clear-sky $@ - $(MAKE) -C examples/all-sky $@ - -docs: - @cd doc; ./build_documentation.sh - -clean: - $(MAKE) -C build $@ - $(MAKE) -C tests $@ - $(MAKE) -C examples/rfmip-clear-sky $@ - $(MAKE) -C examples/all-sky $@ - rm -rf public diff --git a/build/Makefile b/build/Makefile deleted file mode 100644 index efd0341d2..000000000 --- a/build/Makefile +++ /dev/null @@ -1,68 +0,0 @@ -#!/usr/bin/env make - -RTE_DIR = ../rte-frontend -GAS_OPTICS_DIR = ../gas-optics -RRTMGP_DIR = ../rrtmgp-frontend -RTE_KERNEL_DIR = ../rte-kernels -RRTMGP_KERNEL_DIR = ../rrtmgp-kernels -# -# Compiler variables FC, FCFLAGS must be set in the environment -# -# Make all the libraries though we'll only use the interface + kernels -all: librte.a librrtmgp.a \ - librtekernels.a librtef.a librrtmgpkernels.a librrtmgpf.a -separate-libs: librtekernels.a librtef.a librrtmgpkernels.a librrtmgpf.a -libs: all - -COMPILE = $(FC) $(FCFLAGS) $(FCINCLUDE) -c -%.o: %.F90 - $(COMPILE) $< - -include $(RTE_DIR)/Make.depends -include $(RRTMGP_DIR)/Make.depends -include $(RTE_KERNEL_DIR)/Make.depends -include $(RRTMGP_KERNEL_DIR)/Make.depends - -# -# If using OpenACC/OpenMP files in *-kernels/accel take precendence -# -ifeq ($(RTE_KERNELS), accel) - VPATH = $(RTE_KERNEL_DIR)/accel:$(RRTMGP_KERNEL_DIR)/accel -endif -# -# If using external libraries just compile the interfaces -# -ifeq ($(RTE_KERNELS), extern) - VPATH = $(RTE_KERNEL_DIR)/api:$(RRTMGP_KERNEL_DIR)/api -endif -VPATH += $(RTE_DIR):$(RTE_KERNEL_DIR):$(RRTMGP_DIR):$(RRTMGP_KERNEL_DIR):$(GAS_OPTICS_DIR) - -# -# Complete library - kernels plus Fortran front end -# -librte.a: $(RTE_FORTRAN_KERNELS) $(RTE_FORTRAN_INTERFACE) - ar -rvs librte.a $(RTE_FORTRAN_KERNELS) $(RTE_FORTRAN_INTERFACE) -# -# Library with just the kernels... -# -librtekernels.a: $(RTE_FORTRAN_KERNELS) - ar -rvs librtekernels.a $(RTE_FORTRAN_KERNELS) -# -# ... and just the Fortran front-end -# -librtef.a: $(RTE_FORTRAN_INTERFACE) - ar -rvs librtef.a $(RTE_FORTRAN_INTERFACE) -# -# As with RTE, libraries with Fortran front-end and kernels, separate and combined -# -librrtmgp.a: $(RRTMGP_FORTRAN_KERNELS) $(RRTMGP_FORTRAN_INTERFACE) - ar -rvs librrtmgp.a $(RRTMGP_FORTRAN_KERNELS) $(RRTMGP_FORTRAN_INTERFACE) - -librrtmgpkernels.a: $(RRTMGP_FORTRAN_KERNELS) - ar -rvs librrtmgpkernels.a $(RRTMGP_FORTRAN_KERNELS) - -librrtmgpf.a: $(RRTMGP_FORTRAN_INTERFACE) - ar -rvs librrtmgpf.a $(RRTMGP_FORTRAN_INTERFACE) - -clean: - rm -f *.optrpt *.mod *.o lib*.a diff --git a/cmake/CheckFortranNeedsCBool.cmake b/cmake/CheckFortranNeedsCBool.cmake new file mode 100644 index 000000000..1780d21e6 --- /dev/null +++ b/cmake/CheckFortranNeedsCBool.cmake @@ -0,0 +1,72 @@ +# ~~~ +# check_fortran_needs_c_bool() +# ~~~ +# Checks whether the Fortran compiler requires the c_bool kind of the logical +# type. Sets the cache to the result of the check. +# +function(check_fortran_needs_cbool var) + if(DEFINED ${var}) + return() + endif() + + if(NOT CMAKE_REQUIRED_QUIET) + message(CHECK_START "Checking if the Fortran compiler requires C_BOOL") + endif() + + set(CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY) + + set(check_source_code + " + subroutine conftest_foo(a) bind(C) + use iso_c_binding + implicit none +#ifdef RTE_USE_CBOOL + integer, parameter :: wl = c_bool +#else + integer, parameter :: wl = kind(.true.) +#endif + logical(wl) :: a + end subroutine +" + ) + + set(check_source_file + "${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeTmp/src.F90" + ) + + unset(check_result) + file(WRITE "${check_source_file}" "${check_source_code}") + try_compile(try_result "${CMAKE_BINARY_DIR}" "${check_source_file}") + if(try_result) + set(check_result FALSE) + else() + file(WRITE "${check_source_file}" "${check_source_code}") + try_compile( + try_result "${CMAKE_BINARY_DIR}" + "${check_source_file}" + COMPILE_DEFINITIONS "-DRTE_USE_CBOOL" + ) + if(try_result) + set(check_result TRUE) + endif() + endif() + + if(NOT CMAKE_REQUIRED_QUIET) + if(NOT DEFINED check_result) + message(CHECK_FAIL "unknown (assuming no)") + elseif(check_result) + message(CHECK_PASS "yes") + else() + message(CHECK_PASS "no") + endif() + endif() + + if(NOT DEFINED check_result) + set(check_result FALSE) + endif() + + set(${var} + "${check_result}" + CACHE BOOL "Whether the Fortran compiler requires CBOOL type" + ) +endfunction() diff --git a/cmake/CheckPython3Package.cmake b/cmake/CheckPython3Package.cmake new file mode 100644 index 000000000..c70241edd --- /dev/null +++ b/cmake/CheckPython3Package.cmake @@ -0,0 +1,43 @@ +# ~~~ +# check_python3_package( +# [CODE ]) +# ~~~ +# Checks whether the Python3 is available by running the with +# ${Python3_EXECUTABLE} (defaults to 'import '). Fails the +# configuration if the result is negative. +# +function(check_python3_package package) + cmake_parse_arguments(PARSE_ARGV 1 arg "" "CODE" "") + + if(DEFINED ${var}) + return() + endif() + + if(NOT CMAKE_REQUIRED_QUIET) + message( + CHECK_START "Checking if the Python3 package ${package} is available" + ) + endif() + + if(NOT arg_CODE) + set(arg_CODE "import ${package}") + endif() + + execute_process( + COMMAND ${Python3_EXECUTABLE} -c "${arg_CODE}" + RESULT_VARIABLE exit_status + OUTPUT_QUIET ERROR_QUIET + ) + + if(NOT CMAKE_REQUIRED_QUIET) + if(NOT exit_status) + message(CHECK_PASS "yes") + else() + message(CHECK_FAIL "no") + endif() + endif() + + if(exit_status) + message(FATAL_ERROR "Required Python3 package ${package} is not available") + endif() +endfunction() diff --git a/cmake/FindNetCDF_Fortran.cmake b/cmake/FindNetCDF_Fortran.cmake new file mode 100644 index 000000000..7dd8da726 --- /dev/null +++ b/cmake/FindNetCDF_Fortran.cmake @@ -0,0 +1,19 @@ +find_library( + NetCDF_Fortran_LIBRARY + NAMES netcdff + DOC "NetCDF-Fortran library" +) +mark_as_advanced(NetCDF_Fortran_LIBRARY) + +find_path( + NetCDF_Fortran_INCLUDE_DIR + NAMES netcdf.mod NETCDF.mod + DOC "NetCDF_Fortran include directory" +) +mark_as_advanced(NetCDF_Fortran_INCLUDE_DIR) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args( + NetCDF_Fortran + REQUIRED_VARS NetCDF_Fortran_LIBRARY NetCDF_Fortran_INCLUDE_DIR +) diff --git a/doc/jekyll_site/how-tos/build-and-test.md b/doc/jekyll_site/how-tos/build-and-test.md index 07b4dc186..bec95a35b 100644 --- a/doc/jekyll_site/how-tos/build-and-test.md +++ b/doc/jekyll_site/how-tos/build-and-test.md @@ -5,22 +5,26 @@ title: "How to build and run tests" How to build the libraries, tests, and examples, run the tests, and verify the results ## In a nutshell -In the root directory: -- `make libs` makes the RTE and RRTMGP libraries, the unit tests, and the examples -- `make tests` runs the tests -- `make check` uses Python to verify results against reference calculations -- `make` invoked without a target in the top level attempts all three steps. +RTE+RRTMGP uses `CMake`. In the root directory: +`cmake -S . -B build` will guide you through configuration options. + +Environment variables can also be set and passed to `CMake` as in this example, which +builds and runs the tests: + +` +cmake -S . -B build \ + -DCMAKE_Fortran_COMPILER=$FC \ + -DCMAKE_Fortran_FLAGS=$FCFLAGS \ + -DRRTMGP_DATA_VERSION=$RRTMGP_DATA_VERSION \ + -DPRECISION=$FP_MODEL \ + -DUSE_C_BOOL=$RTE_CBOOL \ + -DKERNEL_MODE=$RTE_KERNELS \ + -DENABLE_TESTS=ON \ + -DFAILURE_THRESHOLD=$FAILURE_THRESHOLD +` Evaluating the results of the tests requires `Python` and the packages described in `environment*.yml`. -## Building and testing using the handbuilt Makefiles - -Before using the Makefiles supplied with the `RTE+RRTMGP` repository, the environment variables `FC` and -`FCFLAGS`, identifying the Fortran compiler and flags passed to it, need to be set. - -To build any of the examples in `examples/` or `tests` the locations of the C and Fortran netCDF libraries and the -location of the netCDF Fortran module file (`netcdf.mod`) must be in the search path. -Non-standard paths can also be added via macros `FCINCLUDE` and/or `LDFLAGS`. ## Building and testing using (Gnu) autotools diff --git a/environment-dev.yml b/environment-dev.yml index b9fa161d9..5b46be5b8 100644 --- a/environment-dev.yml +++ b/environment-dev.yml @@ -19,6 +19,8 @@ dependencies: - colorcet - gfortran - netcdf-fortran + - cmake + - ninja variables: FC: gfortran # Debugging flags below diff --git a/environment-noplots.yml b/environment-noplots.yml index dfa6536fc..50a628def 100644 --- a/environment-noplots.yml +++ b/environment-noplots.yml @@ -2,10 +2,13 @@ # Python modules below are needed to run tests and check results # name: rte_rrtmgp_test_noplots - +channels: + - conda-forge + - nodefaults dependencies: - python=3.11 - netcdf4 - xarray - dask - numpy + - cmake diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt new file mode 100644 index 000000000..4f96a3e62 --- /dev/null +++ b/examples/CMakeLists.txt @@ -0,0 +1,31 @@ +if(RTE_ENABLE_SP) + set(default_FAILURE_THRESHOLD "3.5e-1") +else() + set(default_FAILURE_THRESHOLD "7.e-4") +endif() + +set(FAILURE_THRESHOLD + "${default_FAILURE_THRESHOLD}" + CACHE STRING "Default failure threshold for tests" +) + +set(CMAKE_Fortran_MODULE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/modules) + +add_library( + examples_utils STATIC # cmake-format: sort + mo_garand_atmos_io.F90 mo_load_coefficients.F90 mo_simple_netcdf.F90 +) + +target_include_directories( + examples_utils + PUBLIC + $:${CMAKE_Fortran_MODULE_DIRECTORY}>> + ${NetCDF_Fortran_INCLUDE_DIR} +) + +target_link_libraries( + examples_utils PUBLIC rrtmgp rte ${NetCDF_Fortran_LIBRARY} +) + +add_subdirectory(all-sky) +add_subdirectory(rfmip-clear-sky) diff --git a/examples/all-sky/CMakeLists.txt b/examples/all-sky/CMakeLists.txt new file mode 100644 index 000000000..62a3e112c --- /dev/null +++ b/examples/all-sky/CMakeLists.txt @@ -0,0 +1,75 @@ +set(TEST_INPUTS + ${RRTMGP_DATA}/examples/all-sky/reference/rrtmgp-allsky-lw.nc + ${RRTMGP_DATA}/examples/all-sky/reference/rrtmgp-allsky-sw.nc + ${RRTMGP_DATA}/examples/all-sky/reference/rrtmgp-allsky-lw-no-aerosols.nc + ${RRTMGP_DATA}/examples/all-sky/reference/rrtmgp-allsky-sw-no-aerosols.nc +) + +add_custom_target( + all_sky_test_inputs ALL + COMMAND + ${CMAKE_COMMAND} -E copy_if_different ${TEST_INPUTS} + ${CMAKE_CURRENT_BINARY_DIR}/ + COMMENT "Copying required test input files" +) + +add_library( + all_sky_utils STATIC # cmake-format: sort + mo_load_aerosol_coefficients.F90 mo_load_cloud_coefficients.F90 +) + +target_link_libraries(all_sky_utils PUBLIC examples_utils) + +add_executable(rrtmgp_allsky rrtmgp_allsky.F90) +target_link_libraries(rrtmgp_allsky PRIVATE all_sky_utils) +add_dependencies(rrtmgp_allsky all_sky_test_inputs) + +add_test( + NAME allsky_test_lw + COMMAND + rrtmgp_allsky 24 72 1 rrtmgp-allsky-lw.nc + ${RRTMGP_DATA}/rrtmgp-gas-lw-g256.nc ${RRTMGP_DATA}/rrtmgp-clouds-lw.nc + ${RRTMGP_DATA}/rrtmgp-aerosols-merra-lw.nc +) + +add_test( + NAME allsky_test_sw + COMMAND + rrtmgp_allsky 24 72 1 rrtmgp-allsky-sw.nc + ${RRTMGP_DATA}/rrtmgp-gas-sw-g224.nc ${RRTMGP_DATA}/rrtmgp-clouds-sw.nc + ${RRTMGP_DATA}/rrtmgp-aerosols-merra-sw.nc +) + +add_test( + NAME allsky_test_lw_no_aerosols + COMMAND + rrtmgp_allsky 24 72 1 rrtmgp-allsky-lw-no-aerosols.nc + ${RRTMGP_DATA}/rrtmgp-gas-lw-g256.nc ${RRTMGP_DATA}/rrtmgp-clouds-lw.nc +) + +add_test( + NAME allsky_test_sw_no_aerosols + COMMAND + rrtmgp_allsky 24 72 1 rrtmgp-allsky-sw-no-aerosols.nc + ${RRTMGP_DATA}/rrtmgp-gas-sw-g224.nc ${RRTMGP_DATA}/rrtmgp-clouds-sw.nc +) + +add_test( + NAME check_allsky_lw_sw + COMMAND + ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/examples/compare-to-reference.py + --ref_dir ${RRTMGP_DATA}/examples/all-sky/reference --tst_dir + ${CMAKE_CURRENT_BINARY_DIR} --variables lw_flux_up lw_flux_dn sw_flux_up + sw_flux_dn sw_flux_dir --file_names rrtmgp-allsky-lw.nc rrtmgp-allsky-sw.nc + --failure_threshold ${FAILURE_THRESHOLD} +) + +add_test( + NAME check_allsky_no_aerosols_lw_sw + COMMAND + ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/examples/compare-to-reference.py + --ref_dir ${RRTMGP_DATA}/examples/all-sky/reference --tst_dir + ${CMAKE_CURRENT_BINARY_DIR} --variables lw_flux_up lw_flux_dn sw_flux_up + sw_flux_dn sw_flux_dir --file_names rrtmgp-allsky-lw-no-aerosols.nc + rrtmgp-allsky-sw-no-aerosols.nc --failure_threshold ${FAILURE_THRESHOLD} +) diff --git a/examples/all-sky/Makefile b/examples/all-sky/Makefile deleted file mode 100644 index 225946e31..000000000 --- a/examples/all-sky/Makefile +++ /dev/null @@ -1,68 +0,0 @@ -#!/usr/bin/env make -# -# Location of RTE+RRTMGP libraries, module files. -# -RRTMGP_ROOT ?= ../.. -RRTMGP_BUILD = $(RRTMGP_ROOT)/build -# -# RRTMGP library, module files -# -LDFLAGS += -L$(RRTMGP_BUILD) -LIBS += -lrrtmgp -lrte -FCINCLUDE += -I$(RRTMGP_BUILD) - -# netcdf Fortran module files has to be in the search path or added via environment variable FCINCLUDE e.g. -#FCINCLUDE += -I$(NFHOME)/include - -# netcdf C and Fortran libraries have to be in the search path or added via environment variable LDFLAGS e.g. -#LDFLAGS += -L$(NFHOME)/lib -L$(NCHOME)/lib -LIBS += -lnetcdff - -VPATH = ../:$(RRTMGP_ROOT)/rrtmgp-frontend # Needed for cloud_optics and aerosol_optics - -# Compilation rules -%.o: %.F90 - $(FC) $(FCFLAGS) $(FCINCLUDE) -c $< -%: %.o - $(FC) $(FCFLAGS) -o $@ $^ $(LDFLAGS) $(LIBS) - -# -# Extra sources -- extensions to RRTMGP classes, shared infrastructure, local sources -# -ADDITIONS = mo_load_coefficients.o mo_simple_netcdf.o -ADDITIONS += mo_cloud_optics_rrtmgp.o mo_load_cloud_coefficients.o -ADDITIONS += mo_aerosol_optics_rrtmgp_merra.o mo_load_aerosol_coefficients.o - -# -# Targets -# -all: rrtmgp_allsky - -rrtmgp_allsky: $(ADDITIONS) rrtmgp_allsky.o - -rrtmgp_allsky.o: $(ADDITIONS) rrtmgp_allsky.F90 - -mo_cloud_optics_rrtmgp.o: mo_cloud_optics_rrtmgp.F90 -mo_aerosol_optics_rrtmgp_merra.o: mo_aerosol_optics_rrtmgp_merra.F90 -mo_load_coefficients.o: mo_simple_netcdf.o mo_load_coefficients.F90 -mo_load_cloud_coefficients.o: mo_simple_netcdf.o mo_cloud_optics_rrtmgp.o mo_load_cloud_coefficients.F90 -mo_load_aerosol_coefficients.o: mo_simple_netcdf.o mo_aerosol_optics_rrtmgp_merra.o mo_load_aerosol_coefficients.F90 - -# The default location of the input data: -RRTMGP_DATA ?= $(RRTMGP_ROOT)/rrtmgp-data -# Make it available to the scripts: -export RRTMGP_DATA - -tests: rrtmgp_allsky - $(RUN_CMD) bash all_tests.sh - -check: - $${PYTHON-python} ${RRTMGP_ROOT}/examples/compare-to-reference.py --ref_dir ${RRTMGP_DATA}/examples/all-sky/reference --tst_dir ${RRTMGP_ROOT}/examples/all-sky \ - --var lw_flux_up lw_flux_dn sw_flux_up sw_flux_dn sw_flux_dir \ - --file rrtmgp-allsky-lw.nc rrtmgp-allsky-sw.nc - $${PYTHON-python} ${RRTMGP_ROOT}/examples/compare-to-reference.py --ref_dir ${RRTMGP_DATA}/examples/all-sky/reference --tst_dir ${RRTMGP_ROOT}/examples/all-sky \ - --var lw_flux_up lw_flux_dn sw_flux_up sw_flux_dn sw_flux_dir \ - --file rrtmgp-allsky-lw-no-aerosols.nc rrtmgp-allsky-sw-no-aerosols.nc - -clean: - -rm rrtmgp_allsky *.o *.optrpt ../*.optrpt *.mod *.nc diff --git a/examples/all-sky/all_tests.sh b/examples/all-sky/all_tests.sh deleted file mode 100644 index 61a2bceb7..000000000 --- a/examples/all-sky/all_tests.sh +++ /dev/null @@ -1,9 +0,0 @@ -set -eux -./rrtmgp_allsky 24 72 1 rrtmgp-allsky-lw.nc \ - ${RRTMGP_DATA}/rrtmgp-gas-lw-g256.nc ${RRTMGP_DATA}/rrtmgp-clouds-lw.nc ${RRTMGP_DATA}/rrtmgp-aerosols-merra-lw.nc -./rrtmgp_allsky 24 72 1 rrtmgp-allsky-sw.nc \ - ${RRTMGP_DATA}/rrtmgp-gas-sw-g224.nc ${RRTMGP_DATA}/rrtmgp-clouds-sw.nc ${RRTMGP_DATA}/rrtmgp-aerosols-merra-sw.nc -./rrtmgp_allsky 24 72 1 rrtmgp-allsky-lw-no-aerosols.nc \ - ${RRTMGP_DATA}/rrtmgp-gas-lw-g256.nc ${RRTMGP_DATA}/rrtmgp-clouds-lw.nc -./rrtmgp_allsky 24 72 1 rrtmgp-allsky-sw-no-aerosols.nc \ - ${RRTMGP_DATA}/rrtmgp-gas-sw-g224.nc ${RRTMGP_DATA}/rrtmgp-clouds-sw.nc \ No newline at end of file diff --git a/examples/rfmip-clear-sky/CMakeLists.txt b/examples/rfmip-clear-sky/CMakeLists.txt new file mode 100644 index 000000000..6cfd6bba3 --- /dev/null +++ b/examples/rfmip-clear-sky/CMakeLists.txt @@ -0,0 +1,59 @@ +set(TEST_INPUTS + # cmake-format: sort + ${RRTMGP_DATA}/examples/rfmip-clear-sky/inputs/multiple_input4MIPs_radiation_RFMIP_UColorado-RFMIP-1-2_none.nc + ${RRTMGP_DATA}/examples/rfmip-clear-sky/inputs/rld_Efx_RTE-RRTMGP-181204_rad-irf_r1i1p1f1_gn.nc + ${RRTMGP_DATA}/examples/rfmip-clear-sky/inputs/rlu_Efx_RTE-RRTMGP-181204_rad-irf_r1i1p1f1_gn.nc + ${RRTMGP_DATA}/examples/rfmip-clear-sky/inputs/rsd_Efx_RTE-RRTMGP-181204_rad-irf_r1i1p1f1_gn.nc + ${RRTMGP_DATA}/examples/rfmip-clear-sky/inputs/rsu_Efx_RTE-RRTMGP-181204_rad-irf_r1i1p1f1_gn.nc +) + +add_custom_target( + rfmip_clear_sky_test_inputs ALL + COMMAND + ${CMAKE_COMMAND} -E copy_if_different ${TEST_INPUTS} + ${CMAKE_CURRENT_BINARY_DIR}/ + COMMENT "Copying required test input files" +) + +add_library(rfmip_clear_utils STATIC mo_rfmip_io.F90) +target_link_libraries(rfmip_clear_utils PUBLIC examples_utils) + +foreach( + test_executable IN + ITEMS # cmake-format: sort + rrtmgp_rfmip_lw + rrtmgp_rfmip_sw +) + add_executable(${test_executable} ${test_executable}.F90) + target_link_libraries(${test_executable} PRIVATE rfmip_clear_utils) + add_dependencies(${test_executable} rfmip_clear_sky_test_inputs) +endforeach() + +add_test( + NAME rfmip_test_lw + COMMAND + rrtmgp_rfmip_lw 8 + multiple_input4MIPs_radiation_RFMIP_UColorado-RFMIP-1-2_none.nc + ${RRTMGP_DATA}/rrtmgp-gas-lw-g256.nc +) + +add_test( + NAME rfmip_test_sw + COMMAND + rrtmgp_rfmip_sw 8 + multiple_input4MIPs_radiation_RFMIP_UColorado-RFMIP-1-2_none.nc + ${RRTMGP_DATA}/rrtmgp-gas-sw-g224.nc +) + +add_test( + NAME check_rfmip_lw_sw + COMMAND + ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/examples/compare-to-reference.py + --ref_dir ${RRTMGP_DATA}/examples/rfmip-clear-sky/reference --tst_dir + ${CMAKE_CURRENT_BINARY_DIR} --variables rld rlu rsd rsu --file_names + rld_Efx_RTE-RRTMGP-181204_rad-irf_r1i1p1f1_gn.nc + rlu_Efx_RTE-RRTMGP-181204_rad-irf_r1i1p1f1_gn.nc + rsd_Efx_RTE-RRTMGP-181204_rad-irf_r1i1p1f1_gn.nc + rsu_Efx_RTE-RRTMGP-181204_rad-irf_r1i1p1f1_gn.nc --failure_threshold + ${FAILURE_THRESHOLD} +) diff --git a/examples/rfmip-clear-sky/Makefile b/examples/rfmip-clear-sky/Makefile deleted file mode 100644 index 9d60aab9a..000000000 --- a/examples/rfmip-clear-sky/Makefile +++ /dev/null @@ -1,75 +0,0 @@ -#!/usr/bin/env make -# -# Location of RTE+RRTMGP libraries, module files. -# -RRTMGP_ROOT ?= ../.. -RRTMGP_BUILD = $(RRTMGP_ROOT)/build -# -# RRTMGP library, module files -# -LDFLAGS += -L$(RRTMGP_BUILD) -LIBS += -lrrtmgp -lrte -FCINCLUDE += -I$(RRTMGP_BUILD) - -# netcdf Fortran module files has to be in the search path or added via environment variable FCINCLUDE e.g. -#FCINCLUDE += -I$(NFHOME)/include - -# netcdf C and Fortran libraries have to be in the search path or added via environment variable LDFLAGS e.g. -#LDFLAGS += -L$(NFHOME)/lib -L$(NCHOME)/lib -LIBS += -lnetcdff - -VPATH = ../ - -# Compilation rules -%.o: %.F90 - $(FC) $(FCFLAGS) $(FCINCLUDE) -c $< - -%: %.o - $(FC) $(FCFLAGS) -o $@ $^ $(LDFLAGS) $(LIBS) - -# Required netCDF files are in $RRTMGP_DATA -%.nc: - cp ${RRTMGP_DATA}/examples/rfmip-clear-sky/inputs/$@ . - - -# -# Ancillary codes -# -ADDITIONS = mo_simple_netcdf.o mo_rfmip_io.o mo_load_coefficients.o - -all: rrtmgp_rfmip_lw rrtmgp_rfmip_sw - -rrtmgp_rfmip_lw: rrtmgp_rfmip_lw.o $(ADDITIONS) - -rrtmgp_rfmip_lw.o: rrtmgp_rfmip_lw.F90 $(ADDITIONS) - -rrtmgp_rfmip_sw: rrtmgp_rfmip_sw.o $(ADDITIONS) - -rrtmgp_rfmip_sw.o: rrtmgp_rfmip_sw.F90 $(ADDITIONS) - -mo_rfmip_io.o: mo_rfmip_io.F90 mo_simple_netcdf.o - -mo_load_coefficients.o: mo_load_coefficients.F90 mo_simple_netcdf.o - -# The default location of the input data: -RRTMGP_DATA ?= $(RRTMGP_ROOT)/rrtmgp-data - -tests: rrtmgp_rfmip_lw rrtmgp_rfmip_sw \ - multiple_input4MIPs_radiation_RFMIP_UColorado-RFMIP-1-2_none.nc \ - rld_Efx_RTE-RRTMGP-181204_rad-irf_r1i1p1f1_gn.nc rlu_Efx_RTE-RRTMGP-181204_rad-irf_r1i1p1f1_gn.nc \ - rsd_Efx_RTE-RRTMGP-181204_rad-irf_r1i1p1f1_gn.nc rsu_Efx_RTE-RRTMGP-181204_rad-irf_r1i1p1f1_gn.nc - $(RUN_CMD) ./rrtmgp_rfmip_lw 8 multiple_input4MIPs_radiation_RFMIP_UColorado-RFMIP-1-2_none.nc ${RRTMGP_DATA}/rrtmgp-gas-lw-g256.nc - $(RUN_CMD) ./rrtmgp_rfmip_sw 8 multiple_input4MIPs_radiation_RFMIP_UColorado-RFMIP-1-2_none.nc ${RRTMGP_DATA}/rrtmgp-gas-sw-g224.nc - -# The default failure threshold: -FAILURE_THRESHOLD ?= 7.e-4 -# Make it available to the scripts: -export FAILURE_THRESHOLD - -check: - $${PYTHON-python} ${RRTMGP_ROOT}/examples/compare-to-reference.py \ - --ref_dir ${RRTMGP_DATA}/examples/rfmip-clear-sky/reference --tst_dir ${RRTMGP_ROOT}/examples/rfmip-clear-sky \ - --var rld rlu rsd rsu --file r??_Efx_RTE-RRTMGP-181204_rad-irf_r1i1p1f1_gn.nc - -clean: - -rm rrtmgp_rfmip_sw rrtmgp_rfmip_lw *.o *.mod *.optrpt *.nc diff --git a/rrtmgp-frontend/CMakeLists.txt b/rrtmgp-frontend/CMakeLists.txt new file mode 100644 index 000000000..3af68510d --- /dev/null +++ b/rrtmgp-frontend/CMakeLists.txt @@ -0,0 +1,24 @@ +set(gas_optics_source_dir ${PROJECT_SOURCE_DIR}/gas-optics) + +add_library( + rrtmgp STATIC # cmake-format: sort + ${gas_optics_source_dir}/mo_gas_concentrations.F90 + ${gas_optics_source_dir}/mo_gas_optics.F90 + ${gas_optics_source_dir}/mo_gas_optics_constants.F90 + ${gas_optics_source_dir}/mo_gas_optics_util_string.F90 + mo_aerosol_optics_rrtmgp_merra.F90 + mo_cloud_optics_rrtmgp.F90 + mo_gas_optics_rrtmgp.F90 +) + +target_include_directories( + rrtmgp + PUBLIC + $:${CMAKE_Fortran_MODULE_DIRECTORY}>> +) + +target_link_libraries( + rrtmgp + PUBLIC rrtmgpkernels + PRIVATE rte +) diff --git a/rrtmgp-frontend/Make.depends b/rrtmgp-frontend/Make.depends deleted file mode 100644 index 1c70a9a08..000000000 --- a/rrtmgp-frontend/Make.depends +++ /dev/null @@ -1,47 +0,0 @@ -RRTMGP_FORTRAN_INTERFACE = \ - mo_gas_optics_util_string.o \ - mo_gas_optics_constants.o \ - mo_gas_concentrations.o \ - mo_gas_optics.o \ - mo_gas_optics_rrtmgp.o - -##### -# RRTMGP: RRTM for GCM Applications - Parallel -# Built on top of RTE, requiring mo_rte_kind.o, mo_rte_util_array.o, mo_rte_util_array_validation.o, mo_optical_props.o -# -# Physical constants -# -mo_gas_optics_constants.o: $(RTE_FORTRAN_INTERFACE) mo_gas_optics_constants.F90 -# -# Utility -# -mo_gas_optics_util_string.o: mo_gas_optics_util_string.F90 - -# -# Gas concentrations - used by gas optics base class -# -mo_gas_concentrations.o: $(RTE_FORTRAN_INTERFACE) mo_gas_concentrations.F90 - -# -# Gas optics base class -# -mo_gas_optics.o: \ - $(RTE_FORTRAN_INTERFACE) mo_gas_concentrations.o \ - mo_gas_optics.F90 - -# -# RRTMGP gas optics -# -mo_gas_optics_rrtmgp.o: \ - $(RTE_FORTRAN_INTERFACE) \ - mo_gas_optics_constants.o mo_gas_optics_util_string.o \ - mo_gas_concentrations.o \ - mo_gas_optics.o \ - mo_gas_optics_rrtmgp_kernels.o mo_gas_optics_rrtmgp.F90 - -# -# RRTMGP cloud optics -# -mo_cloud_optics_rrtmgp.o: \ - $(RTE_FORTRAN_INTERFACE) \ - mo_cloud_optics_rrtmgp_kernels.o mo_cloud_optics_rrtmgp.F90 diff --git a/rrtmgp-kernels/CMakeLists.txt b/rrtmgp-kernels/CMakeLists.txt new file mode 100644 index 000000000..e85c187cf --- /dev/null +++ b/rrtmgp-kernels/CMakeLists.txt @@ -0,0 +1,38 @@ +add_library(rrtmgpkernels OBJECT) + +if(KERNEL_MODE STREQUAL "extern") + target_sources( + rrtmgpkernels + PRIVATE # cmake-format: sort + api/mo_cloud_optics_rrtmgp_kernels.F90 + api/mo_gas_optics_rrtmgp_kernels.F90 + api/rrtmgp_kernels.h + ) +else() + target_sources( + rrtmgpkernels + PRIVATE # cmake-format: sort + mo_cloud_optics_rrtmgp_kernels.F90 + ) + if(KERNEL_MODE STREQUAL "accel") + target_sources( + rrtmgpkernels + PRIVATE # cmake-format: sort + accel/mo_gas_optics_rrtmgp_kernels.F90 + ) + else() + target_sources( + rrtmgpkernels + PRIVATE # cmake-format: sort + mo_gas_optics_rrtmgp_kernels.F90 + ) + endif() +endif() + +target_include_directories( + rrtmgpkernels + PUBLIC + $:${CMAKE_Fortran_MODULE_DIRECTORY}>> +) + +target_link_libraries(rrtmgpkernels PRIVATE rte) diff --git a/rrtmgp-kernels/Make.depends b/rrtmgp-kernels/Make.depends deleted file mode 100644 index 0953a6247..000000000 --- a/rrtmgp-kernels/Make.depends +++ /dev/null @@ -1,10 +0,0 @@ -RRTMGP_FORTRAN_KERNELS = mo_gas_optics_rrtmgp_kernels.o mo_cloud_optics_rrtmgp_kernels.o - -# -# Gas optics -# -mo_gas_optics_rrtmgp_kernels.o: $(RTE_FORTRAN_KERNELS) mo_gas_optics_rrtmgp_kernels.F90 -# -# Cloud optics -# -mo_cloud_optics_rrtmgp_kernels.o: $(RTE_FORTRAN_KERNELS) mo_cloud_optics_rrtmgp_kernels.F90 diff --git a/rte-frontend/CMakeLists.txt b/rte-frontend/CMakeLists.txt new file mode 100644 index 000000000..3a313038b --- /dev/null +++ b/rte-frontend/CMakeLists.txt @@ -0,0 +1,18 @@ +add_library( + rte STATIC # cmake-format: sort + mo_fluxes.F90 + mo_optical_props.F90 + mo_rte_config.F90 + mo_rte_lw.F90 + mo_rte_sw.F90 + mo_rte_util_array_validation.F90 + mo_source_functions.F90 +) + +target_include_directories( + rte + PUBLIC + $:${CMAKE_Fortran_MODULE_DIRECTORY}>> +) + +target_link_libraries(rte PUBLIC rtekernels) diff --git a/rte-frontend/Make.depends b/rte-frontend/Make.depends deleted file mode 100644 index 5b5b098a5..000000000 --- a/rte-frontend/Make.depends +++ /dev/null @@ -1,51 +0,0 @@ -RTE_FORTRAN_INTERFACE = \ - mo_rte_kind.o \ - mo_rte_config.o \ - mo_rte_util_array_validation.o \ - mo_optical_props.o \ - mo_source_functions.o \ - mo_fluxes.o \ - mo_rte_lw.o \ - mo_rte_sw.o - -################################## -# RTE - Radiative transfer for energetics -################################## -# -# -mo_rte_config.o: mo_rte_config.F90 mo_rte_kind.o -# -# -mo_rte_util_array_validation.o: mo_rte_util_array_validation.F90 mo_rte_kind.o -# -# Optical properties -# -mo_optical_props.o: mo_rte_kind.o mo_rte_util_array_validation.o mo_optical_props_kernels.o mo_optical_props.F90 -# -# Source functions -# -mo_source_functions.o: mo_rte_kind.o mo_optical_props.o mo_source_functions.F90 -# -# Flux reduction -# -mo_fluxes.o: mo_rte_kind.o mo_fluxes_broadband_kernels.o mo_rte_config.o mo_optical_props.o mo_rte_util_array_validation.o mo_fluxes.F90 - -mo_rte_lw.o: mo_rte_kind.o \ - mo_rte_config.o \ - mo_rte_util_array.o \ - mo_rte_util_array_validation.o \ - mo_optical_props.o \ - mo_source_functions.o \ - mo_fluxes.o \ - mo_rte_solver_kernels.o \ - mo_rte_lw.F90 - -mo_rte_sw.o: mo_rte_kind.o \ - mo_rte_config.o \ - mo_rte_util_array.o \ - mo_rte_util_array_validation.o \ - mo_optical_props.o \ - mo_source_functions.o \ - mo_fluxes.o \ - mo_rte_solver_kernels.o \ - mo_rte_sw.F90 diff --git a/rte-kernels/CMakeLists.txt b/rte-kernels/CMakeLists.txt new file mode 100644 index 000000000..d7e4a75a4 --- /dev/null +++ b/rte-kernels/CMakeLists.txt @@ -0,0 +1,51 @@ +add_library(rtekernels OBJECT mo_rte_kind.F90) + +if(KERNEL_MODE STREQUAL "extern") + target_sources( + rtekernels + PRIVATE # cmake-format: sort + api/mo_fluxes_broadband_kernels.F90 + api/mo_optical_props_kernels.F90 + api/mo_rte_solver_kernels.F90 + api/mo_rte_util_array.F90 + api/rte_kernels.h + api/rte_types.h + ) +else() + target_sources( + rtekernels + PRIVATE # cmake-format: sort + mo_fluxes_broadband_kernels.F90 + mo_rte_util_array.F90 + ) + if(KERNEL_MODE STREQUAL "accel") + target_sources( + rtekernels + PRIVATE # cmake-format: sort + accel/mo_optical_props_kernels.F90 + accel/mo_rte_solver_kernels.F90 + ) + else() + target_sources( + rtekernels + PRIVATE # cmake-format: sort + mo_optical_props_kernels.F90 + mo_rte_solver_kernels.F90 + ) + endif() +endif() + +include(CheckFortranNeedsCBool) +check_fortran_needs_cbool(RTE_USE_C_BOOL) + +target_compile_definitions( + rtekernels + PRIVATE $<$:RTE_USE_SP> + $<$:RTE_USE_CBOOL> +) + +target_include_directories( + rtekernels + PUBLIC + $:${CMAKE_Fortran_MODULE_DIRECTORY}>> +) diff --git a/rte-kernels/Make.depends b/rte-kernels/Make.depends deleted file mode 100644 index 6a8911e09..000000000 --- a/rte-kernels/Make.depends +++ /dev/null @@ -1,27 +0,0 @@ -RTE_FORTRAN_KERNELS = \ - mo_rte_kind.o \ - mo_rte_util_array.o \ - mo_rte_solver_kernels.o \ - mo_optical_props_kernels.o \ - mo_fluxes_broadband_kernels.o \ - -# -# Array utilities -# -mo_rte_util_array.o: mo_rte_kind.o mo_rte_util_array.F90 - -# -# Optical properties -# -mo_optical_props_kernels.o: mo_rte_kind.o mo_optical_props_kernels.F90 - -# -# Flux reduction -# -mo_fluxes_broadband_kernels.o : mo_rte_kind.o mo_fluxes_broadband_kernels.F90 - -# -# Radiative transfer -# -mo_rte_solver_kernels.o: mo_rte_kind.o mo_rte_util_array.o mo_rte_solver_kernels.F90 - diff --git a/rte-frontend/mo_rte_kind.F90 b/rte-kernels/mo_rte_kind.F90 similarity index 100% rename from rte-frontend/mo_rte_kind.F90 rename to rte-kernels/mo_rte_kind.F90 diff --git a/setup.sh b/setup.sh new file mode 100755 index 000000000..66bb3a651 --- /dev/null +++ b/setup.sh @@ -0,0 +1,23 @@ +rm -rf build +# conda --version +# conda env create -f environment-dev.yml + +FC=gfortran +FFLAGS='-ffree-line-length-none -m64 -std=f2008 -march=native -fbounds-check -fmodule-private -fimplicit-none -finit-real=nan' +RTE_ENABLE_SP=OFF +KERNEL_MODE=default +FAILURE_THRESHOLD='7.e-4' + +cmake -S . -B build -G "Ninja" \ + -DCMAKE_Fortran_COMPILER=$FC \ + -DCMAKE_Fortran_FLAGS="$FFLAGS" \ + -DRTE_ENABLE_SP=$RTE_ENABLE_SP \ + -DKERNEL_MODE=$KERNEL_MODE \ + -DBUILD_TESTING=ON \ + -DFAILURE_THRESHOLD=$FAILURE_THRESHOLD \ + -DCMAKE_BUILD_TYPE=Release + +# cmake --build build --target all --parallel + +# The --test-dir option is available only starting CMake 3.20: +# ctest -V --test-dir build diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt new file mode 100644 index 000000000..6f92e3281 --- /dev/null +++ b/tests/CMakeLists.txt @@ -0,0 +1,114 @@ +set(TEST_INPUTS + ${RRTMGP_DATA}/examples/rfmip-clear-sky/inputs/multiple_input4MIPs_radiation_RFMIP_UColorado-RFMIP-1-2_none.nc +) + +add_custom_target( + tests_test_inputs ALL + COMMAND + ${CMAKE_COMMAND} -E copy_if_different ${TEST_INPUTS} + ${CMAKE_CURRENT_BINARY_DIR}/ + COMMENT "Copying required test input files" +) + +set(CMAKE_Fortran_MODULE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/modules) + +set(extensions_source_dir ${PROJECT_SOURCE_DIR}/extensions) + +add_library( + test_utils STATIC # cmake-format: sort + ${extensions_source_dir}/mo_compute_bc.F90 + ${extensions_source_dir}/mo_heating_rates.F90 + ${extensions_source_dir}/mo_rrtmgp_clr_all_sky.F90 + ${extensions_source_dir}/mo_zenith_angle_spherical_correction.F90 + ${extensions_source_dir}/solar_variability/mo_solar_variability.F90 + mo_gas_optics_defs_rrtmgp.F90 + mo_rcemip_profiles.F90 + mo_testing_io.F90 + mo_testing_utils.F90 +) + +target_include_directories( + test_utils + PUBLIC + $:${CMAKE_Fortran_MODULE_DIRECTORY}>> + ${NetCDF_Fortran_INCLUDE_DIR} +) + +target_link_libraries(test_utils PUBLIC rfmip_clear_utils) + +foreach( + test_executable IN + ITEMS # cmake-format: sort + check_equivalence + check_variants + rte_lw_solver_unit_tests + rte_optic_prop_unit_tests + rte_sw_solver_unit_tests + test_zenith_angle_spherical_correction +) + add_executable(${test_executable} ${test_executable}.F90) + target_link_libraries(${test_executable} PRIVATE test_utils) +endforeach() + +add_dependencies(check_equivalence tests_test_inputs) +add_dependencies(check_variants tests_test_inputs) + +add_test(NAME rte_optic_prop_unit_tests COMMAND rte_optic_prop_unit_tests) +add_test(NAME rte_lw_solver_unit_tests COMMAND rte_lw_solver_unit_tests) +add_test(NAME rte_sw_solver_unit_tests COMMAND rte_sw_solver_unit_tests) + +add_test( + NAME check_equivalence_lw_g256 + COMMAND + check_equivalence + multiple_input4MIPs_radiation_RFMIP_UColorado-RFMIP-1-2_none.nc + ${RRTMGP_DATA}/rrtmgp-gas-lw-g256.nc +) + +add_test( + NAME check_equivalence_lw_g128 + COMMAND + check_equivalence + multiple_input4MIPs_radiation_RFMIP_UColorado-RFMIP-1-2_none.nc + ${RRTMGP_DATA}/rrtmgp-gas-lw-g128.nc +) + +add_test( + NAME check_equivalence_sw_g224 + COMMAND + check_equivalence + multiple_input4MIPs_radiation_RFMIP_UColorado-RFMIP-1-2_none.nc + ${RRTMGP_DATA}/rrtmgp-gas-sw-g224.nc +) + +add_test( + NAME check_equivalence_sw_g112 + COMMAND + check_equivalence + multiple_input4MIPs_radiation_RFMIP_UColorado-RFMIP-1-2_none.nc + ${RRTMGP_DATA}/rrtmgp-gas-sw-g112.nc +) + +add_test( + NAME check_variants_lw + COMMAND + check_variants + multiple_input4MIPs_radiation_RFMIP_UColorado-RFMIP-1-2_none.nc + ${RRTMGP_DATA}/rrtmgp-gas-lw-g256.nc ${RRTMGP_DATA}/rrtmgp-gas-lw-g128.nc +) + +add_test( + NAME check_variants_sw + COMMAND + check_variants + multiple_input4MIPs_radiation_RFMIP_UColorado-RFMIP-1-2_none.nc + ${RRTMGP_DATA}/rrtmgp-gas-sw-g224.nc ${RRTMGP_DATA}/rrtmgp-gas-sw-g112.nc +) + +add_custom_target( + validation-plots + COMMAND + ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/validation-plots.py + --input_file multiple_input4MIPs_radiation_RFMIP_UColorado-RFMIP-1-2_none.nc + COMMENT "Generating validation plots" +) diff --git a/tests/Makefile b/tests/Makefile deleted file mode 100644 index de61c4a22..000000000 --- a/tests/Makefile +++ /dev/null @@ -1,97 +0,0 @@ -#!/usr/bin/env make -# -# Location of RTE+RRTMGP libraries, module files. -# -RRTMGP_ROOT ?= .. -RRTMGP_BUILD = $(RRTMGP_ROOT)/build -# -# RRTMGP library, module files -# -LDFLAGS += -L$(RRTMGP_BUILD) -LIBS += -lrrtmgp -lrte -FCINCLUDE += -I$(RRTMGP_BUILD) - -# netcdf Fortran module files has to be in the search path or added via environment variable FCINCLUDE e.g. -#FCINCLUDE += -I$(NFHOME)/include - -# netcdf C and Fortran libraries have to be in the search path or added via environment variable LDFLAGS e.g. -#LDFLAGS += -L$(NFHOME)/lib -L$(NCHOME)/lib -LIBS += -lnetcdff - -VPATH = $(RRTMGP_ROOT)/examples:$(RRTMGP_ROOT)/examples/rfmip-clear-sky:$(RRTMGP_ROOT)/examples/all-sky -VPATH += $(RRTMGP_ROOT)/rrtmgp-frontend:$(RRTMGP_ROOT)/extensions:$(RRTMGP_ROOT)/:$(RRTMGP_ROOT)/extensions/solar_variability - -# Compilation rules -%.o: %.F90 - $(FC) $(FCFLAGS) $(FCINCLUDE) -c $< -%: %.o - $(FC) $(FCFLAGS) -o $@ $^ $(LDFLAGS) $(LIBS) - - -# -# Extra sources -- extensions to RRTMGP classes, shared infrastructure, local sources -# -ADDITIONS = mo_heating_rates.o mo_compute_bc.o mo_rrtmgp_clr_all_sky.o -ADDITIONS += mo_gas_optics_defs_rrtmgp.o -# File I/O -ADDITIONS += mo_simple_netcdf.o mo_rfmip_io.o -ADDITIONS += mo_testing_io.o mo_testing_utils.o -# Cloud optics -CLOUDS += mo_cloud_sampling.o mo_cloud_optics_rrtmgp.o mo_load_cloud_coefficients.o mo_garand_atmos_io.o -# Solar variability -ADDITIONS += mo_solar_variability.o - -# Many codes will need to be updated if the library changes -# LIB_DEPS = $(RRTMGP_BUILD)/librte.a $(RRTMGP_BUILD)/librrtmgp.a -# -# Targets -# -all: check_variants check_equivalence test_zenith_angle_spherical_correction rte_sw_solver_unit_tests rte_optic_prop_unit_tests rte_lw_solver_unit_tests - -check_equivalence: $(ADDITIONS) $(LIB_DEPS) check_equivalence.o -check_equivalence.o: $(ADDITIONS) $(LIB_DEPS) check_equivalence.F90 - -check_variants: $(ADDITIONS) $(LIB_DEPS) check_variants.o -check_variants.o: $(ADDITIONS) $(LIB_DEPS) check_variants.F90 - -test_zenith_angle_spherical_correction: mo_zenith_angle_spherical_correction.o mo_rcemip_profiles.o $(ADDITIONS) $(LIB_DEPS) test_zenith_angle_spherical_correction.o -test_zenith_angle_spherical_correction.o: mo_zenith_angle_spherical_correction.o mo_rcemip_profiles.o $(ADDITIONS) $(LIB_DEPS) test_zenith_angle_spherical_correction.F90 - -mo_testing_io.o: $(LIB_DEPS) mo_simple_netcdf.o mo_testing_io.F90 - -mo_cloud_optics_rrtmgp.o: $(LIB_DEPS) mo_cloud_optics_rrtmgp.F90 -mo_load_cloud_coefficients.o: $(LIB_DEPS) mo_simple_netcdf.o mo_cloud_optics_rrtmgp.o mo_load_cloud_coefficients.F90 -mo_cloud_sampling.o: $(LIB_DEPS) mo_cloud_sampling.F90 - -mo_gas_optics_defs_rrtmgp.o: $(LIB_DEPS) mo_testing_utils.o mo_simple_netcdf.o mo_gas_optics_defs_rrtmgp.F90 - -mo_load_coefficients.o: $(LIB_DEPS) mo_simple_netcdf.o mo_load_coefficients.F90 -mo_rfmip_io.o: $(LIB_DEPS) mo_simple_netcdf.o mo_rfmip_io.F90 -mo_simple_netcdf.o: $(LIB_DEPS) mo_simple_netcdf.F90 - -rte_optic_prop_unit_tests.o: $(LIB_DEPS) mo_testing_utils.o rte_optic_prop_unit_tests.F90 -rte_optic_prop_unit_tests : $(LIB_DEPS) mo_testing_utils.o rte_optic_prop_unit_tests.o - -rte_lw_solver_unit_tests.o: $(LIB_DEPS) mo_testing_utils.o rte_lw_solver_unit_tests.F90 -rte_lw_solver_unit_tests : $(LIB_DEPS) mo_testing_utils.o rte_lw_solver_unit_tests.o - -rte_sw_solver_unit_tests.o: $(LIB_DEPS) mo_testing_utils.o rte_sw_solver_unit_tests.F90 -rte_sw_solver_unit_tests : $(LIB_DEPS) mo_testing_utils.o rte_sw_solver_unit_tests.o - - -# The default location of the input data: -RRTMGP_DATA ?= $(RRTMGP_ROOT)/rrtmgp-data -# Make it available to the scripts: -export RRTMGP_DATA - -.PHONY: tests -tests: check_variants check_equivalence test_zenith_angle_spherical_correction rte_sw_solver_unit_tests rte_optic_prop_unit_tests rte_lw_solver_unit_tests - cp ${RRTMGP_DATA}/examples/rfmip-clear-sky/inputs/multiple_input4MIPs_radiation_RFMIP_UColorado-RFMIP-1-2_none.nc ./test_atmospheres.nc - $(RUN_CMD) bash all_tests.sh - - -check: - echo "Nothing to check in tests/" - -clean: - -rm clear_sky_regression *.o *.optrpt *.mod diff --git a/tests/all_tests.sh b/tests/all_tests.sh deleted file mode 100644 index 7752f4064..000000000 --- a/tests/all_tests.sh +++ /dev/null @@ -1,10 +0,0 @@ -set -eux -./rte_optic_prop_unit_tests -./rte_lw_solver_unit_tests -./rte_sw_solver_unit_tests -./check_equivalence test_atmospheres.nc ${RRTMGP_DATA}/rrtmgp-gas-lw-g256.nc -./check_equivalence test_atmospheres.nc ${RRTMGP_DATA}/rrtmgp-gas-lw-g128.nc -./check_equivalence test_atmospheres.nc ${RRTMGP_DATA}/rrtmgp-gas-sw-g224.nc -./check_equivalence test_atmospheres.nc ${RRTMGP_DATA}/rrtmgp-gas-sw-g112.nc -./check_variants test_atmospheres.nc ${RRTMGP_DATA}/rrtmgp-gas-lw-g256.nc ${RRTMGP_DATA}/rrtmgp-gas-lw-g128.nc -./check_variants test_atmospheres.nc ${RRTMGP_DATA}/rrtmgp-gas-sw-g224.nc ${RRTMGP_DATA}/rrtmgp-gas-sw-g112.nc \ No newline at end of file diff --git a/tests/validation-plots.py b/tests/validation-plots.py old mode 100644 new mode 100755 index 4dcc9434b..99847da97 --- a/tests/validation-plots.py +++ b/tests/validation-plots.py @@ -1,3 +1,5 @@ +#! /usr/bin/env python + import colorcet as cc import matplotlib as mpl import matplotlib.pyplot as plt @@ -7,6 +9,7 @@ import xarray as xr from matplotlib.backends.backend_pdf import PdfPages +import argparse def mae(diff, col_dim): # @@ -62,6 +65,21 @@ def construct_lbl_esgf_root(var, esgf_node="llnl"): ######################################################################## def main(): + parser = argparse.ArgumentParser() + parser.add_argument( + "--input_file", + help="Path to the input NetCDF file (test_atmosphere.nc)." + ) + parser.add_argument( + "--output_pdf", + help="Path to the output PDF file for validation plots.", + default="validation-figures.pdf" + ) + args = parser.parse_args() + + input_file = args.input_file + output_pdf = args.output_pdf + warnings.simplefilter("ignore", xr.SerializationWarning) # # Reference values from LBLRTM - download locally, since OpenDAP access is @@ -87,7 +105,7 @@ def main(): # # Open the test results # - gp = xr.open_dataset("test_atmospheres.nc") + gp = xr.open_dataset(input_file) # # Does the flux plus the Jacobian equal a calculation with perturbed surface # temperature? @@ -120,7 +138,7 @@ def main(): plev.load() gpi.load() lbli.load() - with PdfPages('validation-figures.pdf') as pdf: + with PdfPages(output_pdf) as pdf: ######################################################################## # Longwave ########################################################################