Update tests.yml

.github/workflows/tests.yml

@@ -0,0 +1,139 @@
+name: docker
+
+on:
+  push:
+    branches:
+    - 'main'
+  pull_request: 
+jobs:
+  persee:
+    runs-on: self-hosted
+    environment: development
+    steps:
+    -
+      name: Env check
+      run: |
+        source ~/.bash_profile
+        rm -rf build
+        mkdir build
+        hostname >build/hsotname.out
+        pwd > build/pwd.out
+        touch build/test.out
+        module av gcc 
+        echo "Working?"
+
+  docker:
+    runs-on: ubuntu-latest
+    steps:
+    -
+      name: Checkout
+      uses: actions/checkout@v3
+    -
+      name: Set up QEMU
+      uses: docker/setup-qemu-action@v2
+    -
+      name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v2
+    -
+      name: Login to GitHub Container Registry
+      uses: docker/login-action@v2
+      with:
+        registry: ghcr.io
+        username: ${{ github.repository_owner }}
+        password: ${{ secrets.GITHUB_TOKEN }}
+    -
+      name: Build and push to tag
+      uses: docker/build-push-action@v3
+      with:
+        context: ./docker/gyselalibxx_env
+        cache-from: type=registry,ref=ghcr.io/gyselax/gyselalibxx_env:latest
+        pull: true
+        push: true
+        tags: ghcr.io/gyselax/gyselalibxx_env:${{ github.sha }}
+    -
+      name: Build and push latest
+      if: github.event_name == 'push' && github.ref_name == 'main'
+      uses: docker/build-push-action@v3
+      with:
+        context: ./docker/gyselalibxx_env
+        cache-from: type=registry,ref=ghcr.io/gyselax/gyselalibxx_env:latest
+        pull: true
+        push: true
+        tags: ghcr.io/gyselax/gyselalibxx_env:latest
+
+  tests:
+    runs-on: ubuntu-latest
+    needs: docker
+    container:
+      image: ghcr.io/gyselax/gyselalibxx_env:${{ github.sha }}
+      options: --user root
+    steps:
+    - 
+      name: Checkout gyselalibxx
+      uses: actions/checkout@v3
+      with: { submodules: recursive }
+    - 
+      name: Run tests
+      run: |
+        mkdir build
+        cd build
+        cmake -DCMAKE_CXX_FLAGS=-Wall -DBUILD_BENCHMARKS=ON -DKokkos_ENABLE_DEPRECATED_CODE_3=OFF -DKokkos_ENABLE_DEPRECATION_WARNINGS=OFF ..
+        make -j 2 VERBOSE=1
+        ctest --output-on-failure --timeout 5 -LE test_on_Release_only
+
+  tests_Release:
+    runs-on: ubuntu-latest
+    needs: docker
+    container:
+      image: ghcr.io/gyselax/gyselalibxx_env:${{ github.sha }}
+      options: --user root
+    steps:
+    - 
+      name: Checkout gyselalibxx
+      uses: actions/checkout@v3
+      with: { submodules: recursive }
+    - 
+      name: Run tests
+      run: |
+        mkdir build
+        cd build
+        cmake -DCMAKE_CXX_FLAGS=-Wall -DBUILD_BENCHMARKS=ON -DCMAKE_BUILD_TYPE=Release -DKokkos_ENABLE_DEPRECATED_CODE_3=OFF -DKokkos_ENABLE_DEPRECATION_WARNINGS=OFF ..
+        make -j 2 VERBOSE=1
+        ctest --output-on-failure --timeout 5
+
+  Docs:
+    if: github.event_name == 'push' && github.ref_name == 'main'
+    needs: docker
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+        with:
+          path: code_branch
+      - name: Checkout documentation
+        uses: actions/checkout@v3
+        with:
+          ref: documentation
+          path: doc_branch
+      - name: Install doxygen
+        run: |
+          sudo apt install doxygen
+        shell: bash
+      - name: Generate docs
+        run: |
+          cmake -DGYSELALIBXX_COMPILE_SOURCE=OFF -DBUILD_DOCUMENTATION=1 -B build-docs code_branch
+          cmake --build build-docs
+        shell: bash
+      - name: Update documentation
+        run: |
+          cd doc_branch
+          git rm -r *
+          cp -r ../build-docs/docs/html/* .
+          git add .
+          git config --global user.name "GitHub CI Documentation builder"
+          git config --global user.email "username@users.noreply.github.com"
+          git commit -m "Update docs" || true
+          git push
+        shell: bash
+        env:
+          GH_TOKEN: ${{ github.token }}

.gitignore

@@ -0,0 +1,3 @@
+/build*/
+*.pyc
+/log.build.*

.gitlab-ci.yml

@@ -1,81 +1,200 @@
-cmake_tests_non_periodic:
-  only: [ main, merge_requests ]
-  stage: test
-  needs: []
-  image: jbigot/voicexx_env
+
+include:
+  - local: 'ci_tools/reusable_ci_jobs.yml'
+
+cmake_tests:
+  extends: .cmake_tests
+  image: ghcr.io/gyselax/gyselalibxx_env
+  timeout: 2 hours
   variables:
-    GIT_SUBMODULE_STRATEGY: recursive
+    RELATIVE_RESTART_TOLERANCE: '0.0'
+    ABSOLUTE_RESTART_TOLERANCE: '0.0'
   script: |
+    set -x
+    git config --global --add safe.directory /builds/gysela-developpers/gyselalibxx/vendor/kokkos
     rm -rf build || true
     mkdir build
     cd build
-    cmake -DCMAKE_CXX_FLAGS=-Wall -DBUILD_BENCHMARKS=ON -DVOICEXX_ENABLE_PERIODIC_RDIMX=OFF ..
-    make -j 4 VERBOSE=1
+    cmake -DCMAKE_CXX_FLAGS="-Wall -Werror=unused-variable -Werror=parentheses -Wno-sign-compare" -DBUILD_BENCHMARKS=ON $GENERAL_CMAKE_OPTIONS -DKokkos_ENABLE_SERIAL=ON ..
+    make -j 4
     ctest -j 2 --output-on-failure --timeout 5 -LE test_on_Release_only --output-junit tests.xml
   artifacts:
     when: always
+    expire_in: 1 day
     reports:
       junit: ./build/tests.xml
-cmake_tests:
-  only: [ main, merge_requests ]
-  stage: test
-  needs: []
-  image: jbigot/voicexx_env
+
+cmake_tests_Release:
+  extends: .cmake_tests
+  image: ghcr.io/gyselax/gyselalibxx_env
   variables:
-    GIT_SUBMODULE_STRATEGY: recursive
+    RELATIVE_RESTART_TOLERANCE: '0.0'
+    ABSOLUTE_RESTART_TOLERANCE: '0.0'
   script: |
+    set -x
+    git config --global --add safe.directory /builds/gysela-developpers/gyselalibxx/vendor/kokkos
     rm -rf build || true
     mkdir build
     cd build
-    cmake -DCMAKE_CXX_FLAGS=-Wall -DBUILD_BENCHMARKS=ON ..
-    make -j 4 VERBOSE=1
-    ctest -j 2 --output-on-failure --timeout 5 -LE test_on_Release_only --output-junit tests.xml
+    cmake -DCMAKE_CXX_FLAGS="-Wall -Wno-sign-compare" -DBUILD_BENCHMARKS=ON -DCMAKE_BUILD_TYPE=Release $GENERAL_CMAKE_OPTIONS -DKokkos_ENABLE_SERIAL=ON ..
+    make -j 4
+    ctest -j 2 --output-on-failure --timeout 5 --output-junit tests.xml
+    ls ./tests/geometryXVx/landau/fft/growthrate_t0.0to45.0.png
+    ls ./tests/geometryXVx/landau/fft/frequency_t0.0to45.0.png
   artifacts:
     when: always
+    expire_in: 1 day
+    paths:
+      - ./build/tests/geometryXVx/landau/fft/growthrate_t0.0to45.0.png
+      - ./build/tests/geometryXVx/landau/fft/frequency_t0.0to45.0.png
     reports:
       junit: ./build/tests.xml
-cmake_tests_Release:
-  only: [ main, merge_requests ]
-  stage: test
-  needs: []
-  image: jbigot/voicexx_env
+
+cmake_tests_Release_OMP:
+  extends: .cmake_tests
+  image: ghcr.io/gyselax/gyselalibxx_env
   variables:
-    GIT_SUBMODULE_STRATEGY: recursive
+    RELATIVE_RESTART_TOLERANCE: '1e-14'
+    ABSOLUTE_RESTART_TOLERANCE: '1e-14'
   script: |
+    set -x
+    git config --global --add safe.directory /builds/gysela-developpers/gyselalibxx/vendor/kokkos
     rm -rf build || true
     mkdir build
     cd build
-    cmake -DCMAKE_CXX_FLAGS=-Wall -DBUILD_BENCHMARKS=ON -DCMAKE_BUILD_TYPE=Release ..
-    make -j 4 VERBOSE=1
-    ctest -j 2 --output-on-failure --timeout 5 --output-junit tests.xml
+    cmake -DCMAKE_CXX_FLAGS="-Wall -Wno-sign-compare" -DBUILD_BENCHMARKS=ON -DCMAKE_BUILD_TYPE=Release $GENERAL_CMAKE_OPTIONS -DKokkos_ENABLE_OPENMP=ON ..
+    make -j 4
+    OMP_NUM_THREADS=2 ctest -j 2 --output-on-failure --timeout 5 --output-junit tests.xml
+    ls ./tests/geometryXVx/landau/fft/growthrate_t0.0to45.0.png
+    ls ./tests/geometryXVx/landau/fft/frequency_t0.0to45.0.png
+  artifacts:
+    when: always
+    expire_in: 1 day
+    paths:
+      - ./build/tests/geometryXVx/landau/fft/growthrate_t0.0to45.0.png
+      - ./build/tests/geometryXVx/landau/fft/frequency_t0.0to45.0.png
+    reports:
+      junit: ./build/tests.xml
+
+cmake_tests_Release_GPU:
+  extends: .cmake_tests
+  tags:
+    - persee
+  variables:
+    RELATIVE_RESTART_TOLERANCE: '1e-14'
+    ABSOLUTE_RESTART_TOLERANCE: '1e-14'
+  before_script: |
+    . /data/gyselarunner/spack-0.20.0/share/spack/setup-env.sh
+    spack load gcc@11
+    spack env activate gyselalibxx-env-omp-cuda
+  script: |
+    set -x
+    export OMP_PROC_BIND=spread
+    export OMP_PLACES=threads
+    export OMP_NUM_THREADS=8
+    git config --global --add safe.directory /builds/gysela-developpers/gyselalibxx/vendor/kokkos
+    rm -rf build || true
+    mkdir build
+    cd build
+    cmake -DCMAKE_CXX_COMPILER=${CI_PROJECT_DIR}/vendor/kokkos/bin/nvcc_wrapper \
+          -DCMAKE_CXX_FLAGS="-Wall -Wno-sign-compare -Xcudafe --diag_suppress=unsigned_compare_with_zero -Xcudafe --diag_suppress=integer_sign_change" \
+          -DCMAKE_BUILD_TYPE=Release -DDDC_BUILD_TESTS=${DDC_BUILD_TESTING} $GENERAL_CMAKE_OPTIONS -DKokkos_ARCH_VOLTA70=ON -DKokkos_ENABLE_SERIAL=ON -DKokkos_ENABLE_CUDA=ON ..
+    make -j 8
+    ctest -j 2 --output-on-failure --timeout 5 --output-junit tests.xml -LE ResultsNotTested
+    ls ./tests/geometryXVx/landau/fft/growthrate_t0.0to45.0.png
+    ls ./tests/geometryXVx/landau/fft/frequency_t0.0to45.0.png
   artifacts:
     when: always
+    expire_in: 1 day
     paths:
-      - ./build/tests/landau/fft/dampingrate_t0.0to45.0.png
-      - ./build/tests/landau/fft/frequency_t0.0to45.0.png
+      - ./build/tests/geometryXVx/landau/fft/growthrate_t0.0to45.0.png
+      - ./build/tests/geometryXVx/landau/fft/frequency_t0.0to45.0.png
     reports:
       junit: ./build/tests.xml
+
 indentation:
-  only: [ main, merge_requests ]
+  rules:
+    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
   stage: test
   needs: []
-  image: jbigot/voicexx_env
+  image: ghcr.io/gyselax/gyselalibxx_env
   variables:
     GIT_SUBMODULE_STRATEGY: recursive
   script: |
     ./bin/indent -td
-ddc_sync:
-  only: [ triggers ]
+python_best_practices:
+  rules:
+    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+  stage: test
   needs: []
-  image: jbigot/voicexx_env
-  script:
-  - git config user.email "${GITLAB_USER_EMAIL}"
-  - git config user.name "${GITLAB_USER_NAME}"
-  - git checkout "${CI_COMMIT_BRANCH}"
-  - ./bin/ddc_sync --commit
-  - git push "http://root:${SELF_COMMIT_TOKEN}@${CI_SERVER_HOST}/${CI_PROJECT_PATH}.git" "HEAD:${CI_COMMIT_BRANCH}"
+  image: ghcr.io/gyselax/gyselalibxx_env
+  variables:
+    GIT_SUBMODULE_STRATEGY: recursive
+  before_script:
+    export PYTHONPATH="./post-process/PythonScripts"
+  script: |
+    pylint $(find tests -iname *.py)
+    pylint $(find doc -iname *.py)
+    pylint $(find ci_tools -iname *.py)
+    # Find all files with no extension or a .py extension in post-process/PythonScripts/
+    POST_PROCESS_PYTHON_FILES=$(find post-process/PythonScripts/ -type f ! -name "*.*"; find post-process/PythonScripts -iname *.py)
+    # Get pylint errors without failing
+    pylint ${POST_PROCESS_PYTHON_FILES} > post_process_errors.txt || true
+    # Filter errors on changed files
+    git config --global --add safe.directory $(pwd)
+    git fetch origin main
+    for f in $(git diff origin/main --name-only); do grep $f post_process_errors.txt || true; done | tee filtered_errors.txt
+    # Raise an error if post-process in filtered errors
+    ! grep "post-process" filtered_errors.txt >/dev/null
+
 public_mirror:
-  only: [ main ]
+  rules:
+    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+  tags: [ outssh ]
   needs: []
-  image: jbigot/voicexx_env
+  image: ghcr.io/gyselax/gyselalibxx_env
+  variables:
+    GIT_SUBMODULE_STRATEGY: recursive
   script: ./bin/public_mirror
+
+documentation:
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+  stage: test
+  needs: []
+  image: ghcr.io/gyselax/gyselalibxx_env
+  before_script: |
+    git config --global --add safe.directory $CI_PROJECT_DIR
+    git fetch origin ${CI_MERGE_REQUEST_TARGET_BRANCH_NAME} --recurse-submodules=no
+  script: |
+    rm -rf build || true
+    # Make docs
+    cmake -DGYSELALIBXX_COMPILE_SOURCE=OFF -DBUILD_DOCUMENTATION=1 -B build-docs .
+    cmake --build build-docs
+    # Get files which have changed in this merge request
+    git diff origin/${CI_MERGE_REQUEST_TARGET_BRANCH_NAME}..HEAD --no-indent-heuristic --unified=0 --output=pull_diff.txt --no-color
+    # Filter documentation messages to only complain about modified files
+    python3 ci_tools/check_documentation.py pull_diff.txt build-docs/docs/doxygen.log
+    git diff origin/${CI_MERGE_REQUEST_TARGET_BRANCH_NAME}..HEAD --no-indent-heuristic --unified=0 --output=pull_new_files.txt --no-color --diff-filter=A
+    python3 ci_tools/check_readme_presence.py pull_new_files.txt
+  allow_failure: true
+
+markdown_best_practices:
+  rules:
+    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+  stage: test
+  needs: []
+  image: ghcr.io/gyselax/gyselalibxx_env
+  variables:
+    GIT_SUBMODULE_STRATEGY: recursive
+  script: |
+    MARKDOWN_FILES=$(find . -name "*.md" -not -path "./vendor/*")
+    MARKDOWN_FILES="$MARKDOWN_FILES $(find ./vendor/sll/ -name '*.md')"
+    echo $MARKDOWN_FILES
+    set +e
+    python3 ci_tools/markdown_linter.py ${MARKDOWN_FILES}
+  allow_failure:
+    exit_codes: 2