Merge pull request #179 from loganoz/fix_multiphase_bug

Fix multiphase bug

.github/workflows/CI_sequential_gnu_debug_multiphase.yml

@@ -0,0 +1,300 @@
+# This is a basic workflow to help you get started with Actions
+
+name: CI SEQUENTIAL GNU DEBUG MULTIPHASE
+
+########################################################################
+# Controls when the action will run.
+on:
+  # Triggers the workflow on push or pull request events but only for the master branch
+  #push:
+  #  branches: [ master ]
+  #pull_request:
+  #  branches: [ master ]
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+#env:
+#  HDF5_ROOT : /usr/lib/x86_64-linux-gnu/hdf5/serial/
+# A workflow run is made up of one or more jobs that can run sequentially or in parallel
+jobs:
+  # This workflow contains a single job called "build"
+  build:
+    # The type of runner that the job will run on
+    runs-on: ubuntu-latest
+
+    # Define strategy matrix
+    strategy:
+      fail-fast: false
+      matrix:
+        compiler: ['gfortran']
+        mode: ['DEBUG']
+        comm: ['SEQUENTIAL']
+        enable_threads: ['YES']
+        mkl: ['NO']
+
+    # Steps represent a sequence of tasks that will be executed as part of the job
+
+    steps:
+      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
+      - uses: actions/checkout@v3
+
+      # sudo apt-get install -qq pkg-config libhdf5-serial-dev
+#      - name: Install Linux packages
+#        run: |
+#          sudo apt-get update
+#          sudo apt install libhdf5-dev
+#          dpkg -L libhdf5-dev
+
+      # IF COMPILER = IFORT -> INSTALL IFORT #
+      - name: cache-intel-compilers
+        # if: (matrix.compiler == 'ifort')
+        id: cache-intel-compilers
+        uses: actions/cache@v3
+        with:
+          path: /opt/intel
+          key: intel-${{ runner.os }}-compilers-b
+
+      - name: Intel Apt repository
+        if: (steps.cache-intel-compilers.outputs.cache-hit != 'true')
+        run: |
+          wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB
+          sudo apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB
+          rm GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB
+          echo "deb https://apt.repos.intel.com/oneapi all main" | sudo tee /etc/apt/sources.list.d/oneAPI.list
+          sudo apt-get update
+
+      - name: Install Intel oneAPI
+        if: (steps.cache-intel-compilers.outputs.cache-hit != 'true')
+        run: sudo apt-get install intel-oneapi-compiler-fortran intel-oneapi-compiler-dpcpp-cpp-and-cpp-classic intel-oneapi-mpi intel-oneapi-mpi-devel intel-oneapi-mkl ninja-build
+      # Runs a single command using the runners shell
+      ##- name: Install gfortran
+      ##  run: |
+      ##   sudo add-apt-repository ppa:ubuntu-toolchain-r/test
+      ##   sudo apt update
+      ##   sudo apt-get install gfortran-9
+
+      ##- name: Show gfortran version
+      ##  run: gfortran-9 --version
+
+      - name: Show ${{matrix.compiler}} version
+        run: |
+          source /opt/intel/oneapi/setvars.sh || true
+          printenv >> $GITHUB_ENV
+          ${{matrix.compiler}} --version
+
+      - name: Configure
+        working-directory: ./Solver
+        run: ./configure
+
+      - name: Compile horses3d
+        working-directory: ./Solver
+        run: |
+         source /opt/intel/oneapi/setvars.sh || true
+         make allclean MODE=${{matrix.mode}} COMPILER=${{matrix.compiler}} COMM=${{matrix.comm}} ENABLE_THREADS=${{matrix.enable_threads}} WITH_MKL=${{matrix.mkl}}
+         make MODE=${{matrix.mode}} COMPILER=${{matrix.compiler}} COMM=${{matrix.comm}} ENABLE_THREADS=${{matrix.enable_threads}} WITH_MKL=${{matrix.mkl}}
+
+
+########################################################################
+#######                       MULTIPHASE                        ########
+########################################################################
+#
+# 3) Pipe
+# ----------------------------
+
+      - name: Build MultiphasePipe
+        working-directory: ./Solver/test/Multiphase/Pipe/SETUP
+        run: |
+          source /opt/intel/oneapi/setvars.sh || true
+          make MODE=${{matrix.mode}} COMPILER=${{matrix.compiler}} COMM=${{matrix.comm}} ENABLE_THREADS=${{matrix.enable_threads}} WITH_MKL=${{matrix.mkl}}
+        if: '!cancelled()'
+
+      - name: Run MultiphasePipe
+        working-directory: ./Solver/test/Multiphase/Pipe
+        run: |
+          source /opt/intel/oneapi/setvars.sh || true
+          ./horses3d.mu AnnularFlow.control
+        if: '!cancelled()'
+
+#
+# 4) Entropy conserving test
+# ----------------------------
+
+      - name: Build MultiphaseEntropyConservingTest
+        working-directory: ./Solver/test/Multiphase/EntropyConservingTest/SETUP
+        run: |
+          source /opt/intel/oneapi/setvars.sh || true
+          make MODE=${{matrix.mode}} COMPILER=${{matrix.compiler}} COMM=${{matrix.comm}} ENABLE_THREADS=${{matrix.enable_threads}} WITH_MKL=${{matrix.mkl}}
+        if: '!cancelled()'
+
+      - name: Run MultiphaseEntropyConservingTest
+        working-directory: ./Solver/test/Multiphase/EntropyConservingTest
+        run: |
+          source /opt/intel/oneapi/setvars.sh || true
+          ./horses3d.mu EntropyConservingTest.control
+        if: '!cancelled()'
+
+#
+# 1) Convergence
+# ----------------------------
+
+      - name: Build MultiphaseConvergence
+        working-directory: ./Solver/test/Multiphase/Convergence/SETUP
+        run: |
+          source /opt/intel/oneapi/setvars.sh || true
+          make MODE=${{matrix.mode}} COMPILER=${{matrix.compiler}} COMM=${{matrix.comm}} ENABLE_THREADS=${{matrix.enable_threads}} WITH_MKL=${{matrix.mkl}}
+        if: '!cancelled()'
+
+      - name: Run MultiphaseConvergence
+        working-directory: ./Solver/test/Multiphase/Convergence
+        run: |
+          source /opt/intel/oneapi/setvars.sh || true
+          ./horses3d.mu Convergence.control
+        if: '!cancelled()'
+
+#
+# 2) RisingBubble
+# ----------------------------
+
+      - name: Build MultiphaseRisingBubble
+        working-directory: ./Solver/test/Multiphase/RisingBubble/SETUP
+        run: |
+          source /opt/intel/oneapi/setvars.sh || true
+          make MODE=${{matrix.mode}} COMPILER=${{matrix.compiler}} COMM=${{matrix.comm}} ENABLE_THREADS=${{matrix.enable_threads}} WITH_MKL=${{matrix.mkl}}
+        if: '!cancelled()'
+
+      - name: Run MultiphaseRisingBubble
+        working-directory: ./Solver/test/Multiphase/RisingBubble
+        run: |
+          source /opt/intel/oneapi/setvars.sh || true
+          ./horses3d.mu RisingBubble.control
+        if: '!cancelled()'
+
+
+########################################################################
+#######                      INCOMPRESSIBLE                     ########
+########################################################################
+#
+# 1) Convergence
+# ----------------------------
+
+      - name: Build iNS-Convergence
+        working-directory: ./Solver/test/IncompressibleNS/Convergence/SETUP
+        run: |
+          source /opt/intel/oneapi/setvars.sh || true
+          make MODE=${{matrix.mode}} COMPILER=${{matrix.compiler}} COMM=${{matrix.comm}} ENABLE_THREADS=${{matrix.enable_threads}} WITH_MKL=${{matrix.mkl}}
+        if: '!cancelled()'
+
+      - name: Run iNS-Convergence
+        working-directory: ./Solver/test/IncompressibleNS/Convergence
+        run: |
+          source /opt/intel/oneapi/setvars.sh || true
+          ./horses3d.ins Convergence4P5.control
+        if: '!cancelled()'
+
+#
+# 2) Kovasznay
+# ----------------------------
+
+      - name: Build iNS-Kovasznay
+        working-directory: ./Solver/test/IncompressibleNS/Kovasznay/SETUP
+        run: |
+          source /opt/intel/oneapi/setvars.sh || true
+          make MODE=${{matrix.mode}} COMPILER=${{matrix.compiler}} COMM=${{matrix.comm}} ENABLE_THREADS=${{matrix.enable_threads}} WITH_MKL=${{matrix.mkl}}
+        if: '!cancelled()'
+
+      - name: Run iNS-Kovasznay
+        working-directory: ./Solver/test/IncompressibleNS/Kovasznay
+        run: |
+          source /opt/intel/oneapi/setvars.sh || true
+          ./horses3d.ins Kovasznay.control
+        if: '!cancelled()'
+
+#
+# 3) Lid driven cavity
+# ----------------------------
+
+      - name: Build iNS-LidDrivenCavity
+        working-directory: ./Solver/test/IncompressibleNS/LidDrivenCavity/SETUP
+        run: |
+          source /opt/intel/oneapi/setvars.sh || true
+          make MODE=${{matrix.mode}} COMPILER=${{matrix.compiler}} COMM=${{matrix.comm}} ENABLE_THREADS=${{matrix.enable_threads}} WITH_MKL=${{matrix.mkl}}
+        if: '!cancelled()'
+
+      - name: Run iNS-LidDrivenCavity
+        working-directory: ./Solver/test/IncompressibleNS/LidDrivenCavity
+        run: |
+          source /opt/intel/oneapi/setvars.sh || true
+          ./horses3d.ins LDCRe7500.control
+        if: '!cancelled()'
+
+#
+# 4) Taylor Green Vortex
+# ----------------------------
+
+      - name: Build iNS-TaylorGreenVortex
+        working-directory: ./Solver/test/IncompressibleNS/TaylorGreen/SETUP
+        run: |
+          source /opt/intel/oneapi/setvars.sh || true
+          make MODE=${{matrix.mode}} COMPILER=${{matrix.compiler}} COMM=${{matrix.comm}} ENABLE_THREADS=${{matrix.enable_threads}} WITH_MKL=${{matrix.mkl}}
+        if: '!cancelled()'
+
+      - name: Run iNS-TaylorGreenVortex
+        working-directory: ./Solver/test/IncompressibleNS/TaylorGreen
+        run: |
+          source /opt/intel/oneapi/setvars.sh || true
+          ./horses3d.ins TGV_Central.control
+        if: '!cancelled()'
+
+#
+# 5) Rayleigh-Taylor instability
+# -------------------------------
+
+      - name: Build iNS-RayleighTaylor
+        working-directory: ./Solver/test/IncompressibleNS/RayleighTaylor/SETUP
+        run: |
+          source /opt/intel/oneapi/setvars.sh || true
+          make MODE=${{matrix.mode}} COMPILER=${{matrix.compiler}} COMM=${{matrix.comm}} ENABLE_THREADS=${{matrix.enable_threads}} WITH_MKL=${{matrix.mkl}}
+        if: '!cancelled()'
+
+      - name: Run iNS-RayleighTaylor
+        working-directory: ./Solver/test/IncompressibleNS/RayleighTaylor
+        run: |
+          source /opt/intel/oneapi/setvars.sh || true
+          ./horses3d.ins RTI.control
+        if: '!cancelled()'
+
+
+
+########################################################################
+#######                        ADDONS                           ########
+########################################################################
+#
+# 1) Mapping Result - horsesConverter
+# --------------------------------------------
+      - name: Run horsesConverter_MappingResult
+        working-directory: ./Solver/test/AddOns/horsesConverter/MappingResult
+        run: |
+          source /opt/intel/oneapi/setvars.sh || true
+          ./horsesConverter
+        if: '!cancelled()'
+
+#
+# 2) Horses Mesh to OpenFOAM - horsesConverter
+# --------------------------------------------
+      - name: Run horsesConverter_HorsesMesh2OF
+        working-directory: ./Solver/test/AddOns/horsesConverter/HorsesMesh2OF
+        run: |
+          source /opt/intel/oneapi/setvars.sh || true
+          ./horsesConverter
+        if: '!cancelled()'
+
+#
+# 3) OpenFOAM to Horses - horsesConverter
+# --------------------------------------------
+      - name: Run horsesConverter_OF2Horses
+        working-directory: ./Solver/test/AddOns/horsesConverter/OF2Horses
+        run: |
+          source /opt/intel/oneapi/setvars.sh || true
+          OMP_NUM_THREADS=4 ./horsesConverter
+        if: '!cancelled()'

Solver/src/MultiphaseSolver/SpatialDiscretization.f90

@@ -462,7 +462,7 @@ SUBROUTINE ComputeTimeDerivative( mesh, particles, time, mode)
 !
 !$omp do schedule(runtime) private(i,j,k,e,sqrtRho)
          do eID = 1, size(mesh % elements)
-            e => mesh % elements(eID)
+            associate(e => mesh % elements(eID))
             do k = 0, e % Nxyz(3) ; do j = 0, e % Nxyz(2) ; do i = 0, e % Nxyz(1)
                sqrtRho = sqrt(e % storage % rho(i,j,k))
                e % storage % QDot(IMC,i,j,k)      = 0.0_RP
@@ -486,7 +486,7 @@ SUBROUTINE ComputeTimeDerivative( mesh, particles, time, mode)
 
                e % storage % QDot(:,i,j,k) = e % storage % QDot(:,i,j,k) * e % geom % jacobian(i,j,k)
             end do                ; end do                ; end do
-
+            end associate
          end do
 !$omp end do
 

Solver/src/libs/mesh/HexMesh.f90

@@ -3825,6 +3825,14 @@ subroutine HexMesh_AllocateStorage(self,NDOF,controlVariables,computeGradients,F
          RKSteps_num = 0
       end if
 
+#ifdef MULTIPHASE
+      ! This is a fix to prevent a seg fault in debug mode
+      ! implemented by g.rubio@upm.es 09/2023
+      if ( trim(time_int) == "explicit" ) then
+         bdf_order = 1  
+         RKSteps_num = 0   
+      endif  
+#endif 
 !     Construct global and elements' storage
 !     --------------------------------------
       call self % storage % construct (NDOF, self % Nx, self % Ny, self % Nz, computeGradients, .FALSE., bdf_order, RKSteps_num )

Solver/test/Multiphase/Pipe/SETUP/ProblemFile.f90

@@ -89,16 +89,18 @@ subroutine UserDefinedState_f(x, t, nHat, Q, thermodynamics_, dimensionless_, re
          type(RefValues_t),      intent(in)  :: refValues_
       end subroutine UserDefinedState_f
 
-      subroutine UserDefinedGradVars_f(x, t, nHat, Q, U, thermodynamics_, dimensionless_, refValues_)
+      subroutine UserDefinedGradVars_f(x, t, nHat, Q, U, GetGradients, thermodynamics_, dimensionless_, refValues_)
          use SMConstants
          use PhysicsStorage
          use FluidData
+         use VariableConversion, only: GetGradientValues_f
          implicit none
          real(kind=RP), intent(in)          :: x(NDIM)
          real(kind=RP), intent(in)          :: t
          real(kind=RP), intent(in)          :: nHat(NDIM)
          real(kind=RP), intent(in)          :: Q(NCONS)
          real(kind=RP), intent(inout)       :: U(NGRAD)
+         procedure(GetGradientValues_f)     :: GetGradients
          type(Thermodynamics_t), intent(in) :: thermodynamics_
          type(Dimensionless_t),  intent(in) :: dimensionless_
          type(RefValues_t),      intent(in) :: refValues_