diff --git a/.gitattributes b/.gitattributes deleted file mode 100644 index 910b5161..00000000 --- a/.gitattributes +++ /dev/null @@ -1,11 +0,0 @@ -test/HD/SedovBlastWave/python/1Dsolution/data.0001.vtk filter=lfs diff=lfs merge=lfs -text -test/HD/ViscousFlowPastCylinder/python/data.0001.ref.vtk filter=lfs diff=lfs merge=lfs -text -test/MHD/AxisFluxTube/python/data.0001.ref.vtk filter=lfs diff=lfs merge=lfs -text -test/MHD/sod-iso/python/data.ref.vtk filter=lfs diff=lfs merge=lfs -text -test/MHD/OrszagTang/python/data.0001.ref.vtk filter=lfs diff=lfs merge=lfs -text -test/HD/FargoPlanet/python/data.0001.ref.vtk filter=lfs diff=lfs merge=lfs -text -test/MHD/AxisFluxTube/python/data.0001.ref-coarsening.vtk filter=lfs diff=lfs merge=lfs -text -test/MHD/OrszagTang3D/python/data.0001.ref.vtk filter=lfs diff=lfs merge=lfs -text -test/MHD/sod/python/data.ref.vtk filter=lfs diff=lfs merge=lfs -text -test/MHD/FargoMHDSpherical/python/data.0001.ref.vtk filter=lfs diff=lfs merge=lfs -text -test/HD/MachReflection/python/data.0001.ref.vtk filter=lfs diff=lfs merge=lfs -text diff --git a/.github/workflows/idefix-ci.yml b/.github/workflows/idefix-ci.yml index 250aa4af..61e7d293 100644 --- a/.github/workflows/idefix-ci.yml +++ b/.github/workflows/idefix-ci.yml @@ -4,6 +4,7 @@ on: push: branches: - master + - v2.0 pull_request: paths-ignore: - '.github/ISSUE_TEMPLATE/*' @@ -16,17 +17,17 @@ concurrency: cancel-in-progress: true env: - TEST_OPTIONS: -DKokkos_ENABLE_CUDA=ON + TESTME_OPTIONS: -cuda -Werror + PYTHONPATH: ${{ github.workspace }} + IDEFIX_DIR: ${{ github.workspace }} jobs: Linter: # Don't do this in forks - if: github.repository == 'idefix-code/idefix' + if: ${{ github.repository == 'idefix-code/idefix' || github.repository == 'glesur/idefix' }} runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - with: - lfs: false - uses: actions/setup-python@v4 with: python-version: 3.x @@ -34,114 +35,253 @@ jobs: - uses: pre-commit-ci/lite-action@v1.0.0 if: always() - Hydrodynamics: + ShocksHydro: + needs: Linter + runs-on: self-hosted + steps: + - name: Check out repo + uses: actions/checkout@v3 + with: + submodules: recursive + - name: Sod test + run: | + cd $IDEFIX_DIR/test/HD/sod + ./testme.py -all $TESTME_OPTIONS + - name: Isothermal Sod test + run: | + cd $IDEFIX_DIR/test/HD/sod-iso + ./testme.py -all $TESTME_OPTIONS + - name: Mach reflection test + run: | + cd $IDEFIX_DIR/test/HD//MachReflection + ./testme.py -all $TESTME_OPTIONS + + ParabolicHydro: needs: Linter runs-on: self-hosted steps: - name: Check out repo uses: actions/checkout@v3 with: - lfs: true submodules: recursive - # Manually do a git LFS https://github.com/actions/checkout/issues/270 - - run: git lfs pull - - name: Run Hydro test - run: cd test && ./checks_hydro.sh $TEST_OPTIONS + - name: Viscous flow past cylinder + run: | + cd $IDEFIX_DIR/test/HD/ViscousFlowPastCylinder + ./testme.py -all $TESTME_OPTIONS + - name: Viscous disk + run: | + cd $IDEFIX_DIR/test/HD/ViscousDisk + ./testme.py -all $TESTME_OPTIONS + - name: Thermal diffusion + run: | + cd $IDEFIX_DIR/test/HD/thermalDiffusion + ./testme.py -all $TESTME_OPTIONS - Magneto-Hydrodynamics: + ShocksMHD: needs: Linter runs-on: self-hosted steps: - name: Check out repo uses: actions/checkout@v3 with: - lfs: true submodules: recursive - # Manually do a git LFS https://github.com/actions/checkout/issues/270 - - run: git lfs pull - - name: Run MHD test - run: cd test && ./checks_mhd.sh $TEST_OPTIONS + - name: MHD Sod test + run: | + cd $IDEFIX_DIR/test/MHD/sod + ./testme.py -all $TESTME_OPTIONS + - name: MHD Isothermal Sod test + run: | + cd $IDEFIX_DIR/test/MHD/sod-iso + ./testme.py -all $TESTME_OPTIONS + - name: Orszag Tang + run: | + cd $IDEFIX_DIR/test/MHD/OrszagTang + ./testme.py -all $TESTME_OPTIONS + - name: Orszag Tang 3D+restart tests + run: | + cd $IDEFIX_DIR/test/MHD/OrszagTang3D + ./testme.py -all $TESTME_OPTIONS + - MPI: + ParabolicMHD: needs: Linter runs-on: self-hosted steps: - name: Check out repo uses: actions/checkout@v3 with: - lfs: true submodules: recursive - # Manually do a git LFS https://github.com/actions/checkout/issues/270 - - run: git lfs pull - - name: Run MPI test - run: cd test && ./checks_mpi.sh $TEST_OPTIONS + - name: Ambipolar C Shock + run: | + cd $IDEFIX_DIR/test/MHD/AmbipolarCshock + ./testme.py -all $TESTME_OPTIONS + - name: Ambipolar C Shock 3D + run: | + cd $IDEFIX_DIR/test/MHD/AmbipolarCshock3D + ./testme.py -all $TESTME_OPTIONS + - name: Resistive Alfvén wave + run: | + cd $IDEFIX_DIR/test/MHD/ResistiveAlfvenWave + ./testme.py -all $TESTME_OPTIONS + - name: Hall whistler waves + run: | + cd $IDEFIX_DIR/test/MHD/HallWhistler + ./testme.py -all $TESTME_OPTIONS + + Fargo: + needs: [Linter, ShocksHydro, ParabolicHydro, ShocksMHD, ParabolicMHD] + runs-on: self-hosted + steps: + - name: Check out repo + uses: actions/checkout@v3 + with: + submodules: recursive + - name: Fargo + planet + run: | + cd $IDEFIX_DIR/test/HD/FargoPlanet + ./testme.py -all $TESTME_OPTIONS + - name: Fargo MHD spherical + run: | + cd $IDEFIX_DIR/test/MHD/FargoMHDSpherical + ./testme.py -all $TESTME_OPTIONS + + ShearingBox: + needs: [Linter, ShocksHydro, ParabolicHydro, ShocksMHD, ParabolicMHD] + runs-on: self-hosted + steps: + - name: Check out repo + uses: actions/checkout@v3 + with: + submodules: recursive + - name: Hydro shearing box + run: | + cd $IDEFIX_DIR/test/HD/ShearingBox + ./testme.py -all $TESTME_OPTIONS + - name: MHD shearing box + run: | + cd $IDEFIX_DIR/test/MHD/ShearingBox + ./testme.py -all $TESTME_OPTIONS + + SelfGravity: + needs: [Linter, ShocksHydro, ParabolicHydro, ShocksMHD, ParabolicMHD] + runs-on: self-hosted + steps: + - name: Check out repo + uses: actions/checkout@v3 + with: + submodules: recursive + - name: Jeans Instability + run: | + cd $IDEFIX_DIR/test/SelfGravity/JeansInstability + ./testme.py -all $TESTME_OPTIONS + - name: Random sphere spherical + run: | + cd $IDEFIX_DIR/test/SelfGravity/RandomSphere + ./testme.py -all $TESTME_OPTIONS + - name: Random sphere cartesian + run: | + cd $IDEFIX_DIR/test/SelfGravity/RandomSphereCartesian + ./testme.py -all $TESTME_OPTIONS + - name: Uniform spherical collapse + run: | + cd $IDEFIX_DIR/test/SelfGravity/UniformCollapse + ./testme.py -all $TESTME_OPTIONS - VectorPotential: - needs: [Linter, Hydrodynamics, Magneto-Hydrodynamics, MPI] + Planet: + needs: [Linter, ShocksHydro, ParabolicHydro, ShocksMHD, ParabolicMHD] runs-on: self-hosted steps: - name: Check out repo uses: actions/checkout@v3 with: - lfs: true submodules: recursive - # Manually do a git LFS https://github.com/actions/checkout/issues/270 - - run: git lfs pull - - name: Run Vector Potential test - run: cd test && ./checks_vector_potential.sh $TEST_OPTIONS + - name: 3 body + run: | + cd $IDEFIX_DIR/test/Planet/Planet3Body + ./testme.py -all $TESTME_OPTIONS + - name: migration + run: | + cd $IDEFIX_DIR/test/Planet/PlanetMigration2D + ./testme.py -all $TESTME_OPTIONS + - name: planet-planet + run: | + cd $IDEFIX_DIR/test/Planet/PlanetPlanetRK42D + ./testme.py -all $TESTME_OPTIONS + - name: spiral wake + run: | + cd $IDEFIX_DIR/test/Planet/PlanetSpiral2D + ./testme.py -all $TESTME_OPTIONS + - name: torques + run: | + cd $IDEFIX_DIR/test/Planet/PlanetTorque3D + ./testme.py -all $TESTME_OPTIONS + - name: RK5 + run: | + cd $IDEFIX_DIR/test/Planet/PlanetsIsActiveRK52D + ./testme.py -all $TESTME_OPTIONS - HighOrder: - needs: [Linter, Hydrodynamics, Magneto-Hydrodynamics, MPI] + Dust: + needs: [Linter, ShocksHydro, ParabolicHydro, ShocksMHD, ParabolicMHD] runs-on: self-hosted steps: - name: Check out repo uses: actions/checkout@v3 with: - lfs: true submodules: recursive - # Manually do a git LFS https://github.com/actions/checkout/issues/270 - - run: git lfs pull - - name: Run high order test - run: cd test && ./checks_highorder.sh $TEST_OPTIONS + - name: Energy conservation + run: | + cd $IDEFIX_DIR/test/Dust/DustEnergy + ./testme.py -all $TESTME_OPTIONS + - name: Dusty wave + run: | + cd $IDEFIX_DIR/test/Dust/DustyWave + ./testme.py -all $TESTME_OPTIONS - SinglePrecision: - needs: [Linter, Hydrodynamics, Magneto-Hydrodynamics, MPI] + Braginskii: + needs: [Linter, ShocksHydro, ParabolicHydro, ShocksMHD, ParabolicMHD] runs-on: self-hosted steps: - name: Check out repo uses: actions/checkout@v3 with: - lfs: true submodules: recursive - # Manually do a git LFS https://github.com/actions/checkout/issues/270 - - run: git lfs pull - - name: Run single precision test - run: cd test && ./checks_singleprecision.sh $TEST_OPTIONS + - name: MTI + run: | + cd $IDEFIX_DIR/test/MHD/MTI + ./testme.py -all $TESTME_OPTIONS + - name: Spherical anisotropic diffusion + run: | + cd $IDEFIX_DIR/test/MHD/sphBragTDiffusion + ./testme.py -all $TESTME_OPTIONS + - name: Spherical anisotropic viscosity + run: | + cd $IDEFIX_DIR/test/MHD/sphBragViscosity + ./testme.py -all $TESTME_OPTIONS Examples: - needs: [VectorPotential, HighOrder,SinglePrecision] + needs: [Fargo, Dust, Planet, ShearingBox, SelfGravity] runs-on: self-hosted steps: - name: Check out repo uses: actions/checkout@v3 with: - lfs: true submodules: recursive - # Manually do a git LFS https://github.com/actions/checkout/issues/270 - - run: git lfs pull - name: Run examples test run: cd test && ./checks_examples.sh $TEST_OPTIONS Utils: - needs: [VectorPotential, HighOrder,SinglePrecision] + needs: [Fargo, Dust, Planet, ShearingBox, SelfGravity] runs-on: self-hosted steps: - name: Check out repo uses: actions/checkout@v3 with: - lfs: true submodules: recursive - # Manually do a git LFS https://github.com/actions/checkout/issues/270 - - run: git lfs pull - - name: Run utils test - run: cd test && ./checks_utils.sh $TEST_OPTIONS + - name: Lookup table + run: | + cd $IDEFIX_DIR/test/utils/lookupTable + ./testme.py -all $TESTME_OPTIONS + - name: Dump Image + run: | + cd $IDEFIX_DIR/test/utils/dumpImage + ./testme.py -all $TESTME_OPTIONS diff --git a/.gitignore b/.gitignore index 213b33e6..422647c3 100644 --- a/.gitignore +++ b/.gitignore @@ -12,7 +12,7 @@ doc/source/xml/* # generated files *.log -src/gitversion.hpp +src/version.hpp **/libkokkos.a **/KokkosCore_config.h build @@ -34,6 +34,7 @@ test/**/Makefile test/**/KokkosCore* test/**/*.csv test/**/*.pyc +test/**/*.dat # machine specific cache and hidden files .* diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0ebafac3..c184917d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -20,52 +20,127 @@ lint: - pre-commit install - pre-commit run --all-files -HD tests: +Shocks Hydro: stage: base_checks image: lesurg/idefix:latest script: - - cd test - - ./checks_hydro.sh $TEST_OPTIONS + - export IDEFIX_DIR=${PWD} + - cd $IDEFIX_DIR/test/HD/sod + - ./testme.py -all $TESTME_OPTIONS + - cd $IDEFIX_DIR/test/HD/sod-iso + - ./testme.py -all $TESTME_OPTIONS + - cd $IDEFIX_DIR/test/HD/MachReflection + - ./testme.py -all $TESTME_OPTIONS -MHD tests: +Parabolic Hydro: stage: base_checks image: lesurg/idefix:latest script: - - cd test - - ./checks_mhd.sh $TEST_OPTIONS + - export IDEFIX_DIR=${PWD} + - cd $IDEFIX_DIR/test/HD/ViscousFlowPastCylinder + - ./testme.py -all $TESTME_OPTIONS + - cd $IDEFIX_DIR/test/HD/ViscousDisk + - ./testme.py -all $TESTME_OPTIONS + - cd $IDEFIX_DIR/test/HD/thermalDiffusion + - ./testme.py -all $TESTME_OPTIONS -MPI tests: +Shocks MHD: stage: base_checks image: lesurg/idefix:latest script: - - cd test - - ./checks_mpi.sh $TEST_OPTIONS + - export IDEFIX_DIR=${PWD} + - cd $IDEFIX_DIR/test/MHD/sod + - ./testme.py -all $TESTME_OPTIONS + - cd $IDEFIX_DIR/test/MHD/sod-iso + - ./testme.py -all $TESTME_OPTIONS + - cd $IDEFIX_DIR/test/MHD/OrszagTang + - ./testme.py -all $TESTME_OPTIONS + - cd $IDEFIX_DIR/test/MHD/OrszagTang3D + - ./testme.py -all $TESTME_OPTIONS -Vector potential tests: - stage: base_checks +Parabolic MHD: + stage: base_checks + image: lesurg/idefix:latest + script: + - export IDEFIX_DIR=${PWD} + - cd $IDEFIX_DIR/test/MHD/AmbipolarCshock + - ./testme.py -all $TESTME_OPTIONS + - cd $IDEFIX_DIR/test/MHD/AmbipolarCshock3D + - ./testme.py -all $TESTME_OPTIONS + - cd $IDEFIX_DIR/test/MHD/ResistiveAlfvenWave + - ./testme.py -all $TESTME_OPTIONS + - cd $IDEFIX_DIR/test/MHD/HallWhistler + - ./testme.py -all $TESTME_OPTIONS + +Fargo: + stage: advanced_checks image: lesurg/idefix:latest script: - - cd test - - ./checks_vector_potential.sh $TEST_OPTIONS + - export IDEFIX_DIR=${PWD} + - cd $IDEFIX_DIR/test/HD/FargoPlanet + - ./testme.py -all $TESTME_OPTIONS + - cd $IDEFIX_DIR/test/MHD/FargoMHDSpherical + - ./testme.py -all $TESTME_OPTIONS -HighOrder tests: +ShearingBox: stage: advanced_checks image: lesurg/idefix:latest script: - - cd test - - ./checks_highorder.sh $TEST_OPTIONS + - export IDEFIX_DIR=${PWD} + - cd $IDEFIX_DIR/test/HD/ShearingBox + - ./testme.py -all $TESTME_OPTIONS + - cd $IDEFIX_DIR/test/HD/ShearingBox + - ./testme.py -all $TESTME_OPTIONS -SinglePrecision tests: +SelfGravity: stage: advanced_checks image: lesurg/idefix:latest script: - - cd test - - ./checks_singleprecision.sh $TEST_OPTIONS + - export IDEFIX_DIR=${PWD} + - cd $IDEFIX_DIR/test/SelfGravity/JeansInstability + - ./testme.py -all $TESTME_OPTIONS + - cd $IDEFIX_DIR/test/SelfGravity/RandomSphere + - ./testme.py -all $TESTME_OPTIONS + - cd $IDEFIX_DIR/test/SelfGravity/RandomSphereCartesian + - ./testme.py -all $TESTME_OPTIONS + - cd $IDEFIX_DIR/test/SelfGravity/UniformCollapse + - ./testme.py -all $TESTME_OPTIONS + + +Planet: + stage: advanced_checks + image: lesurg/idefix:latest + script: + - export IDEFIX_DIR=${PWD} + - cd $IDEFIX_DIR/test/Planet/Planet3Body + - ./testme.py -all $TESTME_OPTIONS + - cd $IDEFIX_DIR/test/Planet/PlanetMigration2D + - ./testme.py -all $TESTME_OPTIONS + - cd $IDEFIX_DIR/test/Planet/PlanetPlanetRK42D + - ./testme.py -all $TESTME_OPTIONS + - cd $IDEFIX_DIR/test/Planet/PlanetSpiral2D + - ./testme.py -all $TESTME_OPTIONS + - cd $IDEFIX_DIR/test/Planet/PlanetTorque3D + - ./testme.py -all $TESTME_OPTIONS + - cd $IDEFIX_DIR/test/Planet/PlanetsIsActiveRK52D + - ./testme.py -all $TESTME_OPTIONS + + +Dust: + stage: advanced_checks + image: lesurg/idefix:latest + script: + - export IDEFIX_DIR=${PWD} + - cd $IDEFIX_DIR/test/Dust/DustEnergy + - ./testme.py -all $TESTME_OPTIONS + - cd $IDEFIX_DIR/test/Dust/DustyWave + - ./testme.py -all $TESTME_OPTIONS Examples tests: stage: advanced_checks image: lesurg/idefix:latest script: + - export IDEFIX_DIR=${PWD} - cd test - ./checks_examples.sh $TEST_OPTIONS @@ -73,8 +148,11 @@ Utils tests: stage: advanced_checks image: lesurg/idefix:latest script: - - cd test - - ./checks_utils.sh $TEST_OPTIONS + - export IDEFIX_DIR=${PWD} + - cd $IDEFIX_DIR/test/utils/lookupTable + - ./testme.py -all $TESTME_OPTIONS + - cd $IDEFIX_DIR/test/utils/dumpImage + - ./testme.py -all $TESTME_OPTIONS pages: image: lesurg/idefix-documentation:latest diff --git a/.gitmodules b/.gitmodules index 6c4fddec..83fb2456 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,6 @@ [submodule "src/kokkos"] path = src/kokkos url = https://github.com/kokkos/kokkos.git +[submodule "reference"] + path = reference + url = https://github.com/idefix-code/reference diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 11b0176a..06fe7210 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -12,6 +12,8 @@ repos: - id: check-yaml - id: check-executables-have-shebangs - id: check-shebang-scripts-are-executable + - id: check-added-large-files + args: ['--maxkb=100'] ## prevent files larger than 100kB from being commited (exclude git lfs files) - repo: https://github.com/Lucas-C/pre-commit-hooks-nodejs rev: v1.1.2 diff --git a/CHANGELOG.md b/CHANGELOG.md index a4c6ec5e..2ce42182 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,32 +4,60 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## Upcoming +## [2.0.0] 2023-10-23 ### Changed +- Reorganisation of class instances embedded in datablocks to allow for multi-fluid problems using a generalised template class "Fluid" that replaces "Hydro". Most instances (like data.hydro.Vc) have been replaced by pointers (e.g data.hydro->Vc) (!307). - RKL scheme now correctly takes into account grid coarsening when estimating the timestep of parabolic terms (!254) - fixed a bug in the evaluation of gravitational forces from the gradient of the potential in regions where grid coarsening is enabled (!267) +- The CI tests now include a "non-regression" test that validate the code outputs *at machine precision*. This comes in addition to the "standard" test that validate the code against known analytical solution (at truncation precision). Each test now contains a testme.py script that does the full validation (documentation will come, for now use testme -help). (!311) +- use PPM reconstruction for 2D Riemann solvers in Emf Computation when PPM is required. +- the MPI routines have been refactored to guarantee reproductibility at machine precision independently of the domain decomposition. This implies that normal B field component are now exchanged between neighbouring processes. (!308) +- allow the user to specify a particular output directory for vtk and dmp files (!339) +- the -restart option without any number now loads the latest generated dump file, and not the last dump in the directory (!353) - fixed a bug that resulted in erroneous momentum and energy fluxes when using the combination of Fargo and Viscosity in non-cartesian geometries. (!267) - fixed a bug in shock flattening that resulted in loss of conservative properties when using periodic boundaries and/or MPI domain decomposition (!275) +- fixed a bug due to a missing curvature term in the viscosity stress tensor in spherical geometry (!343) +- fixed a bug that led to an incorrect heating rate when both fargo and viscosity were enabled (!333) - fixed a bug in emergency vtk outputs that could lead to an MPI deadlock when the user did not enable VTK outputs (!274) - fixed a bug that could result in MPI deadlocks when an exception is thrown by a single MPI process in the integration loop (!266) - fixed a bug in shock flattening that could lead to the breakup of conservation properties (!275) - fixed a bug in LookupTable that could lead to incorrect interpolation (!286) - fixed a bug that prevented to compile on HIP backend (!291) - - +- fixed a bug that prevented idefix with vector potential to restart from dumps created with vector potential (!306) +- fixed a bug that led to race condition when using GPU offloading, axis boundary condition and domain decomposition along X3 (!309) +- fixed a bug that led to inconsistent results with MPI and UCT_HLLx EMF reconstruction schemes (!310) +- fixed a bug that could result in .dmp file duplication on restart (!354) +- Moving development onto github.com. Some references to merge requests will likely be lost. +- fixed a bug in v2.0 developement branch that resulted in broekn dump files when MHD was enabled with DIMENSIONS<3 (#174) +- IDEFIX_Debug now automatically introduce Kokkos::fence at the end of each idefix_for, enforcing synchronisation between host and device when debugging (#188) +- Do not use git lfs anymore due to bandwidth restrictions imposed by github (#183) +- use proper cell centroid PLM reconstruction when using non-cartesian coordinates (#196) +- fixed a bug that could lead to low (O(1e-10)) leakage of conserved quantities along the axis (#196) +- changed radial pressure curvature term so that machine precision balance is achieved for constant pressure flows in every geometry (#196) ### Added - Self-gravity (!186) +- Multi-dust species as pressureless fluids (!336) +- Passive tracers (!341) +- Planet module (planet migration, planet-planet integration) (!278) +- Custom equation of states (#185) +- Sliced VTK files (beta version) (#195) - Check that the MPI library is GPU-aware when using a GPU backend (!262) - An optional user-defined Setup destructor can now be defined (!260) - performance improvement on CPUs by cleaning loops and rewriting EMF reconstruction (!281) - The tolerance on div(B) allowed by the code can now be set at runtime (!292) - Nan detection is now performed every 100 integration loops by default so as mitigate performance impact on CPUs (!292) +- It is now possible to build a stretch grid from a logarithmic grid section, and not only a uniform grid section (!304) +- Shock flattening can now be used in combination with LimO3 slope limiter (!312) +- XDMF output format (optional, requires HDF5 library on the host machine) (#13) +- new -profile option to perform on-the-fly profiling without requiring Kokkos Tools (#188) +- -v and -h options to show version and list of accepted arguments ### Removed - auto-tuning was removed as it was preventing auto-vectorisation on Intel compilers. Loop tuning are now set at compile time (!281) + ## [1.1.0] 2022-09-07 ### Changed - use buffers for mpi axis exchanges to improve performances on GPUs (!195) diff --git a/CMakeLists.txt b/CMakeLists.txt index f5b0fb54..7777bbde 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2,14 +2,23 @@ cmake_minimum_required(VERSION 3.16) set(CMAKE_BUILD_TYPE Release) set (CMAKE_CXX_STANDARD 17) +set(Idefix_VERSION_MAJOR 2) +set(Idefix_VERSION_MINOR 0) +set(Idefix_VERSION_PATCH 00) + project (idefix VERSION 1.0.0) option(Idefix_MHD "enable MHD" OFF) option(Idefix_MPI "enable Message Passing Interface parallelisation" OFF) option(Idefix_HIGH_ORDER_FARGO "Force Fargo to use a PPM reconstruction scheme" OFF) option(Idefix_DEBUG "Enable Idefix debug features (makes the code very slow)" OFF) +option(Idefix_RUNTIME_CHECKS "Enable runtime sanity checks" OFF) option(Idefix_WERROR "Treat compiler warnings as errors" OFF) set(Idefix_CXX_FLAGS "" CACHE STRING "Additional compiler/linker flag") set(Idefix_DEFS "definitions.hpp" CACHE FILEPATH "Problem definition header file") +option(Idefix_CUSTOM_EOS "Use custom equation of state" OFF) +if(Idefix_CUSTOM_EOS) + set(Idefix_CUSTOM_EOS_FILE "eos_custom.hpp" CACHE FILEPATH "Custom equation of state source file") +endif() set(Idefix_RECONSTRUCTION "Linear" CACHE STRING "Type of cell reconstruction scheme") option(Idefix_HDF5 "Enable HDF5 I/O (requires HDF5 library)" OFF) if(Idefix_MHD) @@ -70,7 +79,6 @@ if(EXISTS ${PROJECT_BINARY_DIR}/CMakeLists.txt) endif() - if(Idefix_MHD) add_compile_definitions("MHD=YES") else() @@ -109,6 +117,10 @@ if(Idefix_DEBUG) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O0 -g") endif() +if(Idefix_RUNTIME_CHECKS) + add_compile_definitions("RUNTIME_CHECKS") +endif() + if(Idefix_HIGH_ORDER_FARGO) add_compile_definitions("HIGH_ORDER_FARGO") endif() @@ -116,14 +128,24 @@ endif() if(Idefix_EVOLVE_VECTOR_POTENTIAL) add_compile_definitions("EVOLVE_VECTOR_POTENTIAL") endif() -#update gitversion.hpp if possible +#update version.hpp if possible git_describe(GIT_SHA1) -write_file(${CMAKE_SOURCE_DIR}/src/gitversion.hpp "#define GITVERSION \"${GIT_SHA1}\"") +set(Idefix_VERSION ${Idefix_VERSION_MAJOR}.${Idefix_VERSION_MINOR}.${Idefix_VERSION_PATCH}-${GIT_SHA1}) +file(WRITE ${CMAKE_SOURCE_DIR}/src/version.hpp "#define IDEFIX_GIT_COMMIT \"${GIT_SHA1}\"\n") +file(APPEND ${CMAKE_SOURCE_DIR}/src/version.hpp "#define IDEFIX_VERSION_MAJOR \"${Idefix_VERSION_MAJOR}\"\n") +file(APPEND ${CMAKE_SOURCE_DIR}/src/version.hpp "#define IDEFIX_VERSION_MINOR \"${Idefix_VERSION_MINOR}\"\n") +file(APPEND ${CMAKE_SOURCE_DIR}/src/version.hpp "#define IDEFIX_VERSION_PATCH \"${Idefix_VERSION_PATCH}\"\n") +file(APPEND ${CMAKE_SOURCE_DIR}/src/version.hpp "#define IDEFIX_VERSION \"${Idefix_VERSION}\"\n") + if(NOT ${Idefix_DEFS} STREQUAL "definitions.hpp") add_compile_definitions("DEFINITIONS_FILE=\"${Idefix_DEFS}\"") endif() +if(Idefix_CUSTOM_EOS) + add_compile_definitions("EOS_FILE=\"${Idefix_CUSTOM_EOS_FILE}\"") +endif() + # Order of the scheme if(${Idefix_RECONSTRUCTION} STREQUAL "Constant") add_compile_definitions("ORDER=1") @@ -172,15 +194,22 @@ target_include_directories(idefix PUBLIC target_include_directories(idefix PUBLIC src/kokkos/core/src src/dataBlock - src/hydro - src/hydro/boundary - src/hydro/electromotiveforce - src/hydro/HDsolvers - src/hydro/MHDsolvers + src/dataBlock/planetarySystem + src/fluid + src/fluid/boundary + src/fluid/braginskii + src/fluid/constrainedTransport + src/fluid/eos + src/fluid/RiemannSolver + src/fluid/RiemannSolver/HDsolvers + src/fluid/RiemannSolver/MHDsolvers + src/fluid/RiemannSolver/Dustsolvers + src/fluid/tracer src/output src/rkl src/gravity src/utils + src/utils/iterativesolver src ) @@ -196,5 +225,8 @@ message(STATUS " MPI: ${Idefix_MPI}") message(STATUS " HDF5: ${Idefix_HDF5}") message(STATUS " Reconstruction: ${Idefix_RECONSTRUCTION}") message(STATUS " Precision: ${Idefix_PRECISION}") -message(STATUS " Version: ${GIT_SHA1}") +message(STATUS " Version: ${Idefix_VERSION}") message(STATUS " Problem definitions: '${Idefix_DEFS}'") +if(Idefix_CUSTOM_EOS) + message(STATUS " EOS: Custom file '${Idefix_CUSTOM_EOS_FILE}'") +endif() diff --git a/README.md b/README.md index 7853cd00..a005f297 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ * [With MPI (gpu)](#with-mpi-gpu) - [Profiling](#profiling) - [Debugging](#debugging) -- [Running tests](#running-tests) +- [Code Validation](#code-validation) - [Contributing](#contributing) @@ -20,14 +20,16 @@ Download: --------- -using either ssh or https url as `
` +Assuming you want to use https to get idefix (easiest option): + ```shell -git clone
idefix +git clone https://github.com/idefix-code/idefix.git idefix cd idefix git submodule init git submodule update ``` + Installation: ------------- @@ -100,31 +102,39 @@ mpirun -np 8 ./idefix -dec 1 2 4 --kokkos-num-devices=4 Profiling ------------------- -use the kokkos profiler tool, which can be downloaded from kokkos-tools (on github) -Then set the environement variable: +use the embedded profiling tool by adding "-profile" when calling idefix (no need to recompile) ```shell -export KOKKOS_PROFILE_LIBRARY=kokkos-tools/src/tools/space-time-stack/kp_space_time_stack.so -```` - -and then simply run the code (no need to recompile) +./idefix -profile +``` Debugging ------------------- Add `-DIdefix_DEBUG=ON` when calling cmake, or activate the `Idefix_DEBUG` option in ccmake, and recompile. Note that this option triggers a lot of outputs and memory access checks which significantly slow down the code. -Running tests -------------------- -Tests require Python 3 along with some third party dependencies to be installed. -To install those deps, run +Code Validation +--------------- + +Most of tests provided in the `test/` directory can be validated against analytical solution (standard test) +and/or pre-computed solutions (non-regression tests). Note that the validation relies on large reference +files that are stored in the separate `idefix-code/reference` repository that is cloned as a submodule. + +Ensure that reference files +were properly downloaded (in the reference/ directory of the root of idefix) before attempting to validate the code. + +In order to do a full validation of a particular test +(with all of the possible combination of algorithms), use the script `testme.py` +with the `-all` option, as in e.g.: ```shell -pip install -r test/python_requirements.txt +cd $IDEFIX_DIR/test/HD/sod +./testme.py -all ``` -The test suite itself is then run with +Tests require Python 3 along with some third party dependencies to be installed. +To install those deps, run ```shell -bash test/checks.sh +pip install -r test/python_requirements.txt ``` Contributing diff --git a/cmake/GetGitRevisionDescription.cmake b/cmake/GetGitRevisionDescription.cmake index 31ea28e3..81b5844b 100644 --- a/cmake/GetGitRevisionDescription.cmake +++ b/cmake/GetGitRevisionDescription.cmake @@ -201,7 +201,8 @@ function(git_describe _var) #message(STATUS "Arguments to execute_process: ${ARGN}") execute_process( - COMMAND "${GIT_EXECUTABLE}" describe --tags --always ${hash} ${ARGN} + #COMMAND "${GIT_EXECUTABLE}" describe --tags --always ${hash} ${ARGN} + COMMAND "${GIT_EXECUTABLE}" rev-parse --short HEAD WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" RESULT_VARIABLE res OUTPUT_VARIABLE out @@ -210,6 +211,8 @@ function(git_describe _var) set(out "${out}-${res}-NOTFOUND") endif() + #set(out "${hash}") + set(${_var} "${out}" PARENT_SCOPE) diff --git a/doc/python_requirements.txt b/doc/python_requirements.txt index f26ddd60..b0acf406 100644 --- a/doc/python_requirements.txt +++ b/doc/python_requirements.txt @@ -12,3 +12,4 @@ sphinx_git==11.0.0 breathe==4.34.0 exhale==0.3.6 m2r2==0.3.2 +sphinx-copybutton==0.5.2 diff --git a/doc/source/conf.py b/doc/source/conf.py index 634077a5..362da4da 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -23,7 +23,7 @@ author = 'Geoffroy Lesur' # The full version, including alpha/beta/rc tags -release = '1.0' +release = '2.0.0' @@ -37,7 +37,8 @@ 'sphinx_git', "breathe", "exhale", - "m2r2" + "m2r2", + "sphinx_copybutton" ] source_suffix = [".rst", ".md"] diff --git a/doc/source/faq.rst b/doc/source/faq.rst index dc3d3170..f7d5461a 100644 --- a/doc/source/faq.rst +++ b/doc/source/faq.rst @@ -24,6 +24,9 @@ How do I use my favourite XXXXX compiler? I have a complex setup, and have written some functions in separate .cpp files. How can I add these files to *Idefix* build tree? Add a ``CmakeLists.txt`` in your problem directory and use the function `add_idefix_source` (see :ref:`customSourceFiles`). +I want to run on the GPUs of xxx machine, how do I proceed? + Check the examples in :ref:`setupExamples` + Compilation ----------- @@ -33,6 +36,9 @@ Is there a way to see explicitely the compilation commands with ``make``? The compilation stops while compiling Kokkos with ``/usr/include/stdlib.h(58): error: expected a ";"`` This happens on Gricad machines when LIBDL is activated (wrong glibc). Simply disable LIBDL. +When using the Intel compiler on a Mac Intel, I get a linking error involving the ``SharedAllocationRecordIvvE18t_tracking_enabledE`` symbol. + This is a known bug of the Intel Mac compiler with Kokkos. Apparently Intel has decided not to fix it. Check the issue on the `Kokkos git page `_. + Execution --------- @@ -44,11 +50,12 @@ How can I stop the code without loosing the current calculation? I'm doing performance measures. How do I disable all outputs in *Idefix*? Add ``-nowrite`` when you call *Idefix* executable. + Developement ------------ I have a serious bug (e.g. segmentation fault), in my setup, how do I proceed? - Add ``-DIdefix_DEBUG=ON`` to ``cmake`` and recompile to find out exactly where the code crashes. + Add ``-DIdefix_DEBUG=ON`` to ``cmake`` and recompile to find out exactly where the code crashes (see :ref:`debugging`). I want to test a modification to *Idefix* source code specific to my problem without modifying files in `$IDEFIX_DIR`. How can I proceed? Add a ``CmakeLists.txt`` in your problem directory and use the function `replace_idefix_source` (see :ref:`customSourceFiles`). diff --git a/doc/source/index.rst b/doc/source/index.rst index 18762cd6..c0d96f12 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -20,18 +20,42 @@ algorithm is essentially the same (but with some major modification to its struc ================ Requirements ================ -*Idefix* is written is standard C++17 and does not rely on any external library in serial (non MPI). *Idefix* requires a C++17 compatible compiler. It has been tested successfully with GCC (>8), Intel compiler suite (>2018) and -Clang on both Intel and AMD CPUs. *Idefix* has also been tested on NVIDIA GPUs (Pascal, Volta and Ampere architectures) using the nvcc (>10) compiler, and on AMD GPUs (Radeon Mi50) using the hipcc compiler. -*Idefix* relies internally on the `Kokkos `_ library, which is bundled with *Idefix* as a git submodule and compiled on the fly, hence no external installation is required. -When using MPI parallelisation, *Idefix* relies on an external MPI library. *Idefix* has been tested successfully with OpenMPI and IntelMPI libraries. When used on GPU architectures, *Idefix* assumes that -the MPI library is GPU-Aware. If unsure, check this last point with your system administrator. +*Idefix* is written is standard C++17 and does not rely on any external library in serial (non MPI). + +Compiler + *Idefix* requires a C++17 compatible compiler. It has been tested successfully with GCC (>8), Intel compiler suite (>2018) and + Clang on both Intel and AMD CPUs. *Idefix* has also been tested on NVIDIA GPUs (Pascal, Volta and Ampere architectures) using the nvcc (>10) compiler, and on AMD GPUs (Radeon Mi50, Mi210, Mi250) using the hipcc compiler. + +Kokkos library + *Idefix* relies internally on the `Kokkos `_ library, which is bundled with *Idefix* as a git submodule and compiled on the fly, hence no external installation is required. + +MPI library + When using MPI parallelisation, *Idefix* relies on an external MPI library. *Idefix* has been tested successfully with OpenMPI and IntelMPI libraries. When used on GPU architectures, *Idefix* assumes that + the MPI library is GPU-Aware. If unsure, check this last point with your system administrator. + +================ +Features +================ +* Compressible hydrodynamics in 1D, 2D, 3D +* Compressible magnetohydrodynamics using constrained transport in 1D, 2D, 3D +* Multiple geometry (cartesian, polar, spherical) +* Variable mesh spacing +* Multiple parallelisation strategies (OpenMP, MPI, GPU offloading, etc...) +* Full non-ideal MHD (Ohmic, ambipolar, Hall) +* Viscosity and thermal diffusion +* Super-timestepping for all parabolic terms +* Orbital advection (Fargo-like) +* Self-gravity +* Multi dust species modelled as pressureless fluids +* Multiple planets interraction =========================== Terms and condition of Use =========================== *Idefix* is distributed freely under the `CeCILL license `_, a free software license adapted to both international and French legal matters, in the spirit of and retaining -compatibility with the GNU General Public License (GPL). We expect *Idefix* to be referenced and acknowledeged by authors in their publications. +compatibility with the GNU General Public License (GPL). We expect *Idefix* to be referenced and acknowledeged by authors in their publications. At the minimum, the authors +should cite the *Idefix* `method paper `_. *Idefix* data structure and algorithm are derived from Andrea Mignone's `PLUTO code `_, released under the GPL license. *Idefix* also relies on the `Kokkos `_ performance portability programming ecosystem released under the terms @@ -84,6 +108,7 @@ under the European Union Horizon 2020 research and innovation programme (Grant a reference modules programmingguide + kokkos contributing faq changelog diff --git a/doc/source/kokkos.rst b/doc/source/kokkos.rst new file mode 100644 index 00000000..3faf1550 --- /dev/null +++ b/doc/source/kokkos.rst @@ -0,0 +1,10 @@ +===================== +Kokkos +===================== + +*Idefix* comes with the `Kokkos `_ library bundled as a git submodule. The bundled version of the Kokkos +library is known to work for the version of Idefix associated to it, and is compiled on the fly when +compiling Idefix. Hence, there is no need to download or compile separately Kokkos. + +For technical details on Kokkos, advanced users are encouraged to read the `Kokkos documentation `_ and watch the +`Kokkos lecture notes `_. diff --git a/doc/source/modules.rst b/doc/source/modules.rst index 41ec159f..13287ece 100644 --- a/doc/source/modules.rst +++ b/doc/source/modules.rst @@ -8,6 +8,23 @@ In this section, you will find a more detailed documentation about each module t :ref:`fargoModule` The orbital advection module, speeds up the computation of flows dominated by an azimuthal motion (such as discs). +:ref:`planetModule` + The planet module, which treats the planet-disk interaction and planet-planet interaction. + +:ref:`dustModule` + The dust module, modeling dust grains as a zero-pressure gas. + +:ref:`eosModule` + The custom equation of state module, allowing the user to define its own equation of state. + +:ref:`selfGravityModule` + The self-gravity computation module, handles the impact of the gas distribution on its own dynamic when massive + enough (as in a core collapse). + +:ref:`braginskiiModule` + The Braginskii module, models the anisotropic flux of heat and momentum + taking place in weakly collisional, magnetised plasma (like the intracluster medium). + :ref:`gridCoarseningModule` The grid coarsening module, that allows to derefine the grid in particular locations to speed up the computation. @@ -17,4 +34,9 @@ In this section, you will find a more detailed documentation about each module t :caption: Contents: modules/fargo.rst + modules/planet.rst + modules/dust.rst + modules/eos.rst + modules/selfGravity.rst + modules/braginskii.rst modules/gridCoarsening.rst diff --git a/doc/source/modules/braginskii.rst b/doc/source/modules/braginskii.rst new file mode 100644 index 00000000..e657cedd --- /dev/null +++ b/doc/source/modules/braginskii.rst @@ -0,0 +1,121 @@ +.. _braginskiiModule: + +Braginskii module +=================== + +Equations solved and methods +--------------------------- + +The ``Braginskii`` module implements the anisotropic heat and momentum fluxes specific +to weakly collisional, magnetised plasma like the intracluster medium +for which this module was originally designed. +In practice, it computes the modified Fourier law :math:`q_\mathrm{B}` and +the modified viscous stress tensor :math:`\Pi_\mathrm{B}` (Latter & Kunz 2012): + +:math:`q_\mathrm{B} = -\kappa_\parallel \left(\hat{b}\cdot \nabla T\right) \hat{b}` + +:math:`\Pi_\mathrm{B} = - \left( p_\perp - p_\parallel \right) \left( \hat{b} \hat{b} - \frac{1}{3} I \right)` + +where +:math:`T` and :math:`v` are respectively the temperature and the velocity of the fluid, +:math:`p_\parallel` and :math:`p_\perp` the pressure in the direction parallel and +perpendicular to the local magnetic field. +:math:`\kappa_\parallel` and :math:`\mu_\parallel` are the parallel thermal conductivity +and dynamic viscosity respectively, while +:math:`\hat{b}` is the unit vector in the direction of the local magnetic field +and :math:`I` the identity. +The pressure anisotropy can be computed from the following closure: +:math:`p_\perp - p_\parallel = 3\mu_\mathrm{B} \left(\hat{b}\hat{b}:\nabla v - \frac{1}{3} \nabla\cdot v \right)` (Schekochihin et al. 2010). + +The anisotropic heat flux from the ``Braginskii`` module is implemented in *Idefix* +according to the centered asymmetric scheme described in Sharma & Hammett (2007, Section 2.1). +The Braginskii viscous stress tensor is implemented with the same scheme, +though adapted to vector quantities. + +.. note:: + By default, the Braginskii module computes the transverse heat flux terms at the right + cell interface by a simple arithmetic average + (Eq. (6)-(7) from Sharma & Hammett 2007). + However in the same paper, the authors showed that this implementation can lead to + unphysical heat flux from high to low temperature regions. + So we implemented slope limiters for the computation of these transverse heat fluxes, + as described in Eq. (17) from Sharma & Hammett (2007). + Only the van Leer and the Monotonized Central (MC) limiters are available + since the minmod limiter has been found to be too diffusive. + Transverse viscous fluxes are limited in the same manner, + as described by ZuHone et al. (2015) in their fifth footnote. + +.. note:: + Perpendicular heat flux + :math:`q_\perp = -\kappa_\perp \left[ \nabla T - \left(\hat{b}\cdot \nabla T\right) \hat{b}\right]` + can also be taken into account + (see section :ref:`braginskiiParameterSection`), + but perpendicular viscous flux cannot in the current implementation. + However, both Braginskii operators can be used *in addition* to the classical + Fourier law and viscosity. Modeling both isotropic and anisotropic diffusion + should be equivalent to having different + parallel and perpendicular diffusivities. + +The main output of the ``Braginskii`` module is the addition of the anisotropic heat and/or +momentum flux terms to the other various fluxes. +The computational cost of the ``Braginskii`` module is therefore one +of a parabolic term, meaning that +the associated Courant-Friedrichs-Lewy (CFL) +condition can be stiff in case of very diffusive plasma. +However, like other parabolic operators in *Idefix*, a Runge-Kutta-Legendre super time-stepping +scheme is available to speed-up the computation (with respect to a fully explicit integration) +of the Braginskii heat flux and viscosity. + +.. + Please refer to Section 2.8 from Lesur et al. + for more details on the this super time-stepping scheme. + +.. _braginskiiParameterSection: + +Main parameters of the module +----------------------------- + +The ``Braginskii`` module can be enabled adding one or two lines in the ``[Hydro]`` section +starting with the keyword +`bragTDiffusion` or/and *bragViscosity*. The following table summarises the different options +associated to the activation of the Braginskii module: + ++--------+-----------------------+-------------------------+---------------------------------------------------------------------------------------+ +| Column | Entry name | Parameter type | Comment | ++========+=======================+=========================+=======================================================================================+ +| 0 | bragModule | string | | Activates Braginskii diffusion. Can be ``bragTDiffusion`` or ``bragViscosity``. | ++--------+-----------------------+-------------------------+---------------------------------------------------------------------------------------+ +| 1 | integration | string | | Specifies the type of scheme to be used to integrate the parabolic term. | +| | | | | Can be ``rkl`` or ``explicit``. | ++--------+-----------------------+-------------------------+---------------------------------------------------------------------------------------+ +| 2 | slope limiter | string | | Choose the type of limiter to be used to compute anisotropic transverse flux terms. | +| | | | | Can be ``mc``, ``vanleer`` or ``nolimiter``. | ++--------+-----------------------+-------------------------+---------------------------------------------------------------------------------------+ +| 3 | diffusivity type | string | | Specifies the type of diffusivity wanted. Can be ``constant`` or ``userdef``. | ++--------+-----------------------+-------------------------+---------------------------------------------------------------------------------------+ +| 4 | parallel diffusivity | real | | Mandatory if the diffusivity type is ``constant``. Not needed otherwise. | +| | | | | Value of the parallel diffusivity. Should be a real number. | ++--------+-----------------------+-------------------------+---------------------------------------------------------------------------------------+ +| 5 | normal diffusivity | real | | When bragModule ``bragTDiffusion`` and diffusivity type ``constant``, | +| | | | | value of the normal diffusivity. Should be a real number. | ++--------+-----------------------+-------------------------+---------------------------------------------------------------------------------------+ + +Numerical checks +--------------- + +In Cartesian geometry, the ``Braginskii`` module has been tested with many setups +and in all configurations of magnetic polarisation: +growth rates of local instabilities (see the MTI test inspired from Parrish et al. 2012) +and damped rates of eigenmodes of the corresponding Braginskii operator (tests not included). +In Cylindrical/Polar geometry, only the anisotropic heat conduction has been tested +with numerical measurements of the damped rates of its eigenmodes, in all directions +(tests not included). +In Spherical geometry, both Braginskii operators have been validated by measuring the damped rates +of their eigenmodes for a purely radial and purely azimuthal magnetic polarisation +(tests included except the viscosity with an azimuthal magnetic field). + +In conclusion, the ``Braginskii`` operators have been thoroughly tested in Cartesian geometry. +The same goes for the anisotropic heat flux in Cylindrical/Polar geometry while +the anisotropic viscosity has *never* been tested in this geometry. +In spherical geometry, both ``Braginskii`` operators have been partially validated +(diffusion along the polar axis has not been directly tested). diff --git a/doc/source/modules/dust.rst b/doc/source/modules/dust.rst new file mode 100644 index 00000000..ca371dad --- /dev/null +++ b/doc/source/modules/dust.rst @@ -0,0 +1,135 @@ +.. _dustModule: + +Dust fluid module +========================= + +Equations +--------- +The dust module is designed to treat dust grains as a zero-pressure gas coupled to the gas by a linear drag force on the velocity. +*Idefix* can handle as many dust species as one wants, each with different coupling constants. For each dust specie :math:`i`, the code solve the dust evolution equation + +.. math:: + + \partial_t \rho_i+\mathbf{\nabla}\cdot \rho \mathbf{v}_i&=0 + + \partial_t \rho_i \mathbf{v}_i + \nabla\cdot \rho \mathbf{v}_i\mathbf{v}_i&=\gamma_i \rho_i \rho (\mathbf{v}-\mathbf{v}_i)-\rho_i\mathbf{\nabla}\psi_G-\rho_i\mathbf{g} + + +where :math:`\rho_i` and :math:`\rho` are the dust and gas densities, :math:`\mathbf{v}_i` and :math:`\mathbf{v}` are the dust and gas velocities and :math:`\gamma_i` is the drag coefficient +between the dust and the gas. Note that by construction, all of the source terms (gravity, rotation, shearing box) of the gas are also automatically applied to each dust specie. + +When the gas follows an ideal or user-defined equation of state, the dust-gas drag leads to friction heating. Because the dust component does not posess an internal energy, *Idefix* +assumes that all of the friction heating is deposited in the gas internal energy as an addition heating term :math:`Q_d`: + +.. math:: + + Q_d = \sum_i \gamma_i \rho_i \rho (\mathbf{v}-\mathbf{v}_i)\cdot\mathbf{v}_i + +When the drag feedback is enabled (default behaviour), the gas is subject to an additional drag force + +.. math:: + + \mathbf{f}=\sum_i \gamma_i \rho_i \rho (\mathbf{v}_i-\mathbf{v}) + +which guarantees total momentum conservation. + +.. note:: + + The heating associated to the feedback does not require any additional source term in the energy equation as *Idefix* conserves the total gas energy by design. + + + +Drag CFL condition +------------------- +*Idefix* computes the drag terms with a time-explicit scheme. Hence, an addition CFL constraint arrises because of the drag: + +.. math:: + + dt < \min(\frac{1}{\sum_i\gamma_i(\rho_i+\rho)}) + +*Idefix* automatically adjusts the CFL to satisfy this inequality, in addition to the usual CFL condition. + +Dust parameters +--------------- + +The dust module can be enabled adding a block `[Dust]` in your input .ini file. The parameters are as follow: + ++----------------+-------------------------+---------------------------------------------------------------------------------------------+ +| Entry name | Parameter type | Comment | ++================+=========================+=============================================================================================+ +| nSpecies | integer | | Number of dust species to solve | ++----------------+-------------------------+---------------------------------------------------------------------------------------------+ +| drag | string, float, ... | | The first parameter describe the drag type. Possible values are: ``gamma``, ``tau``, | +| | | | ``size`` and ``userdef``. | +| | | | The remaining parameters gives the drag parameter :math:`\beta_i` for each dust specie. | +| | | | (see below). *Idefix* expect to have as many drag parameters as there are dust species. | ++----------------+-------------------------+---------------------------------------------------------------------------------------------+ +| drag_feedback | bool | | (optionnal) whether the gas feedback is enabled (default true). | ++----------------+-------------------------+---------------------------------------------------------------------------------------------+ + +The drag parameter :math:`\beta_i` above sets the functional form of :math:`\gamma_i(\rho, \rho_i, c_s)` depending on the drag type: + +``gamma``: + This sets :math:`\gamma_i=\beta_i` +``tau``: + This sets :math:`\gamma_i=1/(\rho \beta_i)` so that :math:`\beta_i` is the (constant) stopping time :math:`\tau_i` of the dust grains. +``size``: + This sets :math:`\gamma_i=c_s/\beta_i` so that :math:`\tau_i=(\rho \gamma_i)^{-1}=\beta_i/(\rho c_s)` where :math:`c_s` is the gas sound speed. + It designed to reproduce the behaviour of a fixed size dust grain of constant density that follows either Epstein or Stokes drag law: + + Epstein drag: + In the Epstein regime, :math:`\beta_i=\rho_s a` where :math:`\rho_s` is the solid density and :math:`a` is the solid size. + Stokes drag: + In the Stokes regime, :math:`\beta_i=4\rho_s a^2/(9\lambda_\mathrm{mfp})` where :math:`\lambda_\mathrm{mfp}` is the gas mean free path. + +``userdef``: + This allows the user to define a specialized drag law, that is a function :math:`\gamma_i(\rho, \rho_i, c_s, \beta_i)`. In this case, a user-defined + function should be enrolled by each drag instance (see the example in `test/Dust/FargoPlanet`). Note that the entry ``drag`` in your + input file should still contain a list of :math:`\beta_i`. + + +.. warning:: + The drag force assumes that the gas density field ``hydro->Vc(RHO...)`` is a volumic density. For 2D problems assuming + a razor-thin geometry, this assumption is incorrect since ``hydro->Vc(RHO...)`` is a surface density. In this case, + the user has to define a specific drag law since *Idefix* has no way to guess how to convert the surface density to + the volumic density (see example in `test/Dust/FargoPlanet`). + + + +Using the dust module +--------------------- + +Several examples are provided in the :file:`test/Dust` directory. Each dust specie is considered in Idefix as a instance of the `Fluid` class, hence +one can apply the technics used for the gas to each dust specie. Because *Idefix* can handle an arbitrarily number of dust species, each specie is stored +in an instance of `Fluid` and stored in a container (:code:`std::vector dust`) in the `DataBlock`. The same is true for the mirror `DataBlockHost`: the +dust primitive variable are all stored in :code:`std::vector dustVc` . For instance, initialising +a single dust specie is done as follow: + +.. code-block:: c++ + + + void Setup::InitFlow(DataBlock &data) { + // Create a host copy + DataBlockHost d(data); + + for(int k = 0; k < d.np_tot[KDIR] ; k++) { + for(int j = 0; j < d.np_tot[JDIR] ; j++) { + for(int i = 0; i < d.np_tot[IDIR] ; i++) { + + d.Vc(RHO,k,j,i) = 1.0; // Set the gas density to 1 + d.dustVc[0](RHO,k,j,i) = 1.0; // Set first dust specie density to 1 + + d.Vc(VX1,k,j,i) = 1; // Set the gas velocity to 1 + d.dustVc[0](VX1,k,j,i) = 0.0; // Set the dust velocity to 0 + + } + } + } + + // Send it all, if needed + d.SyncToDevice(); + } + + + +All of the dust fields are automatically outputed in the dump and vtk outputs created by *Idefix*. diff --git a/doc/source/modules/eos.rst b/doc/source/modules/eos.rst new file mode 100644 index 00000000..041a378d --- /dev/null +++ b/doc/source/modules/eos.rst @@ -0,0 +1,55 @@ +.. _eosModule: + +Equation of state module +========================= + +About +--------- + +By default, *Idefix* can handle either isothermal equation of states (in which case the pressure is never computed, and the code +works with a prescribed sound speed function), or an ideal adiabatic equation of state (assuming a constant adiabatic exponent :math:`\gamma`). + +If one wants to compute the dynamics of more complex fluids (e.g. multiphase flows, partial ionisation, etc.), then the ideal adiabatic equation of +state is not sufficient and one needs to code a *custom* equation of state. This is done by implementing the class ``EquationOfState`` with the functions +required by *Idefix* algorithm. + +Functions needed +----------------- + +*Idefix* requires the definition of 3 functions that are called during the integration loop: a function to compute the first adiabatic exponent :math:`\Gamma_1`, and +the functions needed to convert pressure to internal energy and *vice-versa*. The definition for these function is: + +.. code-block:: c++ + + class EquationOfState { + // First adiabatic exponent. + KOKKOS_INLINE_FUNCTION real GetGamma(real P , real rho ) const { + real gamma; + // Compute gamma (needed for sound speed estimations, 5/3 would work in general) + return gamma; + } + + // Compute the internal energy from pressure and density + KOKKOS_INLINE_FUNCTION + real GetInternalEnergy(real P, real rho) const { + real eint; // = ... + return eint; + } + + // Compute the pressure from internal energy and density + KOKKOS_INLINE_FUNCTION + real GetPressure(real Eint, real rho) const { + real P; // = ... + return P; + } + } + + +How to use a custom EOS +----------------------- + +#. Copy the template file ``eos_template.hpp`` (in src/fluid/eos) in your problem directory and rename it (e.g. ``my_eos.hpp``) +#. Make sure that you have not enabled the ISOTHERMAL approximation in your ``definitions.hpp`` +#. Implement your EOS in ``my_eos.hpp``, and in particular the 3 EOS functions required. +#. in cmake, enable ``Idefix_CUSTOM_EOS`` and set ``Idefix_CUSTOM_EOS_FILE`` to ``my_eos.hpp`` (or the filename you have chosen in #1) +#. Compile and run diff --git a/doc/source/modules/fargo.rst b/doc/source/modules/fargo.rst index 60489ef2..07f98336 100644 --- a/doc/source/modules/fargo.rst +++ b/doc/source/modules/fargo.rst @@ -1,7 +1,7 @@ .. _fargoModule: -The FARGO module -================ +Orbital advection (aka "FARGO") module +======================================= The Fargo module implements the orbital advection algorithm, to accelerate the computation of flows subject to a dominant axisymmetric azimuthal velocity component. This is in particular the case diff --git a/doc/source/modules/gridCoarsening.rst b/doc/source/modules/gridCoarsening.rst index be0d5f6a..987a74ed 100644 --- a/doc/source/modules/gridCoarsening.rst +++ b/doc/source/modules/gridCoarsening.rst @@ -1,6 +1,6 @@ .. _gridCoarseningModule: -The Grid coarsening module +Grid coarsening module ========================== About grid coarsening diff --git a/doc/source/modules/planet.rst b/doc/source/modules/planet.rst new file mode 100644 index 00000000..ac8f510d --- /dev/null +++ b/doc/source/modules/planet.rst @@ -0,0 +1,91 @@ +.. _planetModule: + +Planet module +============== + +The planet module allows one to model one or several planets interacting with the gas and other planets. One can force the planet to be on a fixed orbit (in which case +migration and planet-planet interaction are ignored) or on the contrary integrate the equation of motion of the planet(s) simulatenously with the gas. All of the parameters +used by the Planet module are set in the ``[Planet]`` block of the input file and read at runtime. + +Common parameters for every planets: + ++------------------------+-----------------------+-----------------------------------------------------------------------------------------------------------+ +| Entry name | Parameter type | Comment | ++========================+=======================+===========================================================================================================+ +| smoothing | string, float, float | | Expression of the planet potential (``plummer`` or ``polynomial``) | +| | | | and smoothing length for the regularisation of the planet potential around the planet location. | +| | | | Mandatory | ++------------------------+-----------------------+-----------------------------------------------------------------------------------------------------------+ +| feelDisk | bool | | Whether the planet feels the gravitational forces due to the gas. This parameter enables migration. | +| | | | Mandatory | ++------------------------+-----------------------+-----------------------------------------------------------------------------------------------------------+ +| feelPlanets | bool | | Whether the planet interacts gravitationnaly with other planets. | +| | | | Mandatory | ++------------------------+-----------------------+-----------------------------------------------------------------------------------------------------------+ +| integrator | (string) | | Type of time integrator. Can be ``analytical`` (planets are on fixed orbits) or ``rk4`` (numerical | +| | | | integration). | +| | | | default: ``rk4`` | ++------------------------+-----------------------+-----------------------------------------------------------------------------------------------------------+ +| indirectPlanets | (bool) | | Include indirect planet term arising from the acceleration of the star by the planet. | +| | | | default: ``true`` | ++------------------------+-----------------------+-----------------------------------------------------------------------------------------------------------+ +| torqueNormalization | (float) | | Add a multiplicative factor in the gravitational torque computation | +| | | | when updating the velocity components of the planet(s) if feelDisk=``true``. | +| | | | default: ``1.0`` | ++------------------------+-----------------------+-----------------------------------------------------------------------------------------------------------+ +| hillCut | (bool) | | Exclude Hill's sphere (with radius :math:`r_\mathrm{h}`) when computing the gravitational torque | +| | | | exerted by the gas on the planet. More specifically, we extract the material below | +| | | | :math:`0.5r_\mathrm{h}` and use a :math:`\mathrm{sin}^2` transition until :math:`r_\mathrm{h}`. | +| | | | default: ``false`` | ++------------------------+-----------------------+-----------------------------------------------------------------------------------------------------------+ +| masstaper | (float) | | Whether the planet masses are progressively increased or set initially to their values. If set to ``0``,| +| | | | the masstaper is disabled (default). Otherwise, the value sets the time needed to reach the final | +| | | | planets mass. | ++------------------------+-----------------------+-----------------------------------------------------------------------------------------------------------+ + + +Parameters specific to each planet (each entry is followed by a list of values for each planet): + ++-----------------------+---------------------+----------------------------------------------------------------------------------------------------------------+ +| Entry name | Parameter type | Comment | ++=======================+=====================+================================================================================================================+ +| planetToPrimary | float, (float, ...) | | Mass ratio between the planet and the central star. | +| | | | mandatory | ++-----------------------+---------------------+----------------------------------------------------------------------------------------------------------------+ +| initialDistance | float, (float, ...) | | Planet initial distance :math:`\displaystyle\frac{x_\mathrm{p}}{1+\mathrm{initialEccentricity}}`. | +| | | | mandatory | ++-----------------------+---------------------+----------------------------------------------------------------------------------------------------------------+ +| initialEccentricity | (float, float, ...) | | Planet initial eccentricity. | +| | | | default: ``0`` | ++-----------------------+---------------------+----------------------------------------------------------------------------------------------------------------+ +| initialInclination | (float, float, ...) | | Planet initial inclination. | +| | | | default: ``0`` | ++-----------------------+---------------------+----------------------------------------------------------------------------------------------------------------+ +| tOffset | (float, float, ...) | | Are the planet(s) directly put in the simulations at t = 0 (default)? | +| | | | Otherwise, the value sets the time to wait before introducing the planet(s). | +| | | | default: ``0`` | ++-----------------------+---------------------+----------------------------------------------------------------------------------------------------------------+ + +.. note:: + Note that ``integrator=analytical`` is incompatible with ``feelDisk=true``, ``feelPlanet=true``, ``initialEccentricity!=0``, ``initialInclination!=0``. + +``smoothing`` takes 3 values. The first one gives the shape of the planet potential. The last two values (noted here ``a`` and ``b``) give the expression of the smoothing length, of the form :math:`\epsilon=ax_1^b`. In planet-disk interaction modeling, we usually choose ``a = aspect_ratio*thickness_smoothing`` and ``b = flaring_index``, with ``thickness_smoothing = 0.6`` typically in 2D (see, e.g., Masset & Benitez-Llambay, ApJ 817, 19 (2016)). + +* A ``plummer`` potential has the form: +.. math:: + \Phi_\mathrm{p}=\displaystyle-\frac{\mathrm{planetToPrimary}}{\sqrt{D_\mathrm{p}^2+\epsilon^2}} +in cartesian coordinates :math:`(x,y,z)`, for a planet located at :math:`(x_\mathrm{p},y_\mathrm{p},z_\mathrm{p})` and with :math:`D_\mathrm{p}=\sqrt{(x-x_\mathrm{p})^2+(y-y_\mathrm{p})^2+(z-z_\mathrm{p})^2}`. + +* A ``polynomial`` potential has the form: +.. math:: + \Phi_\mathrm{p}=\left\{ + \begin{array}{ll} + \displaystyle-\mathrm{planetToPrimary}(\frac{D_\mathrm{p}}{\epsilon}^4 - 2\frac{D_\mathrm{p}}{\epsilon}^3 + 2\frac{D_\mathrm{p}}{\epsilon}) & \mbox{if}~D_\mathrm{p}<\epsilon \\ + \displaystyle-\frac{\mathrm{planetToPrimary}}{D_\mathrm{p}} & \mbox{else} + \end{array} + \right. + +.. note:: + In 3D, it is possible to simulate only half of the disk by setting one of the boundary in X3 to 0 (in polar coordinates) or X2 to :math:`\pi/2` (in spherical coordinates). + In this case, we correct the computation of the gravitational torque exerted by the gas onto the planet(s) by removing the contribution of the vertical component of the torque and doubling + the horizontal contributions (hence assuming midplane symmetry). diff --git a/doc/source/modules/selfGravity.rst b/doc/source/modules/selfGravity.rst new file mode 100644 index 00000000..6254caa8 --- /dev/null +++ b/doc/source/modules/selfGravity.rst @@ -0,0 +1,101 @@ +.. _selfGravityModule: + +Self Gravity module +=================== + +Equations solved and method +--------------------------- + +The ``SelfGravity`` module implements the computation of a self-gravitational potential generated +by the gas distribution. Physically, it solves for the potential :math:`\psi_{SG}` that satisfies the +Poisson equation + +:math:`\Delta \psi_{SG}=4\pi G_c\,\rho` (where :math:`G_c` is the gravitational constant in code +units) + +This computation becomes relevant when the said gas distribution +is massive enough to influence its own dynamic. This is the case for example in young protoplanetary +discs, which are subject to gravitational instabilities and for which this module was originally designed. + +The ``SelfGravity`` module implemented in *Idefix* follows the algorithm of the BIConjugate Gradient +STABilized (BICGSTAB) method, a common iterative method designed to solve sparse linear systems (see for example +Saad, Y. (2003). Iterative methods for sparse linear systems. Society for Industrial and Applied Mathematics.). +This specific method allows to solve the Poisson equation in any dimensions and any geometries. + +.. note:: + By default, the selfGravity module uses the BICGSTAB method to invert the Poisson equation. + However, an optional preconditionning feature (PBICGSTAB) is + implemented to handle particularly irregular and inhomogeneous grids, that may prevent the + convergence of the classic BICGSTAB. Note also that a simpler (yet very slow) Jacobi method + has been left for debug purpose. The user can also try the conjugate gradient and minimal residual + methods which have been tested successfully and are faster than BICGSTAB for some problems/grids. + +The main output of the ``SelfGravity`` module is the addition of the self-gravitational potential inferred from the +gas distribution to the various sources of gravitational potential. At the beginning of every (M)HD step, the module is called to compute +the potential due to the mass distribution at the given time. The potential computed by the ``SelfGravity`` module +is then added to the other potential sources by the ``Gravity`` class (planets, central and user-defined potential). +The computational cost of one selfGravity computation depends on the complexity of the +gravitational field and the speed at which the density field evolves. + +Main parameters of the module +----------------------------- + +The ``SelfGravity`` module is a submodule of the ``Gravity`` module to compute the gravitationnal potential. Hence, it is enabled +by adding ``selfgravity`` to the ``potential`` in the ``[Gravity]`` block (see :ref:`gravitySection`). The parameters specific to self gravity are to be +set in a dedicated ``[SelfGravity]`` block: + ++----------------+-------------------------+---------------------------------------------------------------------------------------------+ +| Entry name | Parameter type | Comment | ++================+=========================+=============================================================================================+ +| solver | string | | Specifies which solver should be used. Can be ``Jacobi``, ``CG``, ``MINRES``, ``BICGSTAB``| +| | | | which corresponds to Jacobin, conjugate gradient, Minimal residual or bi-conjugate | +| | | | stabilised method. Note that a preconditionned version is available adding a ``P`` to | +| | | | the solver name (e.g. ``PCG`` or ``PBIGCSTAB`` ). | ++----------------+-------------------------+---------------------------------------------------------------------------------------------+ +| targetError | real | | Set the error allowed in the residual :math:`r=\Delta\psi_{SG}/(4\pi G_c)-\rho`. The error| +| | | | computation is based on a L2 norm. Default is 1e-2. | ++----------------+-------------------------+---------------------------------------------------------------------------------------------+ +| maxIter | int | | Set the maximum number of iterations allowed to the solver to reach convergence. Default | +| | | | is 1000. | ++----------------+-------------------------+---------------------------------------------------------------------------------------------+ +| skip | int | | Set the number of integration cycles between each computation of self-gravity potential. | +| | | | Default is 1 (i.e. self-gravity is computed at every cycle). | ++----------------+-------------------------+---------------------------------------------------------------------------------------------+ + + +Boundary conditions on self-gravitating potential +-------------------------------------------------- + +Solving for the self-gravitating potential require the definition of a boundary condition for said potential. This is done with the entries +``boundary-Xn-dir`` of the ``[SelfGravity]`` block, where ``n`` can be 1, 2 or 3 and is the direction for the boundary condition and ``dir`` can be ``beg`` or ``end`` and +indicates the side of the boundary. + +The boundary conditions can be following + ++-----------------------+------------------------------------------------------------------------------------------------------------------+ +| ``boundary-Xn-dir`` | Comment | ++=======================+==================================================================================================================+ +| nullpot | Zero potential boundary conditions. The potential is set to zero in the ghost cells. | ++-----------------------+------------------------------------------------------------------------------------------------------------------+ +| nullgrad | Zero gradient boundary conditions. The potential in the last active cell is copied in the ghost cells. | ++-----------------------+------------------------------------------------------------------------------------------------------------------+ +| periodic | Periodic boundary conditions. The potential is copied between beg and end sides of the boundary. | ++-----------------------+------------------------------------------------------------------------------------------------------------------+ +| axis | | Axis boundary condition. Should be used in spherical coordinate in the X2 direction when the domain starts/stop| +| | | on the axis. | ++-----------------------+------------------------------------------------------------------------------------------------------------------+ +| origin | | In spherical coordinates, artificially extends the grid used to compute the potential close to R=0. | +| | | should only be used in X1-beg direction. | ++-----------------------+------------------------------------------------------------------------------------------------------------------+ +| userdef | |User-defined boundary conditions. The boundary condition function should be enrolled in the setup constructor | +| | | (see :ref:`userdefBoundaries`). | ++-----------------------+------------------------------------------------------------------------------------------------------------------+ + +.. note:: + The method in fully periodic setups requires the removal of the mean gas density + before solving Poisson equation. This is done automatically if all of the self-gravity boundaries are set to ``periodic``. + Hence, make sure to specify all self-gravity boundary conditions as periodic for such setups, otherwise the solver will + fail to converge. + +The selfGravity module in *Idefix* is fully parallelised. This means that one can have a MPI domain decomposition in any spatial direction +either on CPU or GPU. diff --git a/doc/source/programmingguide.rst b/doc/source/programmingguide.rst index 7a54a2be..18795b50 100644 --- a/doc/source/programmingguide.rst +++ b/doc/source/programmingguide.rst @@ -13,9 +13,10 @@ Data types Because *Idefix* can run on GPUs, and since GPUs experience a significant speedup when working with single precision arithmetic, a specific ``real`` datatype is used for all floating point -operations in *Idefix*. This is by default aliased to ``double`` in `idefix.hpp`, but it can easily be modified -to ``float`` for specific problems. Note however that this loss of precision might have a strong -impact on the quality of the solution and is therefore not recommended. +operations in *Idefix*. This is by default aliased to ``double`` in `idefix.hpp`, but it can easily converted +to ``single`` with cmake ``Idefix_PRECISION`` property. Single precision arithemtic can lead to very significant speedups +on some GPU architecture, but is not recommended for production runs as it can have an impact on the precision or even +convergence of the solution. Host and device =============== @@ -162,9 +163,8 @@ For instance, a sum over all of the elements would be done through: In the above example, ``localSum`` is the temporary variable *on the device* over which portions of the reduction are performed, while ``mySum`` is the final variable, *on the host* where the result is stored. -It is possible to do other reduction operations -like findining minimum, maximum etc (see -`Kokkos custom reductions `_ +It is possible to do other reduction operations like findining minimum, maximum etc (see +`Kokkos custom reductions `_ for a list). For instance, the minimum value is obtained with the following code snippet: @@ -293,9 +293,9 @@ specific to this subprocess, in contrast to ``Grid``). Here is the full API for .. _hydroClass: -``Hydro`` class +``Fluid`` class --------------------- -The ``Hydro`` class (and its sub-classes) contains all of the fields and methods specific to (magneto) hydrodynamics. While +The ``Fluid`` class (and its sub-classes) contains all of the fields and methods specific to (magneto) hydrodynamics. While interested users may want to read in details the implementation of this class, we provide below a list of the most important members @@ -326,6 +326,21 @@ Because the code uses contrained transport, the field defined on cell faces is s array. Just like for ``Vc``, there are aliases, with "s" suffixes defined to simplify the adressing of the magnetic field components, as ``Vs(BX2s,k,j,i)``. +It is important to realise that the ``Fluid`` class is a class template, that depends on the type of +fluid that is modelled (encoded in a ``Physics`` class). By default, *Idefix* always instantiates +one "default" fluid that contains the "default" physics requested by the user. +This default fluid is, for compatibility reasons with *Idefix* v1, called `hydro` and is accessible +from the ``dataBlock`` class as a pointer. It is defined as + +.. code-block:: c++ + + Fluid hydro; + + +Additional fluids can be instantiated by *Idefix* for some problems, such as pressureless fluids +to model dust grains (see :ref:`dustModule`). + + .. _datablockhostClass: ``DataBlockHost`` class @@ -390,7 +405,7 @@ The ``DumpImage`` class definition is class DumpImage { public: - DumpImage(std::string, Output &); // constructor with dump filename and output object as parameters + DumpImage(std::string, DataBlock *); // constructor with dump filename and output object as parameters int np_int[3]; // number of points in each direction int geometry; // geometry of the dump @@ -403,7 +418,7 @@ The ``DumpImage`` class definition is }; -Typically, a ``DumpImage`` object is constructed invoking the ``DumpImage(filename, output)`` constructor, +Typically, a ``DumpImage`` object is constructed invoking the ``DumpImage(filename, data)`` constructor, which essentially opens, allocate and load the dump file in memory (when running with MPI, each processor have access to the full domain covered by the dump, so try to avoid loading very large dumps!). The user can then have access to the dump content using the variable members of the object @@ -422,7 +437,7 @@ finished working with it. An example is provided in :ref:`setupInitDump`. .. _LookupTableClass: ``LookupTable`` class ------------------ +--------------------- The ``LookupTable`` class allows one to read and interpolate elements from a coma-separated value (CSV) file or a numpy file (generated from ``numpy.save`` in python). @@ -524,36 +539,111 @@ The ``Get`` and ``GetHost`` functions expect a C array of size ``nDim`` and retu .. note:: Usage examples are provided in `test/utils/lookupTable`. + +.. _debugging: + Debugging and profiling ======================= -The easiest way to trigger debugging in ``Idefix`` is to switch on ``Idefix_DEBUG`` in cmake (for instance +The easiest way to trigger debugging in *Idefix* is to switch on ``Idefix_DEBUG`` in cmake (for instance adding ``-DIdefix_DEBUG=ON`` when calling cmake). This forces the code to log each time a function is called or returned (this is achieved thanks to the ``idfx::pushRegion(std::string)`` and ``idfx::popRegion()`` which are -found at the beginning and end of each function). In addition, ``Idefix_DEBUG`` enables Kokkos array bound checks, which -will throw an error each time one tries to access an array out of its allocated memory space. Note that all of these +found at the beginning and end of each function). In addition, ``Idefix_DEBUG`` will force *Idefix* to wait for each +``idefix_for`` to finish before continuing to the next instruction (otherwise, ``idefix_for`` are asynchronous when using +an accelerator). This simplifies the detection and identification of bugs in ``idefix_for`` loops. Note that all of these debugging features induce a large overhead, and should therefore not be activated in production runs. -It is also possible to use `Kokkos-tools `_ to debug and profile the code. -For instance, on the fly profiling, can be enabled with the Kokkos ``space-time-stack`` module. To use it, simply clone -``Kokkos-tools`` to the directory of your choice (say ``$KOKKOS_TOOLS``), then ``cd`` to -``$KOKKOS_TOOLS/profiling/space-time-stack`` and compile the module with ``make``. +If you suspect an out-of-bound access, it may be worth enabling additionaly ``Kokkos_ENABLE_BOUNDS_CHECK`` that will check +that you are not trying to access an array outside of its bounds. -Once the profiling module is compiled, you can use it by setting the environement variable ``KOKKOS_PROFILE_LIBRARY``. -For instance, in bash: +If you want to profile the code, the simplest way is to use the embedded profiling tool in *Idefix*, adding ``-profile`` to the command line +when calling the code. This will produce a simplified profiling report when the *Idefix* finishes. + +It is also possible to use `Kokkos-tools `_ for more advanced profiling/debbugging. To use it, +you must compile Kokkos tools in the directory of your choice and enable your favourite tool +by setting the environement variable ``KOKKOS_TOOLS_LIBS`` to the tool path, for instance: .. code-block:: bash - export KOKKOS_PROFILE_LIBRARY=$KOKKOS_TOOLS/profiling/space-time-stack/kp_space_time_stack.so + export KOKKOS_TOOLS_LIBS=/profiling/space-time-stack/libkp_space_time_stack.so -Once this environement variable is set, *Idefix* automatically logs profiling informations when it ends (recompilation of *Idefix* -is not needed). -.. tip:: +.. _defensiveProgramming: + +Defensive Programming +--------------------- + +``idefix.hpp`` defines useful function-like macros to program defensively and create +informative error messages and warnings. + +First and foremost, ``IDEFIX_ERROR`` and ``IDEFIX_WARNING`` can be used in host space. + +.. code-block:: c++ + + #include "idefix.hpp" + + #ifdef HAVE_ENERGY + IDEFIX_WARNING("This setup is not tested with HAVE_ENERGY defined"); + #endif + + real boxSize {-1}; + + // ... determine boxSize at runtime from parameter file + + if(boxSize<1e-5) { + IDEFIX_ERROR("This setup requires a minimal box size of 1e-5"); + } + + +``idefix.hpp`` also defines the more advanced ``RUNTIME_CHECK_HOST`` and +``RUNTIME_CHECK_KERNEL`` macros, which are useful to define arbitrary sanity checks at +runtime in host space and within kernels respectively, together with a nicely formatted +error message. + +Both macros take a condition (a boolean expression that *should* evaluate to ``true`` at +runtime), and an error message. Additional arguments may be supplemented to the error +message using string interpolation. Note however that this only works on CPU, so +``RUNTIME_CHECK_KERNEL`` also expects a default error message that'll be used when +running on GPUs. + +As an illustrative example, here's how they can be used to verify some assumptions at +runtime. + +.. code-block:: c++ - By default, ``Kokkos-tools`` assumes the user code is using MPI. If one wants to perform profiling in serial, one should disable MPI before - compling the ``space-time-stack`` module. This is done by editing the makefile in ``$KOKKOS_TOOLS/profiling/space-time-stack`` - changing the compiler ``CXX`` to a valid serial compiler, and adding ``-DUSE_MPI=0`` to ``CFLAGS``. + #include "idefix.hpp" + + const int MAX_NPARTICLES = 1024; + const int NPARTICLES = 128; + const real lightSpeed = 1.0; + auto particleSpeed = IdefixArray1D("particleSpeed", NPARTICLES); + + RUNTIME_CHECK_HOST( + NPARTICLES<=MAX_NPARTICLES, + "The number of particles requested (%i) is too high (max is %i)", + NPARTICLES, MAX_NPARTICLES + ); + + idefix_for("check particle speeds", + 0, NPARTICLES, + KOKKOS_LAMBDA(int idx) { + RUNTIME_CHECK_KERNEL( + particleSpeed(idx) < lightSpeed, + "Speeding particle(s) detected !", // this default error message is used on GPUs + "Particle at index %i has speed %.16e, which is greater than light speed (%.16e)", + idx, particleSpeed(idx), lightSpeed + ); + } + ); + + +``RUNTIME_CHECK_HOST`` and ``RUNTIME_CHECK_KERNEL`` are considered debug-only tools, so +are by default excluded at compilation, and do not impact performance in production. +To enable them, use the `-D Idefix_RUNTIME_CHECKS=ON` configuration flag. + +It may also be useful to implement debug-only safeguards with custom logic that doesn't +fit `RUNTIME_CHECK_*` macros. This can be achieved by using the compiler directive +`#ifdef RUNTIME_CHECKS` directly. Minimal skeleton ================ diff --git a/doc/source/quickstart.rst b/doc/source/quickstart.rst index 44c56562..870072d8 100644 --- a/doc/source/quickstart.rst +++ b/doc/source/quickstart.rst @@ -8,7 +8,7 @@ Download and install *Idefix* One first need to download *Idefix* from the public git. Say you want to install *Idefix* in the directory ``~/src/idefix``, you need to run:: cd ~/src - git clone https:// idefix + git clone https://github.com/idefix-code/idefix.git idefix cd idefix git submodule init git submodule update @@ -29,11 +29,11 @@ install directory of *Idefix*. We therefore conclude the installation with:: Configure and run the SOD tube test problem =========================================== -The test problem are all located in the ``$IDEFIX_DIR/test`` directory of the distribution. To access the Sod shock tube test, one go to:: +The test problem are all located in the ``$IDEFIX_DIR/test`` directory of the distribution. To access the Sod shock tube test, one goes to:: cd $IDEFIX_DIR/test/HD/sod -From there, one sees 3 files and a directory: +From there, one sees 4 files and a directories: ``definitions.hpp`` The configuration file. It contains C preprocessor directives for configuration options which require a re-compilation of the code. These are mostly @@ -46,11 +46,15 @@ From there, one sees 3 files and a directory: ``idefix.ini`` The input file. This file is read at runtime *only*. It defines most of the properties of the code: resolution, integrator, output, physical modules. -``python`` directory - This directory is provided with most of the tests. Its content allows one to check that the code output is consistent with what is expected. +``testme.py`` + The validation script. It uses the python class ``idfx_test.py`` to validate the code outputs for that particular test. Use ``testme.py -help`` to find out the options. + +``python`` + Directory that contains the standard test validation: i.e. comparison against an analytical solution (when it exists) + For the time being, the files are already set up for the Sod test problem. The only thing lacking is a ``makefile`` to actually compile the code. -In *Idefix* the makefile is created by `Cmake `_ ,a tool to control code generation on diverse platforms. To configure *Idefix*, +In *Idefix* the makefile is created by `cmake `_ ,a tool to control code generation on diverse platforms. To configure *Idefix*, you need Cmake version >=3.16 installed on your machine. For this quickstart, let us configure the code to run on the cpu in serial (default behaviour). Assuming a ``cmake`` is in the PATH, we simply type in:: @@ -66,14 +70,19 @@ Finally, we compile and run the code:: make -j 8 ./idefix -This test being one dimensional, it runs very fast. We can check that the final solution match the prediction of the shock tube problem. To this end, we go to the ``python`` -subdirectory and run the test:: +This test being one dimensional, it runs very fast. We can check that the final solution match the prediction of the shock tube problem. To this end, we +use the ``testme.py`` script: - cd python - python3 ./testidefix.py + ./testme.py -check -If everything goes well, ``testidefix.py`` will load the latest output produced by idefix, display it, compare it with an analytical solution and tell you -whether the error is acceptable or not. +The ``-check`` option tells the test suite that the simulation has already run and we just need a validation of the result. Note that it's also +possible to validate all of the possible combination of algorithms using the ``-all`` option. In that case, the script automatically +configure, compile and validate a series of test with varying combination of algorithms. + +.. warning:: + Note that the validation relies on large reference + files that are stored in the separate `idefix-code/reference` repository that is automatically + cloned as a submodule along with Kokkos if you follow the above procedure. Configure and run the Orszag-Tang test problem @@ -87,8 +96,9 @@ test with:: cmake $IDEFIX_DIR -Once the code is configured, it can be ran:: +Once the code is configured, it can be compiled and ran:: + make -j 8 ./idefix This test can take much more time that the sod test since it is 2D. In the end, you can visualize the results, which are written as VTK files using diff --git a/doc/source/reference.rst b/doc/source/reference.rst index 1c08c970..d4447b70 100644 --- a/doc/source/reference.rst +++ b/doc/source/reference.rst @@ -1,5 +1,5 @@ ===================== -Reference guide +User guide ===================== First, it is good practice to set the environment variable ``IDEFIX_DIR`` to the root path of your *Idefix* distribution, as it is needed at several stages. Setting up *Idefix* for a particular problem implies editing several files, some of which are automatically generated. To set up a particular *Idefix* setup, 5 steps are required. diff --git a/doc/source/reference/commandline.rst b/doc/source/reference/commandline.rst index 105f1569..9feef769 100644 --- a/doc/source/reference/commandline.rst +++ b/doc/source/reference/commandline.rst @@ -21,10 +21,16 @@ Several options can be provided at command line when running the code. These are +--------------------+-------------------------------------------------------------------------------------------------------------------------+ | -maxcycles n | stops when the code has performed ``n`` integration cycles | +--------------------+-------------------------------------------------------------------------------------------------------------------------+ +| -force_init | | call initial conditions before reading dump file (this has no effect if -restart is not also passed). | +| | | This option is useful when more physics is enabled when restarting from a dump (e.g. switching on MHD or dust) | +| | | as it initialize from the initial conditions the quantities that are absent from the restart dump | ++--------------------+-------------------------------------------------------------------------------------------------------------------------+ | -nolog | disable log files | +--------------------+-------------------------------------------------------------------------------------------------------------------------+ | -nowrite | disable all writes (useful for raw performance measures or for tests). This option implies ``-nolog`` | +--------------------+-------------------------------------------------------------------------------------------------------------------------+ +| -profile | Enable on-the-fly performance profiling (a final text report is automatically generated). | ++--------------------+-------------------------------------------------------------------------------------------------------------------------+ | -Werror | warning messages are considered as errors and stop the code with a non-zero exit code. | +--------------------+-------------------------------------------------------------------------------------------------------------------------+ diff --git a/doc/source/reference/idefix.ini.rst b/doc/source/reference/idefix.ini.rst index ada46ff6..ae2b8e5a 100644 --- a/doc/source/reference/idefix.ini.rst +++ b/doc/source/reference/idefix.ini.rst @@ -128,7 +128,11 @@ This section is used by the hydrodynamics class of *Idefix*. It defines the hydr | | | | to be enrolled with ``EnrollIsoSoundSpeed(IsoSoundSpeedFunc)`` | | | | | (see :ref:`functionEnrollment`). In this case, the second parameter is not used. | +----------------+-------------------------+---------------------------------------------------------------------------------------------+ -| gamma | float | Adiabatic index when ISOTHERMAL is not defined. Default to 5/3 if not set. | +| gamma | float | | Adiabatic index when ISOTHERMAL is not defined. Default to 5/3 if not set. | +| | | | NB: this parameter is used only by the default equation of state implemented in *Idefix* | +| | | | Custom equation of states (:ref:`eosModule`) ignore this parameter | ++----------------+-------------------------+---------------------------------------------------------------------------------------------+ +| tracer | integer | Number of passive tracers associated to the fluid. Default to 0 if not set. | +----------------+-------------------------+---------------------------------------------------------------------------------------------+ | resistivity | string, string, (float) | | Switches on Ohmic diffusion. | | | | | The first parameter can be ``explicit`` or ``rkl``. When ``explicit``, diffusion is | @@ -193,6 +197,9 @@ This section is used by the hydrodynamics class of *Idefix*. It defines the hydr | | | | limiter when strong shocks are detected. The entry parameter is the threshold above which | | | | | a shock is considered "strong", in units of :math:`|\nabla P /P|`. A low value hence tends| | | | | to increase the code numerical diffusivity. Typical values are 1 to 10. | +| | | | Note that it is possible to enroll a user-defined function to flag specific cells for | +| | | | shock flattening, in addition to the default flag. This user function can be enrolled | +| | | | with ``Hydro.shockFlattening.EnrollUserShockFlag(UserShockFunc)`` . | +----------------+-------------------------+---------------------------------------------------------------------------------------------+ @@ -227,10 +234,19 @@ This section enables the orbital advection algorithm provided in *Idefix*. More | | | | Default: 10 | +----------------+-------------------------+---------------------------------------------------------------------------------------------+ +.. _gravitySection: + ``Gravity`` section -------------------- -This section enables gravity in the form of a gravitational potential and/or an acceleration vector +This section enables gravity in the form of a gravitational potential and/or an acceleration vector. The gravitational potential used by the code +reads + +:math:`\psi=-G_c M_{\rm central}/R+\psi_{SG}+\psi_{\rm userdef}` + +where :math:`G_c` is the gravitational constant, :math:`M_{\rm central}` is the mass of a central object, :math:`\psi_{SG}` is the +self-gravitational potential and :math:`\psi_{\rm userdef}` is a user-defined potential. Each term can be enabled individually in the gravity +section as followed: +----------------+-------------------------+---------------------------------------------------------------------------------------------+ | Entry name | Parameter type | Comment | @@ -239,21 +255,62 @@ This section enables gravity in the form of a gravitational potential and/or an | | | | total potential used by *Idefix*. | | | | | | | | | | * ``userdef`` allows the user to give *Idefix* a user-defined potential function. In this | -| | | | ``Gravity`` class expects a user-defined potential function to be enrolled with | +| | | | case, ``Gravity`` class expects a user-defined potential function to be enrolled with | | | | | ``Gavity::EnrollPotential(GravPotentialFunc)`` (see :ref:`functionEnrollment`) | | | | | * ``central`` allows the user to automatically add the potential of a central point mass. | | | | | In this case, the central mass is assumed to be 1 in code units. This can be modified | | | | | using the Mcentral parameter, or using the ``Gravity::SetCentralMass(real)`` method. | | | | | * ``selfgravity`` enables the potential computed from solving Poisson equation with the | -| | | | density distribution | +| | | | density distribution (see :ref:`selfGravitySection` and :ref:`selfGravityModule`). | +----------------+-------------------------+---------------------------------------------------------------------------------------------+ | Mcentral | real | | Mass of the central object when a central potential is enabled (see above). Default is 1. | +----------------+-------------------------+---------------------------------------------------------------------------------------------+ +| gravCst | real | | Set the value of the gravitational constant :math:`G_c` used by the central | +| | | | mass potential and self-gravitational potential (when enabled) ). Default is 1. | ++----------------+-------------------------+---------------------------------------------------------------------------------------------+ | bodyForce | string | | Adds an acceleration vector to each cell of the domain. The only value allowed | | | | | is ``userdef``. The ``Gravity`` class then expects a user-defined bodyforce function to | | | | | be enrolled via ``Gavity::EnrollBodyForce(BodyForceFunc)`` (see :ref:`functionEnrollment`)| | | | | See the shearing box tests for examples of using bodyForce. | +----------------+-------------------------+---------------------------------------------------------------------------------------------+ +| skip | int | | Set the number of integration cycles between each computation of the gravity potential. | +| | | | Default is 1 (i.e. gravity is computed at every cycle). | ++----------------+-------------------------+---------------------------------------------------------------------------------------------+ + + + +.. _selfGravitySection: + +``SelfGravity`` section +----------------------- + +This section describes the method used to compute the self-gravitating potential :math:`\psi_{SG}`. More details on the algorithm may be found in the dedicated +:ref:`selfGravityModule` documentation. For this module to be used, self-gravity must be enabled as a source +of gravitational potential in the ``Gravity`` section (see :ref:`gravitySection` above). + ++----------------+-------------------------+---------------------------------------------------------------------------------------------+ +| Entry name | Parameter type | Comment | ++================+=========================+=============================================================================================+ +| solver | string | | Specifies which solver should be used. Can be ``Jacobi``, ``BICGSTAB`` or ``PBICGSTAB`` | +| | | | for the left preconditionned BICGSTAB solve. | ++----------------+-------------------------+---------------------------------------------------------------------------------------------+ +| targetError | real | | Set the error allowed in the residual :math:`r=\Delta\psi_G/(4\pi G_c)-\rho`. The error | +| | | | computation is based on a L2 norm. Default is 1e-2. | ++----------------+-------------------------+---------------------------------------------------------------------------------------------+ +| maxIter | int | | Set the maximum number of iterations allowed to the solver to reach convergence. Default | +| | | | is 1000. | ++----------------+-------------------------+---------------------------------------------------------------------------------------------+ +| boundary-Xn-dir| string | | Boundary condition applied to the potential field computed by self-gravity | +| | | | ``n`` can be 1, 2 or 3 and is the direction for the boundary condition. ``dir`` can be | +| | | | ``beg`` or ``end`` and indicates the side of the boundary. | +| | | | The boundary conditions allowed by the self-gravity solver are described in | +| | | | :ref:`selfGravityModule` | ++----------------+-------------------------+---------------------------------------------------------------------------------------------+ +| skip | int | | Set the number of integration cycles between each computation of self-gravity potential. | +| | | | Default is 1 (i.e. self-gravity is computed at every cycle). | ++----------------+-------------------------+---------------------------------------------------------------------------------------------+ + + ``RKL`` section ------------------ @@ -266,6 +323,8 @@ this block is simply ignored. +================+====================+===========================================================================================================+ | cfl | float | CFL number for the RKL sub-step. Should be <0.5 for stability. Set by default to 0.5 if not provided | +----------------+--------------------+-----------------------------------------------------------------------------------------------------------+ +| rmax_par | float | Maximum ratio between the hyperbolic timestep and the parabolic (RKL) timestep. Set to 100.0 by default. | ++----------------+--------------------+-----------------------------------------------------------------------------------------------------------+ | check_nan | bool | Whether RKL should check the solution when running. This option affects performances. Default false. | +----------------+--------------------+-----------------------------------------------------------------------------------------------------------+ @@ -314,9 +373,32 @@ This section describes the outputs *Idefix* produces. For more details about eac | dmp | float | | Time interval between dump outputs, in code units. | | | | | If negative, periodic dump outputs are disabled. | +----------------+-------------------------+--------------------------------------------------------------------------------------------------+ +| dmp_dir | string | | directory for dump file outputs. Default to "./" | +| | | | The directory is automatically created if it does not exist. | ++----------------+-------------------------+--------------------------------------------------------------------------------------------------+ | vtk | float | | Time interval between vtk outputs, in code units. | | | | | If negative, periodic vtk outputs are disabled. | +----------------+-------------------------+--------------------------------------------------------------------------------------------------+ +| vtk_dir | string | | directory for vtk file outputs. Default to "./" | +| | | | The directory is automatically created if it does not exist. | ++----------------+-------------------------+--------------------------------------------------------------------------------------------------+ +| vtk_sliceN | float, int, float, | | Create VTK files that contain a slice (cut or average) of the full domain. | +| | string | | the "N" of the entry name is an integer that identify each slice, starting from n=1 | +| | | | 1st parameter: Time interval between each slice vtk file | +| | | | 2nd parameter: plane of the slice. 0=(x2,x3) slice, 1=(x1,x3), 2=(x1,x2) | +| | | | 3rd parameter: localisation of the slice (when the slice is an average, this parameter only | +| | | | affect the localisation of the slice in the produced vtk file | +| | | | 4th parameter: slice type. Can be "cut" (for a slice of the full domain) or "average" (for an | +| | | | average along the direction given by the second parameter). NB: "average" performs a naive | +| | | | point average, without any consideration on the cell volumes/areas. | +| | | | NB2: this feature is in beta, and sometimes fail with some MPI implementations. | ++----------------+-------------------------+--------------------------------------------------------------------------------------------------+ +| xdmf | float | | Time interval between xdmf outputs, in code units (requires Idefix to be configured with HDF5) | +| | | | If negative, periodic xdmf outputs are disabled. | ++----------------+-------------------------+--------------------------------------------------------------------------------------------------+ +| xdmf_dir | string | | directory for xdmf file outputs. Default to "./" | +| | | | The directory is automatically created if it does not exist. | ++----------------+-------------------------+--------------------------------------------------------------------------------------------------+ | analysis | float | | Time interval between analysis outputs, in code units. | | | | | If negative, periodic analysis outputs are disabled. | | | | | When this entry is set, *Idefix* expects a user-defined analysis function to be | @@ -326,9 +408,31 @@ This section describes the outputs *Idefix* produces. For more details about eac | | | | When this list is present in the input file, *Idefix* expects a user-defined | | | | | function to be enrolled with ``Output::EnrollUserDefVariables(UserDefVariablesFunc)`` | | | | | (see :ref:`functionEnrollment`). The user-defined variables defined by this function | -| | | | are then written as new variables in vtk outputs. | +| | | | are then written as new variables in vtk and/or xdmf outputs. | +----------------+-------------------------+--------------------------------------------------------------------------------------------------+ .. note:: Even if dumps are not mentionned in your input file (and are therefore disabled), dump files are still produced when *Idefix* captures a signal (see :ref:`signalHandling`) or when ``max_runtime`` is set and reached. + + +.. _dustSection: + +``Dust`` section +---------------------- + +This section describes the dust fields computed using a zero pressure gas approximation (see :ref:`dustModule`). + ++----------------+-------------------------+---------------------------------------------------------------------------------------------+ +| Entry name | Parameter type | Comment | ++================+=========================+=============================================================================================+ +| nSpecies | integer | | Number of dust species to solve | ++----------------+-------------------------+---------------------------------------------------------------------------------------------+ +| drag | string, float, ... | | The first parameter describe the drag type. Possible values are: ``gamma``, ``tau``, | +| | | | ``size`` and ``userdef``. | +| | | | The remaining parameters give the drag parameter :math:`\beta_i` for each dust specie. | +| | | | (see :ref:`dustModule`). *Idefix* expects to have as many drag parameters as there are | +| | | | dust species. | ++----------------+-------------------------+---------------------------------------------------------------------------------------------+ +| drag_feedback | bool | | (optionnal) whether the gas feedback is enabled (default true). | ++----------------+-------------------------+---------------------------------------------------------------------------------------------+ diff --git a/doc/source/reference/makefile.rst b/doc/source/reference/makefile.rst index 7995e05b..8e53c224 100644 --- a/doc/source/reference/makefile.rst +++ b/doc/source/reference/makefile.rst @@ -40,6 +40,13 @@ Several options can be enabled from the command line (or are accessible with ``c Enable debug options in *Idefix*. This triggers a lot of outputs, and automatic bound checks of array accesses. As a result, this option makes the code very slow. +``-D Idefix_RUNTIME_CHECKS=ON`` + Include (potentially expensive) runtime sanity checks implemented with ``RUNTIME_CHECK_HOST`` and ``RUNTIME_CHECK_KERNEL``. + See :ref:`defensiveProgramming`. + +``-D Idefix_HDF5=ON`` + Enable HDF5 outputs. Requires the HDF5 library on the target system. Required for *Idefix* XDMF outputs. + ``-D Idefix_RECONSTRUCTION=x`` Specify the type of reconstruction scheme (replaces the old "ORDER" parameter in ``definitions.hpp``). Accepted values for ``x`` are: + ``Constant``: first order, donor cell reconstruction, @@ -88,6 +95,65 @@ Several options can be enabled from the command line (or are accessible with ``c option to explictely tell ``cmake`` a path to a build=*Idefix* problem directory. +.. _setupExamples: + +Configuration examples for selected clusters +++++++++++++++++++++++++++++++++++++++++++++ + + +AdAstra at CINES, AMD Mi250X GPUs +--------------------------------- + +We recommend the following modules and environement variables on AdAstra: + +.. code-block:: bash + + module load PrgEnv-cray-amd + module load cray-mpich + module load craype-network-ofi + module load cce + module load cpe + module load rocm/5.2.0 + export LDFLAGS="-L${ROCM_PATH}/lib -lamdhip64 -lstdc++fs" + +The last line being there to guarantee the link to the HIP library and the access to specific +C++17 functions. + +Finally, *Idefix* can be configured to run on Mi250 by enabling HIP and the desired architecture with the following options to ccmake: + +.. code-block:: bash + + -DKokkos_ENABLE_HIP=ON -DKokkos_ENABLE_HIP_MULTIPLE_KERNEL_INSTANTIATION=ON -DKokkos_ARCH_VEGA90A=ON`` + + +MPI (multi-GPU) can be enabled by adding ``-DIdefix_MPI=ON`` as usual. + +Jean Zay at IDRIS, Nvidia V100 and A100 GPUs +-------------------------------------------- + +We recommend the following modules and environement variables on Jean Zay: + +.. code-block:: bash + + module load cuda/12.1.0 + module load gcc/12.2.0 + module load openmpi/4.1.1-cuda + module load cmake/3.18.0 + +*Idefix* can then be configured to run on Nvidia V100 with the following options to ccmake: + +.. code-block:: bash + + -DKokkos_ENABLE_CUDA=ON -DKokkos_ENABLE_VOLTA70=ON + +While Ampere A100 GPUs are enabled with + +.. code-block:: bash + + -DKokkos_ENABLE_CUDA=ON -DKokkos_ENABLE_AMPERE80=ON + +MPI (multi-GPU) can be enabled by adding ``-DIdefix_MPI=ON`` as usual. + .. _setupSpecificOptions: Setup-specific options diff --git a/doc/source/reference/outputs.rst b/doc/source/reference/outputs.rst index 6fe2e64a..d531b497 100644 --- a/doc/source/reference/outputs.rst +++ b/doc/source/reference/outputs.rst @@ -7,7 +7,7 @@ Output formats -------------- *Idefix* uses several types of outputs you may want for your setup. By default, *Idefix* allows -for 3 kinds of outputs: +for 4 kinds of outputs: * logs which essentially tells the user what *Idefix* is currently doing. When running in serial, logs are sent to stdout, but when MPI is enabled, only the logs of the rank 0 process is sent to stdout, and each process (including rank 0) simultaneously writes a @@ -17,6 +17,8 @@ for 3 kinds of outputs: * VTK files (.vtk) are Visualation Toolkit files, which are easily readable by visualisation softwares such as `Paraview `_ or `Visit `_. A set of python methods is also provided to read vtk file from your python scripts in the `pytools` directory. +* XDMF files (eXtensible Data Model and Format) is a common format used in many HPC codes, which is easily readable by visualisation softwares such as `Paraview `_ + or `Visit `_. The XDMF format relies on the HDF5 format and therefore requires *Idefix* to be configured with HDF5 support. * user-defined analysis files. These are totally left to the user. They usually consist of ascii tables defined by the user, but they can be anything. diff --git a/doc/source/reference/setup.cpp.rst b/doc/source/reference/setup.cpp.rst index e7f4554a..5c0691db 100644 --- a/doc/source/reference/setup.cpp.rst +++ b/doc/source/reference/setup.cpp.rst @@ -48,7 +48,7 @@ the input file, and for user-defined outputs. This can be seen as a way to link setup to the main code at runtime, and avoid the need to pre-define tens of empty functions. Function enrollment is achieved by calling one of the ``EnrollXXX`` function of the class associated to it. -For instance, the ``Hydro`` class provide the following list of enrollment functions (declared in hydro.hpp): +For instance, the ``Fluid`` class provide the following list of enrollment functions (declared in hydro.hpp): .. code-block:: c++ @@ -72,22 +72,46 @@ For instance, the ``Hydro`` class provide the following list of enrollment funct void EnrollIsoSoundSpeed(IsoSoundSpeedFunc); When called, these function expects the address of the user-defined function. These user-defined -function should have the following signatures (declared in hydro_defs.hpp): +function should have the following signatures (declared in fluid_defs.hpp and boundary.hpp): .. code-block:: c++ - using UserDefBoundaryFunc = void (*) (DataBlock &, int dir, BoundarySide side, + using UserDefBoundaryFunc = void (*) (Fluid *, int dir, BoundarySide side, const real t); using GravPotentialFunc = void (*) (DataBlock &, const real t, IdefixArray1D&, IdefixArray1D&, IdefixArray1D&, IdefixArray3D &); - using SrcTermFunc = void (*) (DataBlock &, const real t, const real dt); - using InternalBoundaryFunc = void (*) (DataBlock &, const real t); + using SrcTermFunc = void (*) (Fluid *, const real t, const real dt); + using InternalBoundaryFunc = void (*) (Fluid*, const real t); using EmfBoundaryFunc = void (*) (DataBlock &, const real t); using DiffusivityFunc = void (*) (DataBlock &, const real t, IdefixArray3D &); using IsoSoundSpeedFunc = void (*) (DataBlock &, const real t, IdefixArray3D &); + +Note that some of these functions involve the template class ``Fluid``. The ``Fluid`` class +is indeed capable of handling several types of fluids (described by the template parameter ``Phys``): +MHD, HD, pressureless, etc... Hence, depending on the type of fluid to which the user-defined +function applies, the signature of the function would be different. For instance, a User-defined +boundary condition for a system that would solve for a gas+dust mixture would read + +.. code-block:: c++ + + void MyBoundary(Fluid * fluid, int dir, BoundarySide side, const real t) { + // Here comes the code for the Gas boundary condition + } + + void MyBoundaryDust(Fluid * fluid, int dir, BoundarySide side, const real t) { + // Here comes the code for the dust boundary condition + } + +Note that *Idefix* defines an alias for the default fluid which is often found in the example provided: + +.. code-block:: c++ + + using Hydro = Fluid; + + Example ******* @@ -114,7 +138,7 @@ constructor which reads a parameter from the .ini file and enroll the user-defin Mass = input.Get("Setup","mass",0); // Enroll the user-defined potential - data.hydro.EnrollGravPotential(&Potential); + data.gravity->EnrollGravPotential(&Potential); } @@ -126,18 +150,18 @@ User-defined boundaries If one (or several) boundaries are set to ``userdef`` in the input file, the user needs to enroll a user-defined boundary function in the ``Setup`` constructor as for the other user-def functions (see :ref:`functionEnrollment`). Note that even if several boundaries are ``userdef`` in the input file, only one user-defined function -is required. When *Idefix* calls the user defined boundary function, it sets the direction of the boundary (``dir=IDIR``, ``JDIR``, +is required per fluid type. When *Idefix* calls the user defined boundary function, it sets the direction of the boundary (``dir=IDIR``, ``JDIR``, or ``KDIR``) and the side of the bondary (``side=left`` or ``side=right``). If conveninent, one can use the ``BoundaryFor`` wrapper functions to automatically loop on the boundary specified by ``dir`` and ``side``. A typical user-defined boundary condition function looks like this: .. code-block:: c++ - void UserdefBoundary(DataBlock& data, int dir, BoundarySide side, real t) { - IdefixArray4D Vc = data.hydro.Vc; - IdefixArray4D Vs = data.hydro.Vs; + void UserdefBoundary(Hydro *hydro, int dir, BoundarySide side, real t) { + IdefixArray4D Vc = hydro->Vc; + IdefixArray4D Vs = hydro->Vs; if(dir==IDIR) { - data.hydro.boundary.BoundaryFor("UserDefBoundary", dir, side, + hydro->boundary->BoundaryFor("UserDefBoundary", dir, side, KOKKOS_LAMBDA (int k, int j, int i) { Vc(RHO,k,j,i) = 1.0; Vc(VX1,k,j,i) = 0.0; @@ -147,17 +171,23 @@ A typical user-defined boundary condition function looks like this: // For magnetic field (defined on cell sides), we need specific wrapper functions // Note that we don't need to initialise the field component parallel to dir, as it is // automatically reconstructed from the solenoidal condition and the tangential components - data.hydro.boundary.BoundaryForX2s("UserDefBoundaryBX2s", dir, side, + hydro->boundary->BoundaryForX2s("UserDefBoundaryBX2s", dir, side, KOKKOS_LAMBDA (int k, int j, int i) { Vs(BX2s,k,j,i) = 0.0; }); - data.hydro.boundary.BoundaryForX3s("UserDefBoundaryBX3s", dir, side, + hydro->boundary->BoundaryForX3s("UserDefBoundaryBX3s", dir, side, KOKKOS_LAMBDA (int k, int j, int i) { Vs(BX3s,k,j,i) = 0.0; }); } } +.. warning:: + + Only the tangential field components should be initialised by user-defined boundary conditions. + *Idefix* automatically reconstruct (and overwrite!) the normal field component from the + divergence-free condition on B and the user-defined tangential magnetic field components. + .. _setupInitflow: @@ -258,6 +288,36 @@ can be automatically derived using ``DataBlockHost::MakeVsFromAmag`` as in the e dataHost.SyncToDevice(); } +Initialising passive tracers +******************************** + +Idefix provides the possibility to use passive tracers, that are scalars passively advected by +the associated fluid. Tracers are enabled by setting on non-zero integer to the parameter ``tracer`` in +a ``[Hydro]`` (for hydro tracers) or a ``[Dust]`` block in your input file, so that the tracers +will follow the main fluid or the dust fluids, with the given number of tracers. + +Each tracer can be accessed in the dataBlockHost as a particular field in the array Vc as in the example below + +.. code-block:: c++ + + void Setup::InitFlow(DataBlock &data) { + // Create a host copy of the DataBlock given in argument + DataBlockHost dataHost(data); + + for(int k = 0; k < dataHost.np_tot[KDIR] ; k++) { + for(int j = 0; j < dataHost.np_tot[JDIR] ; j++) { + for(int i = 0; i < dataHost.np_tot[IDIR] ; i++) { + real x = dataHost.x[IDIR](i); + real y = dataHost.x[JDIR](j); + // First tracer + dataHost.Vc(TRG,k,j,i) = (x < 0 ? 0 : 1); // TRG for gas tracers + // second tracer + dataHost.Vc(TRG+1,k,j,i) = (y < 0 ? 0 : 1); // For gas tracers + +.. note:: + + Note that when using dust tracers, one should use the field ``TRD`` instead of ``TRG``. + .. _setupInitDump: Initialising from a restart dump @@ -270,18 +330,13 @@ a new initial condition by extrapolating or extanding a restart dump (such as in test or a dimension change). In this case, one should use the ``DumpImage`` class which provides all the tools needed to read a restart dump (see also :ref:`dumpImageClass`). -One typically first construct an instance of ``DumpImage`` in the ``Setup`` constructor, and then -use this instance to initialise the flow in ``Setup::InitFlow``. The procedure is examplified below, +One typically first construct an instance of ``DumpImage`` in ``Setup::InitFlow``, and then +use this instance to initialise the flow. The procedure is examplified below, assuming we want to create a dump from ``mydump.dmp``: .. code-block:: c++ - DumpImage *image; // Global pointer to our DumpImage - - // Setup constructor - Setup::Setup(Input &input, Grid &grid, DataBlock &data, Output &output) { - image = new DumpImage("mydump.dmp",output); // load the dump file and store it in a DumpImage - } + #include "dumpImage.hpp" // Flow initialisation, read directly from the DumpImage void Setup::InitFlow(DataBlock &data) { @@ -289,6 +344,8 @@ assuming we want to create a dump from ``mydump.dmp``: // Create a host copy DataBlockHost d(data); + DumpImage image("mydump.dmp", &data); + for(int k = d.beg[KDIR]; k < d.end[KDIR] ; k++) { for(int j = d.beg[JDIR]; j < d.end[JDIR] ; j++) { for(int i = d.beg[IDIR]; i < d.end[IDIR] ; i++) { @@ -299,9 +356,9 @@ assuming we want to create a dump from ``mydump.dmp``: int jglob=j-2*d.beg[JDIR]+d.gbeg[JDIR]; int kglob=k-2*d.beg[KDIR]+d.gbeg[KDIR]; - d.Vc(RHO,k,j,i) = image->arrays["Vc-RHO"](kglob,jglob,iglob); - d.Vc(PRS,k,j,i) = image->arrays["Vc-PRS"](kglob,jglob,iglob); - d.Vc(VX1,k,j,i) = image->arrays["Vc-VX1"](kglob,jglob,iglob); + d.Vc(RHO,k,j,i) = image.arrays["Vc-RHO"](kglob,jglob,iglob); + d.Vc(PRS,k,j,i) = image.arrays["Vc-PRS"](kglob,jglob,iglob); + d.Vc(VX1,k,j,i) = image.arrays["Vc-VX1"](kglob,jglob,iglob); }}} // For magnetic variable, we should fill the entire active domain, hence an additional diff --git a/make_tarballs.py b/make_tarballs.py index f062e749..dfc1b55f 100644 --- a/make_tarballs.py +++ b/make_tarballs.py @@ -17,10 +17,8 @@ KOKKOS_EXCLUDE_LIST = [ ".*", - "example", "appveyor.yml", "scripts", - "*test*", ] @@ -97,6 +95,14 @@ def _make_self_contained_tarball(output_dir, *, suffix="", exclude_list=None): def main(argv=None): parser = argparse.ArgumentParser() parser.add_argument("output_dir", default=".", nargs="?") + parser.add_argument( + "--allow-dirty", + action="store_true", + help=( + "allow to build tarballs with uncommited changes. " + "Useful to test this script" + ), + ) args = parser.parse_args(argv) with pushd(IDEFIX_DIR): @@ -105,7 +111,7 @@ def main(argv=None): # break if local copy is dirty stdout = subprocess.check_output(["git", "diff", "--name-only"]).decode() - if stdout.strip() != "": + if not args.allow_dirty and stdout.strip() != "": print( "ERROR: Local copy is dirty. Commit or stash changes before publishing.", file=sys.stderr, diff --git a/pytools/idfx_test.py b/pytools/idfx_test.py new file mode 100644 index 00000000..ce718f37 --- /dev/null +++ b/pytools/idfx_test.py @@ -0,0 +1,448 @@ +import argparse +import os +import shutil +import subprocess +import sys +import re + +import numpy as np +import matplotlib.pyplot as plt + +from .dump_io import readDump + +class bcolors: + HEADER = '\033[95m' + OKBLUE = '\033[94m' + OKCYAN = '\033[96m' + OKGREEN = '\033[92m' + WARNING = '\033[93m' + FAIL = '\033[91m' + ENDC = '\033[0m' + BOLD = '\033[1m' + UNDERLINE = '\033[4m' + +class idfxTest: + def __init__ (self): + parser = argparse.ArgumentParser() + + idefix_dir_env = os.getenv("IDEFIX_DIR") + + parser.add_argument("-noplot", + dest="noplot", + help="disable plotting in standard tests", + action="store_false") + + parser.add_argument("-ploterr", + help="Enable plotting on error in regression tests", + action="store_true") + + parser.add_argument("-cmake", + default=[], + help="CMake options", + nargs='+') + + parser.add_argument("-definitions", + default="", + help="definitions.hpp file") + + parser.add_argument("-dec", + default="", + help="MPI domain decomposition", + nargs='+') + + parser.add_argument("-check", + help="Only perform regression tests without compilation", + action="store_true") + + parser.add_argument("-cuda", + help="Test on Nvidia GPU using CUDA", + action="store_true") + + parser.add_argument("-hip", + help="Test on AMD GPU using HIP", + action="store_true") + + parser.add_argument("-single", + help="Enable single precision", + action="store_true") + + parser.add_argument("-vectPot", + help="Enable vector potential formulation", + action="store_true") + + parser.add_argument("-reconstruction", + type=int, + default=2, + help="set reconstruction scheme (2=PLM, 3=LimO3, 4=PPM)") + + parser.add_argument("-idefixDir", + default=idefix_dir_env, + help="Set directory for idefix source files (default $IDEFIX_DIR)") + + parser.add_argument("-mpi", + help="Enable MPI", + action="store_true") + + parser.add_argument("-all", + help="Do all test suite (otherwise, just do the test with the current configuration)", + action="store_true") + + parser.add_argument("-init", + help="Reinit reference files for non-regression tests (dangerous!)", + action="store_true") + + parser.add_argument("-Werror", + help="Consider warnings as errors", + action="store_true") + + + args, unknown=parser.parse_known_args() + + # transform all arguments from args into attributes of this instance + self.__dict__.update(vars(args)) + self.referenceDirectory = os.path.join(idefix_dir_env,"reference") + # current directory relative to $IDEFIX_DIR/test (used to retrieve the path ot reference files) + self.testDir=os.path.relpath(os.curdir,os.path.join(idefix_dir_env,"test")) + + def configure(self,definitionFile=""): + comm=["cmake"] + # add source directory + comm.append(self.idefixDir) + # add specific options + for opt in self.cmake: + comm.append("-D"+opt) + + if self.cuda: + comm.append("-DKokkos_ENABLE_CUDA=ON") + # disable fmad operations on Cuda to make it compatible with CPU arithmetics + comm.append("-DIdefix_CXX_FLAGS=--fmad=false") + + if self.hip: + comm.append("-DKokkos_ENABLE_HIP=ON") + # disable fmad operations on HIP to make it compatible with CPU arithmetics + comm.append("-DIdefix_CXX_FLAGS=-ffp-contract=off") + + #if we use single precision + if(self.single): + comm.append("-DIdefix_PRECISION=Single") + else: + comm.append("-DIdefix_PRECISION=Double") + + + if(self.vectPot): + comm.append("-DIdefix_EVOLVE_VECTOR_POTENTIAL=ON") + else: + comm.append("-DIdefix_EVOLVE_VECTOR_POTENTIAL=OFF") + + if(self.Werror): + comm.append("-DIdefix_WERROR=ON") + + # add a definition file if provided + if(definitionFile): + self.definitions=definitionFile + else: + self.definitions="definitions.hpp" + + comm.append("-DIdefix_DEFS="+self.definitions) + + if(self.mpi): + comm.append("-DIdefix_MPI=ON") + else: + comm.append("-DIdefix_MPI=OFF") + + if(self.reconstruction == 2): + comm.append("-DIdefix_RECONSTRUCTION=Linear") + elif(self.reconstruction==3): + comm.append("-DIdefix_RECONSTRUCTION=LimO3") + elif(self.reconstruction==4): + comm.append("-DIdefix_RECONSTRUCTION=Parabolic") + + + try: + cmake=subprocess.run(comm) + cmake.check_returncode() + except subprocess.CalledProcessError as e: + print(bcolors.FAIL+"***************************************************") + print("Cmake failed") + print("***************************************************"+bcolors.ENDC) + raise e + + def compile(self,jobs=8): + try: + make=subprocess.run(["make","-j"+str(jobs)]) + make.check_returncode() + except subprocess.CalledProcessError as e: + print(bcolors.FAIL+"***************************************************") + print("Compilation failed") + print("***************************************************"+bcolors.ENDC) + raise e + + def run(self, inputFile="", np=2, nowrite=False, restart=-1): + comm=["./idefix"] + if inputFile: + comm.append("-i") + comm.append(inputFile) + if self.mpi: + if self.dec: + np=1 + for n in range(len(self.dec)): + np=np*int(self.dec[n]) + + comm.insert(0,"mpirun") + comm.insert(1,"-np") + comm.insert(2,str(np)) + if self.dec: + comm.append("-dec") + for n in range(len(self.dec)): + comm.append(str(self.dec[n])) + + if nowrite: + comm.append("-nowrite") + + if self.Werror: + comm.append("-Werror") + + if restart>=0: + comm.append("-restart") + comm.append(str(restart)) + + try: + make=subprocess.run(comm) + make.check_returncode() + except subprocess.CalledProcessError as e: + print(bcolors.FAIL+"***************************************************") + print("Execution failed") + print("***************************************************"+bcolors.ENDC) + raise e + + self._readLog() + + def _readLog(self): + if not os.path.exists('./idefix.0.log'): + # When no idefix file is produced, we leave + return + + with open('./idefix.0.log','r') as file: + log = file.read() + + if "SINGLE PRECISION" in log: + self.single = True + else: + self.single = False + + if "Kokkos CUDA target ENABLED" in log: + self.cuda = True + else: + self.cuda = False + + if "Kokkos HIP target ENABLED" in log: + self.hip = True + else: + self.hip = False + + + self.reconstruction = 2 + if "3rd order (LimO3)" in log: + self.reconstruction = 3 + + if "4th order (PPM)" in log: + self.reconstruction = 4 + + self.mpi=False + if "MPI ENABLED" in log: + self.mpi=True + + + # Get input file from log + line = re.search('(?<=Input Parameters using input file )(.*)', log) + self.inifile=line.group(0)[:-1] + + # Get performances from log + line = re.search('Main: Perfs are (.*) cell', log) + self.perf=float(line.group(1)) + + def checkOnly(self, filename, tolerance=0): + # Assumes the code has been run manually using some configuration, so we simply + # do the test suite witout configure/compile/run + self._readLog() + self._showConfig() + if self.cuda or self.hip: + print(bcolors.WARNING+"***********************************************") + print("WARNING: Idefix guarantees floating point arithmetic accuracy") + print("ONLY when fmad instruction are explicitely disabled at compilation.") + print("Otheriwse, this test will likely fail.") + print("***********************************************"+bcolors.ENDC) + self.standardTest() + self.nonRegressionTest(filename, tolerance) + + def standardTest(self): + if os.path.exists(os.path.join('python', 'testidefix.py')): + os.chdir("python") + comm = [sys.executable, "testidefix.py"] + if self.noplot: + comm.append("-noplot") + + print(bcolors.OKCYAN+"Running standard test...") + try: + make=subprocess.run(comm) + make.check_returncode() + except subprocess.CalledProcessError as e: + print(bcolors.FAIL+"***************************************************") + print("Standard test execution failed") + print("***************************************************"+bcolors.ENDC) + raise e + print(bcolors.OKCYAN+"Standard test succeeded"+bcolors.ENDC) + os.chdir("..") + else: + print(bcolors.WARNING+"No standard testidefix.py for this test"+bcolors.ENDC) + sys.stdout.flush() + + def nonRegressionTest(self, filename,tolerance=0): + + fileref=os.path.join(self.referenceDirectory, self.testDir, self._getReferenceFilename()) + if not(os.path.exists(fileref)): + raise Exception("Reference file "+fileref+ " doesn't exist") + + filetest=filename + if not(os.path.exists(filetest)): + raise Exception("Test file "+filetest+ " doesn't exist") + + Vref=readDump(fileref) + Vtest=readDump(filetest) + error=self._computeError(Vref,Vtest) + if error > tolerance: + print(bcolors.FAIL+"Non-Regression test failed!") + self._showConfig() + print(bcolors.ENDC) + if self.ploterr: + self._plotDiff(Vref,Vtest) + assert error <= tolerance, bcolors.FAIL+"Error (%e) above tolerance (%e)"%(error,tolerance)+bcolors.ENDC + print(bcolors.OKGREEN+"Non-regression test succeeded with error=%e"%error+bcolors.ENDC) + sys.stdout.flush() + + def compareDump(self, file1, file2,tolerance=0): + Vref=readDump(file1) + Vtest=readDump(file2) + error=self._computeError(Vref,Vtest) + if error > tolerance: + print(bcolors.FAIL+"Files are different !") + print(bcolors.ENDC) + + self._plotDiff(Vref,Vtest) + assert error <= tolerance, bcolors.FAIL+"Error (%e) above tolerance (%e)"%(error,tolerance)+bcolors.ENDC + print(bcolors.OKGREEN+"Files are identical up to error=%e"%error+bcolors.ENDC) + sys.stdout.flush() + + + def makeReference(self,filename): + self._readLog() + targetDir = os.path.join(self.referenceDirectory,self.testDir) + if not os.path.exists(targetDir): + print("Creating reference directory") + os.makedirs(targetDir, exist_ok=True) + fileout = os.path.join(targetDir, self._getReferenceFilename()) + if(os.path.exists(fileout)): + ans=input(bcolors.WARNING+"This will overwrite already existing reference file:\n"+fileout+"\nDo you confirm? (type yes to continue): "+bcolors.ENDC) + if(ans != "yes"): + print(bcolors.WARNING+"Reference creation aborpted"+bcolors.ENDC) + return + + shutil.copy(filename,fileout) + print(bcolors.OKGREEN+"Reference file "+fileout+" created"+bcolors.ENDC) + sys.stdout.flush() + + def _showConfig(self): + print("**************************************************************") + if self.cuda: + print("Nvidia Cuda enabled.") + if self.hip: + print("AMD HIP enabled.") + print("CMake Opts: " +" ".join(self.cmake)) + print("Definitions file:"+self.definitions) + print("Input File: "+self.inifile) + if(self.single): + print("Precision: Single") + else: + print("Precision: Double") + if(self.reconstruction==2): + print("Reconstruction: PLM") + elif(self.reconstruction==3): + print("Reconstruction: LimO3") + elif(self.reconstruction==4): + print("Reconstruction: PPM") + if(self.vectPot): + print("Vector Potential: ON") + else: + print("Vector Potential: OFF") + if self.mpi: + print("MPI: ON") + else: + print("MPI: OFF") + + print("**************************************************************") + + def _getReferenceFilename(self): + strReconstruction="plm" + if self.reconstruction == 3: + strReconstruction = "limo3" + if self.reconstruction == 4: + strReconstruction= "ppm" + + strPrecision="double" + if self.single: + strPrecision="single" + + fileref='dump.ref.'+strPrecision+"."+strReconstruction+"."+self.inifile + if self.vectPot: + fileref=fileref+".vectPot" + + fileref=fileref+'.dmp' + return(fileref) + + def _computeError(self,Vref,Vtest): + ntested=0 + error=0 + for fld in Vtest.data.keys(): + if(Vtest.data[fld].ndim==3): + if fld in Vref.data.keys(): + #print("error in "+fld+" = "+str(np.sqrt(np.mean((Vref.data[fld]-Vtest.data[fld])**2)))) + error = error+np.sqrt(np.mean((Vref.data[fld]-Vtest.data[fld])**2)) + ntested=ntested+1 + + if ntested==0: + raise Exception(bcolors.FAIL+"There is no common field between the reference and current file"+bcolors.ENDC) + + error=error/ntested + return(error) + + def _plotDiff(self,Vref,Vtest): + + for fld in Vtest.data.keys(): + if(Vtest.data[fld].ndim==3): + if fld in Vref.data.keys(): + plt.figure() + plt.title(fld) + x1=Vref.x1 + if Vref.data[fld].shape[0] == Vref.x1.size+1: + x1=np.zeros(Vref.data[fld].shape[0]) + x1[:-1]=Vref.x1l + x1[-1]=Vref.x1r[-1] + x2=Vref.x2 + if Vref.data[fld].shape[1] == Vref.x2.size+1: + x2=np.zeros(Vref.data[fld].shape[1]) + x2[:-1]=Vref.x2l + x2[-1]=Vref.x2r[-1] + x3=Vref.x3 + if Vref.data[fld].shape[2] == Vref.x3.size+1: + x3=np.zeros(Vref.data[fld].shape[2]) + x3[:-1]=Vref.x3l + x3[-1]=Vref.x3r[-1] + if Vref.data[fld].shape[1]>1: + plt.pcolor(x1, x2, Vref.data[fld][:,:,0].T-Vtest.data[fld][:,:,0].T,cmap='seismic') + plt.xlabel("x1") + plt.ylabel("x2") + plt.colorbar() + else: + plt.plot(x1, Vref.data[fld][:,0,0]-Vtest.data[fld][:,0,0]) + plt.xlabel("x1") + plt.show() diff --git a/reference b/reference new file mode 160000 index 00000000..7f26b7ed --- /dev/null +++ b/reference @@ -0,0 +1 @@ +Subproject commit 7f26b7ed94671e875decbf3407f397bae674e4e5 diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 5ab15369..727f2150 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,5 +1,5 @@ -add_subdirectory(hydro) +add_subdirectory(fluid) add_subdirectory(dataBlock) add_subdirectory(output) add_subdirectory(rkl) diff --git a/src/dataBlock/CMakeLists.txt b/src/dataBlock/CMakeLists.txt index 67a06575..c3410f41 100644 --- a/src/dataBlock/CMakeLists.txt +++ b/src/dataBlock/CMakeLists.txt @@ -1,3 +1,5 @@ +add_subdirectory(planetarySystem) + target_sources(idefix PUBLIC ${CMAKE_CURRENT_LIST_DIR}/coarsen.cpp PUBLIC ${CMAKE_CURRENT_LIST_DIR}/dataBlock.cpp diff --git a/src/dataBlock/coarsen.cpp b/src/dataBlock/coarsen.cpp index 0ee7cd52..2da3049c 100644 --- a/src/dataBlock/coarsen.cpp +++ b/src/dataBlock/coarsen.cpp @@ -9,6 +9,7 @@ #include "../idefix.hpp" #include "dataBlock.hpp" #include "dataBlockHost.hpp" +#include "fluid.hpp" void DataBlock::Coarsen() { if(!haveGridCoarsening) { @@ -16,9 +17,9 @@ void DataBlock::Coarsen() { } ComputeGridCoarseningLevels(); // This routine coarsen the *conservative* variables - hydro.CoarsenFlow(hydro.Uc); + hydro->CoarsenFlow(hydro->Uc); #if MHD==YES - hydro.CoarsenMagField(hydro.Vs); + hydro->CoarsenMagField(hydro->Vs); #endif } diff --git a/src/dataBlock/dataBlock.cpp b/src/dataBlock/dataBlock.cpp index c11d5178..0414bb13 100644 --- a/src/dataBlock/dataBlock.cpp +++ b/src/dataBlock/dataBlock.cpp @@ -5,11 +5,20 @@ // Licensed under CeCILL 2.1 License, see COPYING for more information // *********************************************************************************** -#include "../idefix.hpp" +#include +#include "idefix.hpp" #include "dataBlock.hpp" +#include "fluid.hpp" +#include "gravity.hpp" +#include "planetarySystem.hpp" +#include "vtk.hpp" +#include "dump.hpp" +#ifdef WITH_HDF5 +#include "xdmf.hpp" +#endif -void DataBlock::InitFromGrid(Grid &grid, Input &input) { - idfx::pushRegion("DataBlock::InitFromGrid"); +DataBlock::DataBlock(Grid &grid, Input &input) { + idfx::pushRegion("DataBlock::DataBlock"); this->mygrid=&grid; @@ -17,18 +26,6 @@ void DataBlock::InitFromGrid(Grid &grid, Input &input) { GridHost gridHost(grid); gridHost.SyncFromDevice(); - nghost = std::vector(3); - np_int = std::vector(3); - np_tot = std::vector(3); - lbound = std::vector(3); - rbound = std::vector(3); - beg = std::vector(3); - end = std::vector(3); - gbeg = std::vector(3); - gend = std::vector(3); - - xbeg = std::vector(3); - xend = std::vector(3); // Get the number of points from the parent grid object for(int dir = 0 ; dir < 3 ; dir++) { @@ -70,23 +67,23 @@ void DataBlock::InitFromGrid(Grid &grid, Input &input) { std::string label; for(int dir = 0 ; dir < 3 ; dir++) { label = "DataBlock_x" + std::to_string(dir); - x.push_back(IdefixArray1D(label, np_tot[dir])); + x[dir] = IdefixArray1D(label, np_tot[dir]); label = "DataBlock_xr" + std::to_string(dir); - xr.push_back(IdefixArray1D(label,np_tot[dir])); + xr[dir] = IdefixArray1D(label,np_tot[dir]); label = "DataBlock_xl" + std::to_string(dir); - xl.push_back(IdefixArray1D(label,np_tot[dir])); + xl[dir] = IdefixArray1D(label,np_tot[dir]); label = "DataBlock_dx" + std::to_string(dir); - dx.push_back(IdefixArray1D(label,np_tot[dir])); + dx[dir] = IdefixArray1D(label,np_tot[dir]); label = "DataBlock_xgc" + std::to_string(dir); - xgc.push_back(IdefixArray1D(label,np_tot[dir])); + xgc[dir] = IdefixArray1D(label,np_tot[dir]); label = "DataBlock_A" + std::to_string(dir); - A.push_back(IdefixArray3D(label, - np_tot[KDIR]+KOFFSET, np_tot[JDIR]+JOFFSET, np_tot[IDIR]+IOFFSET)); + A[dir] = IdefixArray3D(label, + np_tot[KDIR]+KOFFSET, np_tot[JDIR]+JOFFSET, np_tot[IDIR]+IOFFSET); } dV = IdefixArray3D("DataBlock_dV",np_tot[KDIR],np_tot[JDIR],np_tot[IDIR]); @@ -100,59 +97,10 @@ void DataBlock::InitFromGrid(Grid &grid, Input &input) { dmu = IdefixArray1D("DataBlock_dmu",np_tot[JDIR]); #endif + // Initialize our sub-domain + this->ExtractSubdomain(); - - // Copy the relevant part of the coordinate system to the datablock - for(int dir = 0 ; dir < 3 ; dir++) { - int offset=gbeg[dir]-beg[dir]; - - IdefixArray1D x_input = grid.x[dir]; - IdefixArray1D x_output= x[dir]; - IdefixArray1D xr_input = grid.xr[dir]; - IdefixArray1D xr_output= xr[dir]; - IdefixArray1D xl_input = grid.xl[dir]; - IdefixArray1D xl_output= xl[dir]; - IdefixArray1D dx_input = grid.dx[dir]; - IdefixArray1D dx_output= dx[dir]; - - idefix_for("coordinates",0,np_tot[dir], - KOKKOS_LAMBDA (int i) { - x_output(i) = x_input(i+offset); - xr_output(i) = xr_input(i+offset); - xl_output(i) = xl_input(i+offset); - dx_output(i) = dx_input(i+offset); - } - ); - } - - // Initialize grid coarsening if needed - if(grid.haveGridCoarsening != GridCoarsening::disabled) { - this->haveGridCoarsening = grid.haveGridCoarsening; - this->coarseningDirection = grid.coarseningDirection; - this->coarseningLevel = std::vector>(3); - - for(int dir = 0 ; dir < 3 ; dir++) { - if(coarseningDirection[dir]) { - const int Xt = (dir == IDIR ? JDIR : IDIR); - const int Xb = (dir == KDIR ? JDIR : KDIR); - - // Allocate coarsening level arrays - coarseningLevel[dir] = IdefixArray2D( - "DataBlock_corseLevel", - np_tot[Xb], - np_tot[Xt]); - // Make a local reference - IdefixArray2D coarseInit = coarseningLevel[dir]; - // Init coarsening level array to one everywhere - idefix_for("init_coarsening", 0, np_tot[Xb], 0, np_tot[Xt], - KOKKOS_LAMBDA(int j, int i) { - coarseInit(j,i) = 1; - }); - } - } - } - - // Iniaitlize the geometry + // Initialize the geometry this->MakeGeometry(); // Initialise the state containers @@ -161,38 +109,189 @@ void DataBlock::InitFromGrid(Grid &grid, Input &input) { this->states["current"] = StateContainer(); + // Initialize the Dump object + this->dump = std::make_unique(input, this); + + // Initialize the VTK object + this->vtk = std::make_unique(input, this, "data"); + + // Init XDMF objects for HDF5 outputs + #ifdef WITH_HDF5 + this->xdmf= std::make_unique(input,this); + #endif + + // Initialize the hydro object attached to this datablock - this->hydro.Init(input, grid, this); + this->hydro = std::make_unique>(grid, input, this); // Initialise Fargo if needed if(input.CheckBlock("Fargo")) { - fargo.Init(input, this); + this->fargo = std::make_unique(input, DefaultPhysics::nvar, this); this->haveFargo = true; } - // Initialise gravity if needed - if(input.CheckBlock("Gravity")) { - gravity.Init(input, this); + // initialise planets if needed + if(input.CheckBlock("Planet")) { + this->planetarySystem = std::make_unique(input, this); + this->haveplanetarySystem = true; + } + + // Initialise gravity if needed (automatically if planets are present) + if(input.CheckBlock("Gravity") || haveplanetarySystem) { + this->gravity = std::make_unique(input, this); this->haveGravity = true; } + // Initialise dust grains if needed + if(input.CheckBlock("Dust")) { + haveDust = true; + int nSpecies = input.Get("Dust","nSpecies",0); + for(int i = 0 ; i < nSpecies ; i++) { + dust.emplace_back(std::make_unique>(grid, input, this, i)); + } + } + // Register variables that need to be saved in case of restart dump + dump->RegisterVariable(&t, "time"); + dump->RegisterVariable(&dt, "dt"); + + idfx::popRegion(); +} + +/** + * @brief Construct a new Data Block as a subgrid + * + * @param subgrid : subgrid from which the local datablock should be extracted from + */ +DataBlock::DataBlock(SubGrid *subgrid) { + idfx::pushRegion("DataBlock:DataBlock(SubGrid)"); + Grid *grid = subgrid->parentGrid; + this->mygrid = subgrid->grid.get(); + + // Make a local copy of the grid for future usage. + GridHost gridHost(*grid); + gridHost.SyncFromDevice(); + + + // Get the number of points from the parent grid object + for(int dir = 0 ; dir < 3 ; dir++) { + nghost[dir] = grid->nghost[dir]; + // Domain decomposition: decompose the full domain size in grid by the number of processes + // in that direction + np_int[dir] = grid->np_int[dir]/grid->nproc[dir]; + np_tot[dir] = np_int[dir]+2*nghost[dir]; + + // Boundary conditions + if (grid->xproc[dir]==0) { + lbound[dir] = grid->lbound[dir]; + if(lbound[dir]==axis) this->haveAxis = true; + } else { + lbound[dir] = internal; + } + + if (grid->xproc[dir] == grid->nproc[dir]-1) { + rbound[dir] = grid->rbound[dir]; + if(rbound[dir]==axis) this->haveAxis = true; + } else { + rbound[dir] = internal; + } + + beg[dir] = grid->nghost[dir]; + end[dir] = grid->nghost[dir]+np_int[dir]; + + // Where does this datablock starts and end in the grid? + // This assumes even distribution of points between procs + gbeg[dir] = grid->nghost[dir] + grid->xproc[dir]*np_int[dir]; + gend[dir] = grid->nghost[dir] + (grid->xproc[dir]+1)*np_int[dir]; + + // Local start and end of current datablock + xbeg[dir] = gridHost.xl[dir](gbeg[dir]); + xend[dir] = gridHost.xr[dir](gend[dir]-1); + } + + // Next we erase the info the slice direction + int refdir = subgrid->direction; + nghost[refdir] = 0; + np_int[refdir] = 1; + np_tot[refdir] = 1; + beg[refdir] = 0; + end[refdir] = 1; + gbeg[refdir] = subgrid->index; + gend[refdir] = subgrid->index+1; + xbeg[refdir] = gridHost.x[refdir](subgrid->index); + xend[refdir] = gridHost.x[refdir](subgrid->index); + + // Allocate the required fields (only a limited set for a datablock from a subgrid) + std::string label; + for(int dir = 0 ; dir < 3 ; dir++) { + label = "DataBlock_x" + std::to_string(dir); + x[dir] = IdefixArray1D(label, np_tot[dir]); + + label = "DataBlock_xr" + std::to_string(dir); + xr[dir] = IdefixArray1D(label,np_tot[dir]); + + label = "DataBlock_xl" + std::to_string(dir); + xl[dir] = IdefixArray1D(label,np_tot[dir]); + + label = "DataBlock_dx" + std::to_string(dir); + dx[dir] = IdefixArray1D(label,np_tot[dir]); + } + + // Initialize our sub-domain + this->ExtractSubdomain(); + + // Reset gbeg/gend so that they don't refer anymore to the parent grid + gbeg[refdir] = 0; + gend[refdir] = 1; idfx::popRegion(); } void DataBlock::ResetStage() { - this->hydro.ResetStage(); + this->hydro->ResetStage(); + if(haveDust) { + for(int i = 0 ; i < dust.size() ; i++) { + dust[i]->ResetStage(); + } + } +} + +void DataBlock::ConsToPrim() { + this->hydro->ConvertConsToPrim(); + if(haveDust) { + for(int i = 0 ; i < dust.size() ; i++) { + dust[i]->ConvertConsToPrim(); + } + } +} + +void DataBlock::PrimToCons() { + this->hydro->ConvertPrimToCons(); + if(haveDust) { + for(int i = 0 ; i < dust.size() ; i++) { + dust[i]->ConvertPrimToCons(); + } + } } // Set the boundaries of the data structures in this datablock void DataBlock::SetBoundaries() { if(haveGridCoarsening) { ComputeGridCoarseningLevels(); - hydro.CoarsenFlow(hydro.Vc); + hydro->CoarsenFlow(hydro->Vc); #if MHD==YES - hydro.CoarsenMagField(hydro.Vs); + hydro->CoarsenMagField(hydro->Vs); #endif + if(haveDust) { + for(int i = 0 ; i < dust.size() ; i++) { + dust[i]->CoarsenFlow(dust[i]->Vc); + } + } } - hydro.boundary.SetBoundaries(t); + if(haveDust) { + for(int i = 0 ; i < dust.size() ; i++) { + dust[i]->boundary->SetBoundaries(t); + } + } + hydro->boundary->SetBoundaries(t); } @@ -206,9 +305,23 @@ void DataBlock::ShowConfig() { << "...." << xend[dir] << std::endl; } } - hydro.ShowConfig(); - if(haveFargo) fargo.ShowConfig(); - if(haveGravity) gravity.ShowConfig(); + hydro->ShowConfig(); + if(haveFargo) fargo->ShowConfig(); + if(haveplanetarySystem) planetarySystem->ShowConfig(); + if(haveGravity) gravity->ShowConfig(); + if(haveUserStepFirst) idfx::cout << "DataBlock: User's first step has been enrolled." + << std::endl; + if(haveUserStepLast) idfx::cout << "DataBlock: User's last step has been enrolled." + << std::endl; + if(haveDust) { + idfx::cout << "DataBlock: evolving " << dust.size() << " dust species." << std::endl; + // Only show the config the first dust specie + dust[0]->ShowConfig(); + /* + for(int i = 0 ; i < dust.size() ; i++) { + dust[i]->ShowConfig(); + }*/ + } } @@ -216,7 +329,7 @@ real DataBlock::ComputeTimestep() { // Compute the timestep using all of the enabled modules in the current dataBlock // First with the hydro block - auto InvDt = hydro.InvDt; + auto InvDt = hydro->InvDt; real dt; idefix_reduce("Timestep_reduction", beg[KDIR], end[KDIR], @@ -226,6 +339,62 @@ real DataBlock::ComputeTimestep() { dtmin=FMIN(ONE_F/InvDt(k,j,i),dtmin); }, Kokkos::Min(dt)); + if(haveDust) { + for(int n = 0 ; n < dust.size() ; n++) { + real dtDust; + auto InvDt = dust[n]->InvDt; + idefix_reduce("Timestep_reduction_dust", + beg[KDIR], end[KDIR], + beg[JDIR], end[JDIR], + beg[IDIR], end[IDIR], + KOKKOS_LAMBDA (int k, int j, int i, real &dtmin) { + dtmin=FMIN(ONE_F/InvDt(k,j,i),dtmin); + }, + Kokkos::Min(dtDust)); + dt = std::min(dt,dtDust); + } + } Kokkos::fence(); return(dt); } + +// Recompute magnetic fields from vector potential in dedicated fluids +void DataBlock::DeriveVectorPotential() { + if constexpr(DefaultPhysics::mhd) { + #ifdef EVOLVE_VECTOR_POTENTIAL + hydro->emf->ComputeMagFieldFromA(hydro->Ve, hydro->Vs); + #endif + } +} + +void DataBlock::LaunchUserStepLast() { + if(haveUserStepLast) { + idfx::pushRegion("User::UserStepLast"); + if(userStepLast != nullptr) + userStepLast(*this, this->t, this->dt); + else + IDEFIX_ERROR("UserStepLast not properly initialized"); + idfx::popRegion(); + } +} + +void DataBlock::LaunchUserStepFirst() { + if(haveUserStepFirst) { + idfx::pushRegion("User::UserStepFirst"); + if(userStepFirst != nullptr) + userStepFirst(*this, this->t, this->dt); + else + IDEFIX_ERROR("UserStepLast not properly initialized"); + idfx::popRegion(); + } +} + +void DataBlock::EnrollUserStepLast(StepFunc func) { + haveUserStepLast = true; + userStepLast = func; +} + +void DataBlock::EnrollUserStepFirst(StepFunc func) { + haveUserStepFirst = true; + userStepFirst = func; +} diff --git a/src/dataBlock/dataBlock.hpp b/src/dataBlock/dataBlock.hpp index 01c9a166..12f0847e 100644 --- a/src/dataBlock/dataBlock.hpp +++ b/src/dataBlock/dataBlock.hpp @@ -11,11 +11,12 @@ #include #include #include +#include + #include "idefix.hpp" #include "grid.hpp" #include "gridHost.hpp" -#include "hydro.hpp" -#include "fargo.hpp" +#include "planetarySystem.hpp" #include "gravity.hpp" #include "stateContainer.hpp" @@ -30,17 +31,33 @@ // forward class declaration (used by enrollment functions) class DataBlock; +class Vtk; +class Dump; +class Xdmf; +class Fargo; +class Gravity; +class PlanetarySystem; +template +class Fluid; +class SubGrid; +class Vtk; +class Dump; + +// Forward class hydro declaration +#include "physics.hpp" using GridCoarseningFunc = void(*) (DataBlock &); +using StepFunc = void (*) (DataBlock &, const real t, const real dt); + class DataBlock { public: // Local grid information - std::vector> x; ///< geometrical central points - std::vector> xr; ///< cell right interface - std::vector> xl; ///< cell left interface - std::vector> dx; ///< cell width - std::vector> xgc; ///< cell geometrical cell center + std::array,3> x; ///< geometrical central points + std::array,3> xr; ///< cell right interface + std::array,3> xl; ///< cell left interface + std::array,3> dx; ///< cell width + std::array,3> xgc; ///< cell geometrical cell center IdefixArray1D rt; ///< In spherical coordinates, gives $\tilde{r}$ IdefixArray1D sinx2m; ///< In spherical coordinates, ///< gives sin(th) at a j-1/2 interface @@ -51,23 +68,23 @@ class DataBlock { IdefixArray1D dmu; ///< In spherical coordinates, ///< gives the $\theta$ volume = fabs(cos(th_m) - cos(th_p)) - std::vector> coarseningLevel; ///< Grid coarsening levels + std::array,3> coarseningLevel; ///< Grid coarsening levels ///< (only defined when coarsening ///< is enabled) - std::vector coarseningDirection; ///< whether a coarsening is used in each direction + std::array coarseningDirection; ///< whether a coarsening is used in each direction - std::vector xbeg; ///< Beginning of active domain in datablock - std::vector xend; ///< End of active domain in datablock + std::array xbeg; ///< Beginning of active domain in datablock + std::array xend; ///< End of active domain in datablock IdefixArray3D dV; ///< cell volume - std::vector> A; ///< cell right interface area + std::array,3> A; ///< cell left interface area - std::vector np_tot; ///< total number of grid points in datablock - std::vector np_int; ///< active number of grid points in datablock (excl. ghost cells) + std::array np_tot; ///< total number of grid points in datablock + std::array np_int; ///< active number of grid points in datablock (excl. ghost cells) - std::vector nghost; ///< number of ghost cells at each boundary - std::vector lbound; ///< Boundary condition to the left - std::vector rbound; ///< Boundary condition to the right + std::array nghost; ///< number of ghost cells at each boundary + std::array lbound; ///< Boundary condition to the left + std::array rbound; ///< Boundary condition to the right bool haveAxis{false}; ///< DataBlock contains points on the axis and a special treatment ///< has been required for these. @@ -79,11 +96,11 @@ class DataBlock { - std::vector beg; ///< First local index of the active domain - std::vector end; ///< Last local index of the active domain+1 + std::array beg; ///< First local index of the active domain + std::array end; ///< Last local index of the active domain+1 - std::vector gbeg; ///< First global index of the active domain of this datablock - std::vector gend; ///< Last global index of the active domain of this datablock + std::array gbeg; ///< First global index of the active domain of this datablock + std::array gend; ///< Last global index of the active domain of this datablock real dt; ///< Current timestep real t; ///< Current time @@ -94,18 +111,39 @@ class DataBlock { ///< conservative state of the datablock ///< (contains references to dedicated objects) - Hydro hydro; ///< The Hydro object attached to this datablock + std::unique_ptr> hydro; ///< The Hydro object attached to this datablock + bool haveDust{false}; + std::vector>> dust; ///< Holder for zero pressure dust fluid + + std::unique_ptr vtk; + std::unique_ptr dump; + #ifdef WITH_HDF5 + std::unique_ptr xdmf; + #endif - void InitFromGrid(Grid &, Input &); ///< init from a Grid object - void MakeGeometry(); ///< Compute geometrical terms + + DataBlock(Grid &, Input &); ///< init from a Grid object + explicit DataBlock(SubGrid *); ///< init a minimal datablock for a subgrid + + void ExtractSubdomain(); ///< initialise datablock sub-domain according to domain decomp. + void MakeGeometry(); ///< Compute geometrical terms void DumpToFile(std::string); ///< Dump current datablock to a file for inspection void Validate(); ///< error out early in case problems are found in IC int CheckNan(); ///< Return the number of cells which have Nans + // The Planetary system + bool haveplanetarySystem{false}; + std::unique_ptr planetarySystem; + + bool rklCycle{false}; ///< // Set to true when we're inside a RKL call void EvolveStage(); ///< Evolve this DataBlock by dt + void EvolveRKLStage(); ///< Evolve this DataBlock by dt for terms impacted by RKL void SetBoundaries(); ///< Enforce boundary conditions to this datablock + void ConsToPrim(); ///< Convert conservative to primitive variables + void PrimToCons(); ///< Convert primitive to conservative variables + void DeriveVectorPotential(); ///< Compute magnetic fields from vector potential where applicable void Coarsen(); ///< Coarsen this datablock and its objects void ShowConfig(); ///< Show the datablock's configuration real ComputeTimestep(); ///< compute maximum timestep from current state of affairs @@ -115,21 +153,32 @@ class DataBlock { void EnrollGridCoarseningLevels(GridCoarseningFunc); ///< Enroll a user function to compute coarsening levels void CheckCoarseningLevels(); ///< Check that coarsening levels satisfy requirements - DataBlock() = default; // Do we use fargo-like scheme ? (orbital advection) bool haveFargo{false}; - Fargo fargo; + std::unique_ptr fargo; // Do we have Gravity ? bool haveGravity{false}; - Gravity gravity; + std::unique_ptr gravity; + + // User step functions (before or after the main integrator step) + void LaunchUserStepFirst(); ///< perform user-defined step before main integration step + void LaunchUserStepLast(); ///< Perform user-defined step after main integration step + + void EnrollUserStepFirst(StepFunc); + void EnrollUserStepLast(StepFunc); private: void WriteVariable(FILE* , int , int *, char *, void*); - - template void LoopDir(); ///< // recursive loop on dimensions void ComputeGridCoarseningLevels(); ///< Call user defined function to define Coarsening levels + + // User Steps (either before or after the main integration loop) + bool haveUserStepFirst{false}; + bool haveUserStepLast{false}; + + StepFunc userStepFirst{nullptr}; + StepFunc userStepLast{nullptr}; }; #endif // DATABLOCK_DATABLOCK_HPP_ diff --git a/src/dataBlock/dataBlockHost.cpp b/src/dataBlock/dataBlockHost.cpp index 2add0039..ca26e5cb 100644 --- a/src/dataBlock/dataBlockHost.cpp +++ b/src/dataBlock/dataBlockHost.cpp @@ -7,6 +7,7 @@ #include "idefix.hpp" #include "dataBlockHost.hpp" +#include "fluid.hpp" DataBlockHost::DataBlockHost(DataBlock& datain) { idfx::pushRegion("DataBlockHost::DataBlockHost(DataBlock)"); @@ -19,11 +20,11 @@ DataBlockHost::DataBlockHost(DataBlock& datain) { // Create mirrors (should be mirror_view) for(int dir = 0 ; dir < 3 ; dir++) { - x.push_back(Kokkos::create_mirror_view(data->x[dir])); - xr.push_back(Kokkos::create_mirror_view(data->xr[dir])); - xl.push_back(Kokkos::create_mirror_view(data->xl[dir])); - dx.push_back(Kokkos::create_mirror_view(data->dx[dir])); - A.push_back(Kokkos::create_mirror_view(data->A[dir])); + x[dir] = Kokkos::create_mirror_view(data->x[dir]); + xr[dir] = Kokkos::create_mirror_view(data->xr[dir]); + xl[dir] = Kokkos::create_mirror_view(data->xl[dir]); + dx[dir] = Kokkos::create_mirror_view(data->dx[dir]); + A[dir] = Kokkos::create_mirror_view(data->A[dir]); } np_tot = data->np_tot; @@ -38,35 +39,41 @@ DataBlockHost::DataBlockHost(DataBlock& datain) { end = data->end; gbeg = data->gbeg; gend = data->gend; + haveDust = data->haveDust; // TO BE COMPLETED... dV = Kokkos::create_mirror_view(data->dV); - Vc = Kokkos::create_mirror_view(data->hydro.Vc); - Uc = Kokkos::create_mirror_view(data->hydro.Uc); - InvDt = Kokkos::create_mirror_view(data->hydro.InvDt); + Vc = Kokkos::create_mirror_view(data->hydro->Vc); + Uc = Kokkos::create_mirror_view(data->hydro->Uc); + InvDt = Kokkos::create_mirror_view(data->hydro->InvDt); #if MHD == YES - Vs = Kokkos::create_mirror_view(data->hydro.Vs); - this->haveCurrent = data->hydro.haveCurrent; - if(data->hydro.haveCurrent) { - J = Kokkos::create_mirror_view(data->hydro.J); + Vs = Kokkos::create_mirror_view(data->hydro->Vs); + this->haveCurrent = data->hydro->haveCurrent; + if(data->hydro->haveCurrent) { + J = Kokkos::create_mirror_view(data->hydro->J); } #ifdef EVOLVE_VECTOR_POTENTIAL - Ve = Kokkos::create_mirror_view(data->hydro.Ve); + Ve = Kokkos::create_mirror_view(data->hydro->Ve); #endif - D_EXPAND( Ex3 = Kokkos::create_mirror_view(data->hydro.emf.ez); , + D_EXPAND( Ex3 = Kokkos::create_mirror_view(data->hydro->emf->ez); , , - Ex1 = Kokkos::create_mirror_view(data->hydro.emf.ex); - Ex2 = Kokkos::create_mirror_view(data->hydro.emf.ey); ) + Ex1 = Kokkos::create_mirror_view(data->hydro->emf->ex); + Ex2 = Kokkos::create_mirror_view(data->hydro->emf->ey); ) #endif + if(haveDust) { + dustVc = std::vector>(data->dust.size()); + for(int i = 0 ; i < data->dust.size() ; i++) { + dustVc[i] = Kokkos::create_mirror_view(data->dust[i]->Vc); + } + } // if grid coarsening is enabled if(data->haveGridCoarsening) { this->haveGridCoarsening = data->haveGridCoarsening; this->coarseningDirection = data->coarseningDirection; - this->coarseningLevel = std::vector::HostMirror>(3); for(int dir = 0 ; dir < 3 ; dir++) { if(coarseningDirection[dir]) { coarseningLevel[dir] = Kokkos::create_mirror_view(data->coarseningLevel[dir]); @@ -84,6 +91,9 @@ DataBlockHost::DataBlockHost(DataBlock& datain) { Kokkos::deep_copy(dV,data->dV); + this->haveplanetarySystem = data->haveplanetarySystem; + this->planetarySystem = data->planetarySystem.get(); + idfx::popRegion(); } @@ -91,23 +101,28 @@ DataBlockHost::DataBlockHost(DataBlock& datain) { void DataBlockHost::SyncToDevice() { idfx::pushRegion("DataBlockHost::SyncToDevice()"); - Kokkos::deep_copy(data->hydro.Vc,Vc); - Kokkos::deep_copy(data->hydro.InvDt,InvDt); + Kokkos::deep_copy(data->hydro->Vc,Vc); + Kokkos::deep_copy(data->hydro->InvDt,InvDt); #if MHD == YES - Kokkos::deep_copy(data->hydro.Vs,Vs); - if(this->haveCurrent && data->hydro.haveCurrent) Kokkos::deep_copy(data->hydro.J,J); + Kokkos::deep_copy(data->hydro->Vs,Vs); + if(this->haveCurrent && data->hydro->haveCurrent) Kokkos::deep_copy(data->hydro->J,J); #ifdef EVOLVE_VECTOR_POTENTIAL - Kokkos::deep_copy(data->hydro.Ve,Ve); + Kokkos::deep_copy(data->hydro->Ve,Ve); #endif - D_EXPAND( Kokkos::deep_copy(data->hydro.emf.ez,Ex3); , + D_EXPAND( Kokkos::deep_copy(data->hydro->emf->ez,Ex3); , , - Kokkos::deep_copy(data->hydro.emf.ex,Ex1); - Kokkos::deep_copy(data->hydro.emf.ey,Ex2); ) + Kokkos::deep_copy(data->hydro->emf->ex,Ex1); + Kokkos::deep_copy(data->hydro->emf->ey,Ex2); ) #endif + if(haveDust) { + for(int i = 0 ; i < dustVc.size() ; i++) { + Kokkos::deep_copy(data->dust[i]->Vc, dustVc[i]); + } + } - Kokkos::deep_copy(data->hydro.Uc,Uc); + Kokkos::deep_copy(data->hydro->Uc,Uc); if(haveGridCoarsening) { for(int dir = 0 ; dir < 3 ; dir++) { @@ -122,22 +137,28 @@ void DataBlockHost::SyncToDevice() { void DataBlockHost::SyncFromDevice() { idfx::pushRegion("DataBlockHost::SyncFromDevice()"); - Kokkos::deep_copy(Vc,data->hydro.Vc); - Kokkos::deep_copy(InvDt,data->hydro.InvDt); + Kokkos::deep_copy(Vc,data->hydro->Vc); + Kokkos::deep_copy(InvDt,data->hydro->InvDt); #if MHD == YES - Kokkos::deep_copy(Vs,data->hydro.Vs); - if(this->haveCurrent && data->hydro.haveCurrent) Kokkos::deep_copy(J,data->hydro.J); + Kokkos::deep_copy(Vs,data->hydro->Vs); + if(this->haveCurrent && data->hydro->haveCurrent) Kokkos::deep_copy(J,data->hydro->J); #ifdef EVOLVE_VECTOR_POTENTIAL - Kokkos::deep_copy(Ve,data->hydro.Ve); + Kokkos::deep_copy(Ve,data->hydro->Ve); #endif - D_EXPAND( Kokkos::deep_copy(Ex3,data->hydro.emf.ez); , + D_EXPAND( Kokkos::deep_copy(Ex3,data->hydro->emf->ez); , , - Kokkos::deep_copy(Ex1,data->hydro.emf.ex); - Kokkos::deep_copy(Ex2,data->hydro.emf.ey); ) + Kokkos::deep_copy(Ex1,data->hydro->emf->ex); + Kokkos::deep_copy(Ex2,data->hydro->emf->ey); ) #endif - Kokkos::deep_copy(Uc,data->hydro.Uc); + Kokkos::deep_copy(Uc,data->hydro->Uc); + + if(haveDust) { + for(int i = 0 ; i < dustVc.size() ; i++) { + Kokkos::deep_copy(dustVc[i], data->dust[i]->Vc); + } + } if(haveGridCoarsening) { for(int dir = 0 ; dir < 3 ; dir++) { diff --git a/src/dataBlock/dataBlockHost.hpp b/src/dataBlock/dataBlockHost.hpp index 580c29d5..86c8c570 100644 --- a/src/dataBlock/dataBlockHost.hpp +++ b/src/dataBlock/dataBlockHost.hpp @@ -25,15 +25,19 @@ class DataBlockHost { public: // Local grid information - std::vector::HostMirror> x; ///< geometrical central points - std::vector::HostMirror> xr; ///< cell right interface - std::vector::HostMirror> xl; ///< cell left interface - std::vector::HostMirror> dx; ///< cell width + std::array::HostMirror,3> x; ///< geometrical central points + std::array::HostMirror,3> xr; ///< cell right interface + std::array::HostMirror,3> xl; ///< cell left interface + std::array::HostMirror,3> dx; ///< cell width IdefixArray3D::HostMirror dV; ///< cell volume - std::vector::HostMirror> A; ///< cell right interface area + std::array::HostMirror,3> A; ///< cell right interface area IdefixArray4D::HostMirror Vc; ///< Main cell-centered primitive variables index + + bool haveDust{false}; + std::vector> dustVc; ///< Cell-centered primitive variables index for dust + #if MHD == YES IdefixArray4D::HostMirror Vs; ///< Main face-centered primitive variables index IdefixArray4D::HostMirror Ve; ///< Main edge-centered primitive variables index @@ -47,35 +51,40 @@ class DataBlockHost { IdefixArray4D::HostMirror Uc; ///< Main cell-centered conservative variables IdefixArray3D::HostMirror InvDt; ///< Inverse of maximum timestep in each cell - std::vector::HostMirror> coarseningLevel; ///< Grid coarsening level + std::array::HostMirror,3> coarseningLevel; ///< Grid coarsening level ///< (only defined when coarsening ///< is enabled) - std::vector coarseningDirection; ///< whether a coarsening is used in each direction + std::array coarseningDirection; ///< whether a coarsening is used in each direction - std::vector xbeg; ///> Beginning of dataBlock - std::vector xend; ///> End of dataBlock + std::array xbeg; ///> Beginning of dataBlock + std::array xend; ///> End of dataBlock - std::vector np_tot; ///< total number of grid points - std::vector np_int; ///< internal number of grid points + std::array np_tot; ///< total number of grid points + std::array np_int; ///< internal number of grid points - std::vector nghost; ///< number of ghost cells + std::array nghost; ///< number of ghost cells - std::vector lbound; ///< Boundary condition to the left - std::vector rbound; ///< Boundary condition to the right + std::array lbound; ///< Boundary condition to the left + std::array rbound; ///< Boundary condition to the right - std::vector beg; ///< Begining of internal indices - std::vector end; ///< End of internal indices + std::array beg; ///< Begining of internal indices + std::array end; ///< End of internal indices - std::vector gbeg; ///< Begining of local block in the grid (internal) - std::vector gend; ///< End of local block in the grid (internal) + std::array gbeg; ///< Begining of local block in the grid (internal) + std::array gend; ///< End of local block in the grid (internal) explicit DataBlockHost(DataBlock &); ///< Constructor from a device datablock ///< (NB: does not sync any data) DataBlockHost() = default; ///< Default constructor + // The Planetary system (actually a copy from the dataBlock) + bool haveplanetarySystem{false}; + PlanetarySystem* planetarySystem; + + void MakeVsFromAmag(IdefixHostArray4D &); ///< Compute a face-centered mag. field in Vs from ///< potential vector in argument diff --git a/src/dataBlock/dumpToFile.cpp b/src/dataBlock/dumpToFile.cpp index 3bd09745..9980d09c 100644 --- a/src/dataBlock/dumpToFile.cpp +++ b/src/dataBlock/dumpToFile.cpp @@ -7,7 +7,8 @@ #include "../idefix.hpp" #include "dataBlock.hpp" -#include "gitversion.hpp" +#include "version.hpp" +#include "fluid.hpp" #define NAMESIZE 16 #define HEADERSIZE 128 @@ -42,15 +43,15 @@ void DataBlock::DumpToFile(std::string filebase) { // TODO(lesurg) Make datablock a friend of hydro to get the Riemann flux? - //IdefixArray4D::HostMirror locFlux = Kokkos::create_mirror_view(this->hydro.FluxRiemann); + //IdefixArray4D::HostMirror locFlux = Kokkos::create_mirror_view(this->hydro->FluxRiemann); //Kokkos::deep_copy(locFlux, this->FluxRiemann); #if MHD == YES IdefixArray4D::HostMirror locJ; - if(hydro.haveCurrent) { - locJ = Kokkos::create_mirror_view(this->hydro.J); - Kokkos::deep_copy(locJ, this->hydro.J); + if(hydro->haveCurrent) { + locJ = Kokkos::create_mirror_view(this->hydro->J); + Kokkos::deep_copy(locJ, this->hydro->J); } #endif @@ -74,12 +75,12 @@ void DataBlock::DumpToFile(std::string filebase) { // Write Header char header[HEADERSIZE]; - std::snprintf(header, HEADERSIZE, "Idefix %s Debug DataBlock", GITVERSION); + std::snprintf(header, HEADERSIZE, "Idefix %s Debug DataBlock", IDEFIX_VERSION); fwrite (header, sizeof(char), HEADERSIZE, fileHdl); // Write Vc - IdefixArray4D::HostMirror locVc = Kokkos::create_mirror_view(this->hydro.Vc); - Kokkos::deep_copy(locVc,this->hydro.Vc); + IdefixArray4D::HostMirror locVc = Kokkos::create_mirror_view(this->hydro->Vc); + Kokkos::deep_copy(locVc,this->hydro->Vc); dims[0] = this->np_tot[IDIR]; dims[1] = this->np_tot[JDIR]; dims[2] = this->np_tot[KDIR]; @@ -107,8 +108,8 @@ void DataBlock::DumpToFile(std::string filebase) { // Write Vs #if MHD == YES // Write Vs - IdefixArray4D::HostMirror locVs = Kokkos::create_mirror_view(this->hydro.Vs); - Kokkos::deep_copy(locVs,this->hydro.Vs); + IdefixArray4D::HostMirror locVs = Kokkos::create_mirror_view(this->hydro->Vs); + Kokkos::deep_copy(locVs,this->hydro->Vs); dims[0] = this->np_tot[IDIR]+IOFFSET; dims[1] = this->np_tot[JDIR]+JOFFSET; dims[2] = this->np_tot[KDIR]+KOFFSET; @@ -124,23 +125,23 @@ void DataBlock::DumpToFile(std::string filebase) { dims[2] = this->np_tot[KDIR]; std::snprintf(fieldName,NAMESIZE,"Ex3"); - IdefixArray3D::HostMirror locE = Kokkos::create_mirror_view(this->hydro.emf.ez); - Kokkos::deep_copy(locE,this->hydro.emf.ez); + IdefixArray3D::HostMirror locE = Kokkos::create_mirror_view(this->hydro->emf->ez); + Kokkos::deep_copy(locE,this->hydro->emf->ez); WriteVariable(fileHdl, 3, dims, fieldName, locE.data()); #if DIMENSIONS == 3 std::snprintf(fieldName,NAMESIZE,"Ex1"); - Kokkos::deep_copy(locE,this->hydro.emf.ex); + Kokkos::deep_copy(locE,this->hydro->emf->ex); WriteVariable(fileHdl, 3, dims, fieldName, locE.data()); std::snprintf(fieldName,NAMESIZE,"Ex2"); - Kokkos::deep_copy(locE,this->hydro.emf.ey); + Kokkos::deep_copy(locE,this->hydro->emf->ey); WriteVariable(fileHdl, 3, dims, fieldName, locE.data()); #endif - if(hydro.haveCurrent) { - IdefixArray4D::HostMirror locJ = Kokkos::create_mirror_view(this->hydro.J); - Kokkos::deep_copy(locJ,this->hydro.J); + if(hydro->haveCurrent) { + IdefixArray4D::HostMirror locJ = Kokkos::create_mirror_view(this->hydro->J); + Kokkos::deep_copy(locJ,this->hydro->J); dims[0] = this->np_tot[IDIR]; dims[1] = this->np_tot[JDIR]; dims[2] = this->np_tot[KDIR]; diff --git a/src/dataBlock/evolveStage.cpp b/src/dataBlock/evolveStage.cpp index b7351fe4..981c060a 100644 --- a/src/dataBlock/evolveStage.cpp +++ b/src/dataBlock/evolveStage.cpp @@ -7,60 +7,27 @@ #include "../idefix.hpp" #include "dataBlock.hpp" -#include "calcRightHandSide.hpp" -#include "calcParabolicFlux.hpp" -#include "calcRiemannFlux.hpp" - -template void DataBlock::LoopDir() { - // Step 2: compute the intercell flux with our Riemann solver, store the resulting InvDt - hydro.CalcRiemannFlux(this->t); - - // Step 2.5: compute intercell parabolic flux when needed - if(hydro.haveExplicitParabolicTerms) hydro.CalcParabolicFlux(this->t); - - // Step 3: compute the resulting evolution of the conserved variables, stored in Uc - hydro.CalcRightHandSide(this->t, this->dt); - - // Recursive: do next dimension - LoopDir(); -} - -template<> void DataBlock::LoopDir() { - // Do nothing -} - +#include "fluid.hpp" // Evolve one step forward in time of hydro void DataBlock::EvolveStage() { idfx::pushRegion("DataBlock::EvolveStage"); - // Compute current when needed - if(hydro.needExplicitCurrent) hydro.CalcCurrent(); - - // enable shock flattening - if(hydro.haveShockFlattening) hydro.shockFlattening.FindShock(); - - // Loop on all of the directions - LoopDir(); - // Step 4: add source terms to the conserved variables (curvature, rotation, etc) - if(hydro.haveSourceTerms) hydro.AddSourceTerms(this->t, this->dt); + hydro->EvolveStage(this->t,this->dt); -#if MHD == YES && DIMENSIONS >= 2 - // Compute the field evolution according to CT - hydro.emf.CalcCornerEMF(this->t); - if(hydro.resistivityStatus.isExplicit || hydro.ambipolarStatus.isExplicit) { - hydro.emf.CalcNonidealEMF(this->t); + if(haveDust) { + for(int i = 0 ; i < dust.size() ; i++) { + dust[i]->EvolveStage(this->t,this->dt); + } } - hydro.emf.EnforceEMFBoundary(); - #ifdef EVOLVE_VECTOR_POTENTIAL - hydro.emf.EvolveVectorPotential(this->dt, hydro.Ve); - hydro.emf.ComputeMagFieldFromA(hydro.Ve, hydro.Vs); - #else - hydro.emf.EvolveMagField(this->t, this->dt, hydro.Vs); - #endif - hydro.boundary.ReconstructVcField(hydro.Uc); -#endif + idfx::popRegion(); +} +void DataBlock::EvolveRKLStage() { + idfx::pushRegion("DataBlock::EvolveRKLStage"); + if(hydro->haveRKLParabolicTerms) { + hydro->rkl->Cycle(); + } idfx::popRegion(); } diff --git a/src/dataBlock/fargo.cpp b/src/dataBlock/fargo.cpp index 8906c946..c9999f21 100644 --- a/src/dataBlock/fargo.cpp +++ b/src/dataBlock/fargo.cpp @@ -9,127 +9,15 @@ #include #include "idefix.hpp" -#include "hydro.hpp" +#include "fluid.hpp" #include "dataBlock.hpp" #include "fargo.hpp" -// If no high order fargo, then choose the order according to reconstruction -#ifndef HIGH_ORDER_FARGO - #if ORDER >= 3 - #define HIGH_ORDER_FARGO - #endif -#endif - -#ifdef HIGH_ORDER_FARGO -KOKKOS_FORCEINLINE_FUNCTION real PPMLim(real dvp, real dvm) { - if(dvp*dvm >0.0) { - real dqc = 0.5*(dvp+dvm); - real d2q = 2.0*( fabs(dvp) < fabs(dvm) ? dvp : dvm); - return( fabs(d2q) < fabs(dqc) ? d2q : dqc); - } - return(ZERO_F); -} - -KOKKOS_INLINE_FUNCTION real FargoFlux(const IdefixArray4D &Vin, int n, int k, int j, int i, - int so, int ds, int sbeg, real eps, - bool haveDomainDecomposition) { - // compute shifted indices, taking into account the fact that we're periodic - int sop1 = so+1; - if(!haveDomainDecomposition && (sop1-sbeg >= ds)) sop1 = sop1-ds; - int sop2 = sop1+1; - if(!haveDomainDecomposition && (sop2-sbeg >= ds)) sop2 = sop2-ds; - - int som1 = so-1; - if(!haveDomainDecomposition && (som1-sbeg< 0 )) som1 = som1+ds; - int som2 = som1-1; - if(!haveDomainDecomposition && (som2-sbeg< 0 )) som2 = som2+ds; - - real dqm2,dqm1,dqp1,dqp2, q0,qm1, qp1; - #if GEOMETRY == CARTESIAN || GEOMETRY == POLAR - q0 = Vin(n,k,so,i); - qm1 = Vin(n,k,som1,i); - qp1 = Vin(n,k,sop1,i); - dqm2 = qm1 - Vin(n,k,som2,i); - dqm1 = q0 - qm1; - dqp1 = qp1 - q0; - dqp2 = Vin(n,k,sop2,i) - qp1; - #elif GEOMETRY == SPHERICAL - q0 = Vin(n,so,j,i); - qm1 = Vin(n,som1,j,i); - qp1 = Vin(n,sop1,j,i); - dqm2 = qm1 - Vin(n,som2,j,i); - dqm1 = q0 - qm1; - dqp1 = qp1 - q0; - dqp2 = Vin(n,sop2,j,i) - qp1; - #endif - // slope limited values around the reference point - real dqlm = PPMLim(dqm1,dqm2); - real dql0 = PPMLim(dqp1,dqm1); - real dqlp = PPMLim(dqp2,dqp1); - - real dqp = 0.5 * dqp1 - (dqlp - dql0) / 6.0; - real dqm = -0.5 * dqm1 - (dql0 - dqlm) / 6.0; - - if(dqp*dqm>0.0) { - dqp = dqm = 0.0; - } else { - if(FABS(dqp) >= 2.0*FABS(dqm)) dqp = -2.0*dqm; - if(FABS(dqm) >= 2.0*FABS(dqp)) dqm = -2.0*dqp; - } - - real qp = q0 + dqp; - real qm = q0 + dqm; - - real dqc = dqp - dqm; - real d2q = dqp + dqm; - - real F; - if(eps > 0.0) { - F = eps*(qp - 0.5*eps*(dqc + d2q*(3.0 - 2.0*eps))); - } else { - F = eps*(qm - 0.5*eps*(dqc - d2q*(3.0 + 2.0*eps))); - } - - return(F); -} - -#else// HIGH_ORDER_FARGO -KOKKOS_INLINE_FUNCTION real FargoFlux(const IdefixArray4D &Vin, int n, int k, int j, int i, - int so, int ds, int sbeg, real eps, - bool haveDomainDecomposition) { - // compute shifted indices, taking into account the fact that we're periodic - int sop1 = so+1; - if(!haveDomainDecomposition && (sop1-sbeg >= ds)) sop1 = sop1-ds; - int som1 = so-1; - if(!haveDomainDecomposition && (som1-sbeg< 0 )) som1 = som1+ds; - int sign = (eps>=0) ? 1 : -1; - real F, dqm, dqp, dq, V0; - #if GEOMETRY == CARTESIAN || GEOMETRY == POLAR - V0 = Vin(n,k,so,i); - dqm = V0 - Vin(n,k,som1,i); - dqp = Vin(n,k,sop1,i) - V0; - #elif GEOMETRY == SPHERICAL - V0 = Vin(n,so,j,i); - dqm = V0 - Vin(n,som1,j,i); - dqp = Vin(n,sop1,j,i) - V0; - #endif - dq = (dqp*dqm > ZERO_F ? TWO_F*dqp*dqm/(dqp + dqm) : ZERO_F); - F = eps*(V0 + sign*0.5*dq*(1.0-sign*eps)); - return(F); -} - -#endif // HIGH_ORDER_FARGO - -KOKKOS_INLINE_FUNCTION int modPositive(int x, int divisor) { - int m = x % divisor; - return m + ((m >> 31) & divisor); // equivalent to m + (m < 0 ? divisor : 0); -} -void Fargo::Init(Input &input, DataBlock *data) { +Fargo::Fargo(Input &input, int nmax, DataBlock *data) { idfx::pushRegion("Fargo::Init"); this->data = data; - this->hydro = &(data->hydro); // A bit of arithmetic to get the sizes of the working array this->nghost = data->nghost; @@ -226,7 +114,7 @@ void Fargo::Init(Input &input, DataBlock *data) { } else { // A separate allocation for scrhVs is only needed with domain decomposition, otherwise, // we just make a reference to scrhVs - this->scrhVs = hydro->Vs; + this->scrhVs = data->hydro->Vs; } #endif #ifdef WITH_MPI @@ -306,7 +194,7 @@ void Fargo::CheckMaxDisplacement() { IdefixArray1D xj; IdefixArray1D dxk; [[maybe_unused]] FargoType fargoType = type; - [[maybe_unused]] real sbS = hydro->sbS; + [[maybe_unused]] real sbS = data->hydro->sbS; real invDt = 0; // Get domain size @@ -357,584 +245,39 @@ void Fargo::CheckMaxDisplacement() { void Fargo::AddVelocity(const real t) { idfx::pushRegion("Fargo::AddVelocity"); - if(type==userdef) { - GetFargoVelocity(t); - } - IdefixArray1D x1 = data->x[IDIR]; - IdefixArray4D Vc = hydro->Vc; - IdefixArray2D meanV = this->meanVelocity; - [[maybe_unused]] FargoType fargoType = type; - [[maybe_unused]] real sbS = hydro->sbS; - idefix_for("FargoAddVelocity", - 0,data->np_tot[KDIR], - 0,data->np_tot[JDIR], - 0,data->np_tot[IDIR], - KOKKOS_LAMBDA(int k, int j, int i) { - #if GEOMETRY == CARTESIAN || GEOMETRY == POLAR - if(fargoType == userdef) { - Vc(VX2,k,j,i) += meanV(k,i); - } else if(fargoType == shearingbox) { - Vc(VX2,k,j,i) += sbS*x1(i); - } - #elif GEOMETRY == SPHERICAL - Vc(VX3,k,j,i) += meanV(j,i); - #endif - }); + this->AddVelocityFluid(t, data->hydro.get()); + if(data->haveDust) { + for(int i = 0 ; i < data->dust.size() ; i++) { + this->AddVelocityFluid(t, data->dust[i].get()); + } + } idfx::popRegion(); } void Fargo::SubstractVelocity(const real t) { idfx::pushRegion("Fargo::SubstractVelocity"); - if(type==userdef) { - GetFargoVelocity(t); - } - IdefixArray1D x1 = data->x[IDIR]; - IdefixArray4D Vc = hydro->Vc; - [[maybe_unused]] IdefixArray2D meanV = this->meanVelocity; - [[maybe_unused]] FargoType fargoType = type; - [[maybe_unused]] real sbS = hydro->sbS; - idefix_for("FargoSubstractVelocity", - 0,data->np_tot[KDIR], - 0,data->np_tot[JDIR], - 0,data->np_tot[IDIR], - KOKKOS_LAMBDA(int k, int j, int i) { - #if GEOMETRY == CARTESIAN || GEOMETRY == POLAR - if(fargoType == userdef) { - Vc(VX2,k,j,i) -= meanV(k,i); - } else if(fargoType == shearingbox) { - Vc(VX2,k,j,i) -= sbS*x1(i); - } - #elif GEOMETRY == SPHERICAL - Vc(VX3,k,j,i) -= meanV(j,i); - #endif - }); + this->SubstractVelocityFluid(t, data->hydro.get()); + if(data->haveDust) { + for(int i = 0 ; i < data->dust.size() ; i++) { + this->SubstractVelocityFluid(t, data->dust[i].get()); + } + } idfx::popRegion(); } -void Fargo::StoreToScratch() { - IdefixArray4D Uc = hydro->Uc; - IdefixArray4D scrhUc = this->scrhUc; - bool haveDomainDecomposition = this->haveDomainDecomposition; - int maxShift = this->maxShift; - - idefix_for("Fargo:StoreUc", - 0,NVAR, - data->beg[KDIR],data->end[KDIR], - data->beg[JDIR],data->end[JDIR], - data->beg[IDIR],data->end[IDIR], - KOKKOS_LAMBDA(int n, int k, int j, int i) { - if(!haveDomainDecomposition) { - scrhUc(n,k,j,i) = Uc(n,k,j,i); - } else { - #if GEOMETRY==POLAR || GEOMETRY==CARTESIAN - scrhUc(n,k,j+maxShift,i) = Uc(n,k,j,i); - #elif GEOMETRY == SPHERICAL - scrhUc(n,k+maxShift,j,i) = Uc(n,k,j,i); - #endif - } - }); - - #if MHD == YES - #ifdef EVOLVE_VECTOR_POTENTIAL - // Update Vs to its latest - hydro->emf.ComputeMagFieldFromA(hydro->Ve,hydro->Vs); - #endif - // in MHD mode, we need to copy Vs only when there is domain decomposition, otherwise, - // we just make a reference (this is already done by init) - if(haveDomainDecomposition) { - IdefixArray4D Vs = hydro->Vs; - IdefixArray4D scrhVs = this->scrhVs; - idefix_for("Fargo:StoreVs", - 0,DIMENSIONS, - data->beg[KDIR],data->end[KDIR]+KOFFSET, - data->beg[JDIR],data->end[JDIR]+JOFFSET, - data->beg[IDIR],data->end[IDIR]+IOFFSET, - KOKKOS_LAMBDA(int n, int k, int j, int i) { - #if GEOMETRY==POLAR || GEOMETRY==CARTESIAN - scrhVs(n,k,j+maxShift,i) = Vs(n,k,j,i); - #elif GEOMETRY == SPHERICAL - scrhVs(n,k+maxShift,j,i) = Vs(n,k,j,i); - #endif - }); - } - #endif - #if WITH_MPI - if(haveDomainDecomposition) { - #if GEOMETRY == CARTESIAN || GEOMETRY == POLAR - this->mpi.ExchangeX2(scrhUc, scrhVs); - #elif GEOMETRY == SPHERICAL - this->mpi.ExchangeX3(scrhUc, scrhVs); - #endif - } - #endif -} - void Fargo::ShiftSolution(const real t, const real dt) { - idfx::pushRegion("Fargo::ShiftSolution"); + idfx::pushRegion("Fargo::ShiftFluid"); - #if GEOMETRY == CYLINDRICAL - IDEFIX_ERROR("Fargo is not compatible with cylindrical geometry " - "(which is intended to be 2D axisymmetric)"); - #else - - // Refresh the fargo velocity function - if(type==userdef) { - GetFargoVelocity(t); - } - if(haveDomainDecomposition && dt>dtMax) { - std::stringstream message; - message << "Your dt is too large with your domain decomposition and Fargo." << std::endl - << "Got dt=" << dt << " and Fargo:dtmax=" << dtMax << "." << std::endl - << "Try to increase [Fargo]:maxShift to a value larger than " - << static_cast(ceil(dt/(dtMax/maxShift))) << std::endl; - IDEFIX_ERROR(message); + this->ShiftFluid(t,dt,data->hydro.get()); + if(data->haveDust) { + for(int i = 0 ; i < data->dust.size() ; i++) { + this->ShiftFluid(t,dt,data->dust[i].get()); + } } - IdefixArray4D Uc = hydro->Uc; - IdefixArray4D scrh = this->scrhUc; - IdefixArray2D meanV = this->meanVelocity; - IdefixArray1D x1 = data->x[IDIR]; - IdefixArray1D x2 = data->x[JDIR]; - IdefixArray1D dx2 = data->dx[JDIR]; - IdefixArray1D dx3 = data->dx[KDIR]; - IdefixArray1D sinx2 = data->sinx2; - IdefixArray1D sinx2m = data->sinx2m; - [[maybe_unused]] FargoType fargoType = type; - [[maybe_unused]] real sbS = hydro->sbS; - bool haveDomainDecomposition = this->haveDomainDecomposition; - int maxShift = this->maxShift; - - real Lphi; - int sbeg, send; - #if GEOMETRY == CARTESIAN || GEOMETRY == POLAR - Lphi = data->mygrid->xend[JDIR] - data->mygrid->xbeg[JDIR]; - sbeg = data->beg[JDIR]; - send = data->end[JDIR]; - #elif GEOMETRY == SPHERICAL - Lphi = data->mygrid->xend[KDIR] - data->mygrid->xbeg[KDIR]; - sbeg = data->beg[KDIR]; - send = data->end[KDIR]; - #else - Lphi = 1.0; // Do nothing, but initialize this. - #endif - - // move Uc to scratch, and fill the ghost zones if required. - StoreToScratch(); - - idefix_for("Fargo:ShiftVc", - 0,NVAR, - data->beg[KDIR],data->end[KDIR], - data->beg[JDIR],data->end[JDIR], - data->beg[IDIR],data->end[IDIR], - KOKKOS_LAMBDA(int n, int k, int j, int i) { - real w,dphi; - int s; - #if GEOMETRY == CARTESIAN - if(fargoType==userdef) { - w = meanV(k,i); - } else if(fargoType==shearingbox) { - w = sbS*x1(i); - } - dphi = dx2(j); - s = j; - #elif GEOMETRY == POLAR - w = meanV(k,i)/x1(i); - dphi = dx2(j); - s = j; - #elif GEOMETRY == SPHERICAL - w = meanV(j,i)/(x1(i)*sinx2(j)); - dphi = dx3(k); - s = k; - #endif - - // Compute the offset in phi, modulo the full domain size - real dL = std::fmod(w*dt, Lphi); - - // Translate this into # of cells - int m = static_cast (std::floor(dL/dphi+HALF_F)); - - // get the remainding shift - real eps = dL/dphi - m; - - // origin index before the shift - // Note the trick to get a positive module i%%n = (i%n + n)%n; - int ds = send-sbeg; - - // so is the "origin" index - int so; - if(haveDomainDecomposition) { - so = s-m + maxShift; // maxshift corresponds to the offset between - // the indices in scrh and in Uc - } else { - so = sbeg + modPositive(s-m-sbeg, ds); - } - - // Define Left and right fluxes - // Fluxes are defined from slop-limited interpolation - // Using Van-leer slope limiter (consistently with the main advection scheme) - real Fl,Fr; - - if(eps>=ZERO_F) { - int som1 = so-1; - if(!haveDomainDecomposition && som1-sbeg< 0 ) som1 = som1+ds; - Fl = FargoFlux(scrh, n, k, j, i, som1, ds, sbeg, eps, haveDomainDecomposition); - Fr = FargoFlux(scrh, n, k, j, i, so, ds, sbeg, eps, haveDomainDecomposition); - } else { - int sop1 = so+1; - if(!haveDomainDecomposition && sop1-sbeg >= ds) sop1 = sop1-ds; - Fl = FargoFlux(scrh, n, k, j, i, so, ds, sbeg, eps, haveDomainDecomposition); - Fr = FargoFlux(scrh, n, k, j, i, sop1, ds, sbeg, eps, haveDomainDecomposition); - } - - #if GEOMETRY == CARTESIAN || GEOMETRY == POLAR - Uc(n,k,s,i) = scrh(n,k,so,i) - (Fr - Fl); - #elif GEOMETRY == SPHERICAL - Uc(n,s,j,i) = scrh(n,so,j,i) - (Fr - Fl); - #endif - }); - -#if MHD == YES - IdefixArray4D scrhVs = this->scrhVs; - IdefixArray3D ex = hydro->emf.Ex1; - IdefixArray3D ey = hydro->emf.Ex2; - IdefixArray3D ez = hydro->emf.Ex3; - IdefixArray1D x1m = data->xl[IDIR]; - IdefixArray1D x2m = data->xl[JDIR]; - IdefixArray1D dmu = data->dmu; - IdefixArray1D dx1 = data->dx[IDIR]; - - - - #if GEOMETRY == CARTESIAN || GEOMETRY == POLAR - IdefixArray3D ek = ez; - #elif GEOMETRY == SPHERICAL - IdefixArray3D ek = ey; - #endif - - idefix_for("Fargo:ComputeEk", - data->beg[KDIR],data->end[KDIR]+KOFFSET, - data->beg[JDIR],data->end[JDIR]+JOFFSET, - data->beg[IDIR],data->end[IDIR]+IOFFSET, - KOKKOS_LAMBDA(int k, int j, int i) { - real w,dphi; - int s; - #if GEOMETRY == CARTESIAN - if(fargoType==userdef) { - w = 0.5*(meanV(k,i-1)+meanV(k,i)); - } else if(fargoType==shearingbox) { - w = sbS*x1m(i); - } - dphi = dx2(j); - s = j; - #elif GEOMETRY == POLAR - w = 0.5*(meanV(k,i-1)+meanV(k,i))/x1m(i); - dphi = dx2(j); - s = j; - #elif GEOMETRY == SPHERICAL - w = 0.5*(meanV(j,i-1)/x1(i-1)+meanV(j,i)/x1(i))/sinx2(j); - dphi = dx3(k); - s = k; - #endif - - // Compute the offset in phi, modulo the full domain size - real dL = std::fmod(w*dt, Lphi); - - // Translate this into # of cells - int m = static_cast (std::floor(dL/dphi+HALF_F)); - - // get the remainding shift - real eps = dL/dphi - m; - - // origin index before the shift - // Note the trick to get a positive module i%%n = (i%n + n)%n; - int n = send-sbeg; - - // so is the "origin" index - int so; - if(haveDomainDecomposition) { - so = s-m + maxShift; // maxshift corresponds to the offset between - // the indices in scrh and in Uc - } else { - so = sbeg + modPositive(s-m-sbeg,n); - } - - #if GEOMETRY == CARTESIAN || GEOMETRY == POLAR - if(eps>=ZERO_F) { - int som1; - if(haveDomainDecomposition) { - som1 = so - 1; - } else { - som1 = sbeg + modPositive(so-1-sbeg,n); - } - ek(k,s,i) = FargoFlux(scrhVs, BX1s, k, j, i, som1, n, sbeg, eps, haveDomainDecomposition); - - } else { - ek(k,s,i) = FargoFlux(scrhVs, BX1s, k, j, i, so, n, sbeg, eps, haveDomainDecomposition); - } - if(m>0) { - for(int ss = s-m ; ss < s ; ss++) { - int sc; - if(haveDomainDecomposition) { - sc = ss; - } else { - sc = sbeg + modPositive(ss-sbeg,n); - } - ek(k,s,i) += scrhVs(BX1s,k,sc,i); - } - } else { - for(int ss = s ; ss < s-m ; ss++) { - int sc; - if(haveDomainDecomposition) { - sc = ss; - } else { - sc = sbeg + modPositive(ss-sbeg,n); - } - ek(k,s,i) -= scrhVs(BX1s,k,sc,i); - } - } - #elif GEOMETRY == SPHERICAL - if(eps>=ZERO_F) { - int som1; - if(haveDomainDecomposition) { - som1 = so - 1; - } else { - som1 = sbeg + modPositive(so-1-sbeg,n); - } - ek(s,j,i) = FargoFlux(scrhVs, BX1s, k, j, i, som1, n, sbeg, eps, haveDomainDecomposition); - - } else { - ek(s,j,i) = FargoFlux(scrhVs, BX1s, k, j, i, so, n, sbeg, eps, haveDomainDecomposition); - } - if(m>0) { - for(int ss = s-m ; ss < s ; ss++) { - int sc; - if(haveDomainDecomposition) { - sc = ss; - } else { - sc = sbeg + modPositive(ss-sbeg,n); - } - ek(s,j,i) += scrhVs(BX1s,sc,j,i); - } - } else { - for(int ss = s ; ss < s-m ; ss++) { - int sc; - if(haveDomainDecomposition) { - sc = ss; - } else { - sc = sbeg + modPositive(ss-sbeg,n); - } - ek(s,j,i) -= scrhVs(BX1s,sc,j,i); - } - } - #endif // GEOMETRY - - ek(k,j,i) *= dphi; - }); - -#if DIMENSIONS == 3 - // In cartesian and polar coordinates, ei is actually -Ex - IdefixArray3D ei = ex; - - idefix_for("Fargo:ComputeEi", - data->beg[KDIR],data->end[KDIR]+KOFFSET, - data->beg[JDIR],data->end[JDIR]+JOFFSET, - data->beg[IDIR],data->end[IDIR]+IOFFSET, - KOKKOS_LAMBDA(int k, int j, int i) { - real w,dphi; - int s; - #if GEOMETRY == CARTESIAN - if(fargoType==userdef) { - w = 0.5*(meanV(k,i)+meanV(k-1,i)); - } else if(fargoType==shearingbox) { - w = sbS*x1(i); - } - - dphi = dx2(j); - s = j; - #elif GEOMETRY == POLAR - w = 0.5*(meanV(k-1,i)+meanV(k,i))/x1(i); - dphi = dx2(j); - s = j; - #elif GEOMETRY == SPHERICAL - w = 0.5*(meanV(j-1,i)/sinx2(j-1)+meanV(j,i)/sinx2(j))/(x1(i)); - dphi = dx3(k); - s = k; - #endif - - // Compute the offset in phi, modulo the full domain size - real dL = std::fmod(w*dt, Lphi); - - // Translate this into # of cells - int m = static_cast (std::floor(dL/dphi+HALF_F)); - - // get the remainding shift - real eps = dL/dphi - m; - - // origin index before the shift - // Note the trick to get a positive module i%%n = (i%n + n)%n; - int n = send-sbeg; - - // so is the "origin" index - int so; - if(haveDomainDecomposition) { - so = s-m + maxShift; // maxshift corresponds to the offset between - // the indices in scrh and in Uc - } else { - so = sbeg + modPositive(s-m-sbeg,n); - } - - // Compute EMF due to the shift via second order reconstruction - #if GEOMETRY == CARTESIAN || GEOMETRY == POLAR - if(eps>=ZERO_F) { - int som1; - if(haveDomainDecomposition) { - som1 = so - 1; - } else { - som1 = sbeg + modPositive(so-1-sbeg,n); - } - ei(k,s,i) = FargoFlux(scrhVs, BX3s, k, j, i, som1, n, sbeg, eps, haveDomainDecomposition); - } else { - ei(k,s,i) = FargoFlux(scrhVs, BX3s, k, j, i, so, n, sbeg, eps, haveDomainDecomposition); - } - if(m>0) { - for(int ss = s-m ; ss < s ; ss++) { - int sc; - if(haveDomainDecomposition) { - sc = ss; - } else { - sc = sbeg + modPositive(ss-sbeg,n); - } - ei(k,s,i) += scrhVs(BX3s,k,sc,i); - } - } else { - for(int ss = s ; ss < s-m ; ss++) { - int sc; - if(haveDomainDecomposition) { - sc = ss; - } else { - sc = sbeg + modPositive(ss-sbeg,n); - } - ei(k,s,i) -= scrhVs(BX3s,k,sc,i); - } - } - #elif GEOMETRY == SPHERICAL - if(eps>=ZERO_F) { - int som1; - if(haveDomainDecomposition) { - som1 = so - 1; - } else { - som1 = sbeg + modPositive(so-1-sbeg,n); - } - ei(s,j,i) = FargoFlux(scrhVs, BX2s, k, j, i, som1, n, sbeg, eps, haveDomainDecomposition); - } else { - ei(s,j,i) = FargoFlux(scrhVs, BX2s, k, j, i, so, n, sbeg, eps, haveDomainDecomposition); - } - if(m>0) { - for(int ss = s-m ; ss < s ; ss++) { - int sc; - if(haveDomainDecomposition) { - sc = ss; - } else { - sc = sbeg + modPositive(ss-sbeg,n); - } - ei(s,j,i) += scrhVs(BX2s,sc,j,i); - } - } else { - for(int ss = s ; ss < s-m ; ss++) { - int sc; - if(haveDomainDecomposition) { - sc = ss; - } else { - sc = sbeg + modPositive(ss-sbeg,n); - } - ei(s,j,i) -= scrhVs(BX2s,sc,j,i); - } - } - - #endif // GEOMETRY - - ei(k,j,i) *= dphi; - }); -#endif - - // Update field components according to the computed EMFS - #ifndef EVOLVE_VECTOR_POTENTIAL - IdefixArray4D Vs = hydro->Vs; - idefix_for("Fargo::EvolvMagField", - data->beg[KDIR],data->end[KDIR]+KOFFSET, - data->beg[JDIR],data->end[JDIR]+JOFFSET, - data->beg[IDIR],data->end[IDIR]+IOFFSET, - KOKKOS_LAMBDA (int k, int j, int i) { - #if GEOMETRY == CARTESIAN || GEOMETRY == POLAR - Vs(BX1s,k,j,i) += - (ez(k,j+1,i) - ez(k,j,i) ) / dx2(j); - - #elif GEOMETRY == SPHERICAL - Vs(BX1s,k,j,i) += - sinx2(j)*dx2(j)/dmu(j)*(ey(k+1,j,i) - ey(k,j,i) ) / dx3(k); - #endif - - #if GEOMETRY == CARTESIAN - Vs(BX2s,k,j,i) += D_EXPAND( 0.0 , - + (ez(k,j,i+1) - ez(k,j,i)) / dx1(i) , - + (ex(k+1,j,i) - ex(k,j,i)) / dx3(k) ); - #elif GEOMETRY == POLAR - Vs(BX2s,k,j,i) += D_EXPAND( 0.0 , - + (x1m(i+1) * ez(k,j,i+1) - x1m(i)*ez(k,j,i)) / dx1(i) , - + x1(i) * (ex(k+1,j,i) - ex(k,j,i)) / dx3(k) ); - #elif GEOMETRY == SPHERICAL - #if DIMENSIONS == 3 - Vs(BX2s,k,j,i) += - (ex(k+1,j,i) - ex(k,j,i)) / dx3(k); - #endif - #endif // GEOMETRY - - #if DIMENSIONS == 3 - #if GEOMETRY == CARTESIAN || GEOMETRY == POLAR - Vs(BX3s,k,j,i) -= (ex(k,j+1,i) - ex(k,j,i) ) / dx2(j); - #elif GEOMETRY == SPHERICAL - real A1p = x1m(i+1)*x1m(i+1); - real A1m = x1m(i)*x1m(i); - real A2m = FABS(sinx2m(j)); - real A2p = FABS(sinx2m(j+1)); - Vs(BX3s,k,j,i) += sinx2(j) * (A1p * ey(k,j,i+1) - A1m * ey(k,j,i))/(x1(i)*dx1(i)) - + (A2p * ex(k,j+1,i) - A2m * ex(k,j,i))/dx2(j); - #endif - #endif// DIMENSIONS - }); - - #else // EVOLVE_VECTOR_POTENTIAL - // evolve field using vector potential - IdefixArray4D Ve = hydro->Ve; - idefix_for("Fargo::EvolvMagField", - data->beg[KDIR],data->end[KDIR]+KOFFSET, - data->beg[JDIR],data->end[JDIR]+JOFFSET, - data->beg[IDIR],data->end[IDIR]+IOFFSET, - KOKKOS_LAMBDA (int k, int j, int i) { - #if GEOMETRY == CARTESIAN - #if DIMENSIONS == 3 - Ve(AX1e,k,j,i) += ex(k,j,i); - #endif - Ve(AX3e,k,j,i) += - ez(k,j,i); - #elif GEOMETRY == POLAR - #if DIMENSIONS == 3 - Ve(AX1e,k,j,i) += x1(i) * ex(k,j,i); - #endif - Ve(AX3e,k,j,i) += - x1m(i) * ez(k,j,i); - #elif GEOMETRY == SPHERICAL - #if DIMENSIONS == 3 - Ve(AX1e,k,j,i) += - x1(i) * sinx2m(j) * ex(k,j,i); - Ve(AX2e,k,j,i) += x1m(i) * sinx2(j) * ey(k,j,i); - #endif - #endif - }); - - #endif // EVOLVE_VECTOR_POTENTIAL - - -#endif // MHD -#endif // GEOMETRY==CYLINDRICAL - idfx::popRegion(); } diff --git a/src/dataBlock/fargo.hpp b/src/dataBlock/fargo.hpp index eb8163fa..54b1bc29 100644 --- a/src/dataBlock/fargo.hpp +++ b/src/dataBlock/fargo.hpp @@ -10,32 +10,53 @@ #include #include "idefix.hpp" +#ifdef WITH_MPI + #include "mpi.hpp" +#endif // Forward class hydro declaration -class Hydro; +#include "physics.hpp" +template class Fluid; +using Hydro = Fluid; class DataBlock; using FargoVelocityFunc = void (*) (DataBlock &, IdefixArray2D &); + + class Fargo { public: enum FargoType {none, userdef, shearingbox}; - void Init(Input &, DataBlock*); // Initialisation + Fargo(Input &, int, DataBlock*); // Initialisation void ShiftSolution(const real t, const real dt); // Effectively shift the solution void SubstractVelocity(const real); void AddVelocity(const real); void EnrollVelocity(FargoVelocityFunc); - void StoreToScratch(); void CheckMaxDisplacement(); void ShowConfig(); +// For internal use + template + void AddVelocityFluid(const real, Fluid* ); + + template + void SubstractVelocityFluid(const real, Fluid* ); + + template + void ShiftFluid(const real t, const real dt, Fluid* ); + + template + void StoreToScratch(Fluid*); + + void GetFargoVelocity(real); + + IdefixArray2D meanVelocity; + FargoType type{none}; // By default, Fargo is disabled + private: - friend class Hydro; + friend Hydro; DataBlock *data; - Hydro *hydro; - FargoType type{none}; // By default, Fargo is disabled - IdefixArray2D meanVelocity; IdefixArray4D scrhUc; IdefixArray4D scrhVs; @@ -43,16 +64,733 @@ class Fargo { Mpi mpi; // Fargo-specific MPI layer #endif - std::vector beg; - std::vector end; - std::vector nghost; + std::array beg; + std::array end; + std::array nghost; int maxShift; //< maximum number of cells along which we plan to shift. real dtMax{0}; //< Maximum allowable dt for a given Fargo velocity //< when domain decomposition is enabled bool velocityHasBeenComputed{false}; bool haveDomainDecomposition{false}; - void GetFargoVelocity(real); + FargoVelocityFunc fargoVelocityFunc{NULL}; // The user-defined fargo velocity function }; + +// If no high order fargo, then choose the order according to reconstruction +#ifndef HIGH_ORDER_FARGO + #if ORDER >= 3 + #define HIGH_ORDER_FARGO + #endif +#endif + +#ifdef HIGH_ORDER_FARGO +KOKKOS_FORCEINLINE_FUNCTION real PPMLim(real dvp, real dvm) { + if(dvp*dvm >0.0) { + real dqc = 0.5*(dvp+dvm); + real d2q = 2.0*( fabs(dvp) < fabs(dvm) ? dvp : dvm); + return( fabs(d2q) < fabs(dqc) ? d2q : dqc); + } + return(ZERO_F); +} + +KOKKOS_INLINE_FUNCTION real FargoFlux(const IdefixArray4D &Vin, int n, int k, int j, int i, + int so, int ds, int sbeg, real eps, + bool haveDomainDecomposition) { + // compute shifted indices, taking into account the fact that we're periodic + int sop1 = so+1; + if(!haveDomainDecomposition && (sop1-sbeg >= ds)) sop1 = sop1-ds; + int sop2 = sop1+1; + if(!haveDomainDecomposition && (sop2-sbeg >= ds)) sop2 = sop2-ds; + + int som1 = so-1; + if(!haveDomainDecomposition && (som1-sbeg< 0 )) som1 = som1+ds; + int som2 = som1-1; + if(!haveDomainDecomposition && (som2-sbeg< 0 )) som2 = som2+ds; + + real dqm2,dqm1,dqp1,dqp2, q0,qm1, qp1; + #if GEOMETRY == CARTESIAN || GEOMETRY == POLAR + q0 = Vin(n,k,so,i); + qm1 = Vin(n,k,som1,i); + qp1 = Vin(n,k,sop1,i); + dqm2 = qm1 - Vin(n,k,som2,i); + dqm1 = q0 - qm1; + dqp1 = qp1 - q0; + dqp2 = Vin(n,k,sop2,i) - qp1; + #elif GEOMETRY == SPHERICAL + q0 = Vin(n,so,j,i); + qm1 = Vin(n,som1,j,i); + qp1 = Vin(n,sop1,j,i); + dqm2 = qm1 - Vin(n,som2,j,i); + dqm1 = q0 - qm1; + dqp1 = qp1 - q0; + dqp2 = Vin(n,sop2,j,i) - qp1; + #endif + // slope limited values around the reference point + real dqlm = PPMLim(dqm1,dqm2); + real dql0 = PPMLim(dqp1,dqm1); + real dqlp = PPMLim(dqp2,dqp1); + + real dqp = 0.5 * dqp1 - (dqlp - dql0) / 6.0; + real dqm = -0.5 * dqm1 - (dql0 - dqlm) / 6.0; + + if(dqp*dqm>0.0) { + dqp = dqm = 0.0; + } else { + if(FABS(dqp) >= 2.0*FABS(dqm)) dqp = -2.0*dqm; + if(FABS(dqm) >= 2.0*FABS(dqp)) dqm = -2.0*dqp; + } + + real qp = q0 + dqp; + real qm = q0 + dqm; + + real dqc = dqp - dqm; + real d2q = dqp + dqm; + + real F; + if(eps > 0.0) { + F = eps*(qp - 0.5*eps*(dqc + d2q*(3.0 - 2.0*eps))); + } else { + F = eps*(qm - 0.5*eps*(dqc - d2q*(3.0 + 2.0*eps))); + } + + return(F); +} + +#else// HIGH_ORDER_FARGO +KOKKOS_INLINE_FUNCTION real FargoFlux(const IdefixArray4D &Vin, int n, int k, int j, int i, + int so, int ds, int sbeg, real eps, + bool haveDomainDecomposition) { + // compute shifted indices, taking into account the fact that we're periodic + int sop1 = so+1; + if(!haveDomainDecomposition && (sop1-sbeg >= ds)) sop1 = sop1-ds; + int som1 = so-1; + if(!haveDomainDecomposition && (som1-sbeg< 0 )) som1 = som1+ds; + int sign = (eps>=0) ? 1 : -1; + real F, dqm, dqp, dq, V0; + #if GEOMETRY == CARTESIAN || GEOMETRY == POLAR + V0 = Vin(n,k,so,i); + dqm = V0 - Vin(n,k,som1,i); + dqp = Vin(n,k,sop1,i) - V0; + #elif GEOMETRY == SPHERICAL + V0 = Vin(n,so,j,i); + dqm = V0 - Vin(n,som1,j,i); + dqp = Vin(n,sop1,j,i) - V0; + #endif + dq = (dqp*dqm > ZERO_F ? TWO_F*dqp*dqm/(dqp + dqm) : ZERO_F); + F = eps*(V0 + sign*0.5*dq*(1.0-sign*eps)); + return(F); +} + +#endif // HIGH_ORDER_FARGO + +KOKKOS_INLINE_FUNCTION int modPositive(int x, int divisor) { + int m = x % divisor; + return m + ((m >> 31) & divisor); // equivalent to m + (m < 0 ? divisor : 0); +} + + +template +void Fargo::AddVelocityFluid(const real t, Fluid* hydro ) { + idfx::pushRegion("Fargo::AddVelocityFluid"); + if(type==userdef) { + GetFargoVelocity(t); + } + IdefixArray1D x1 = data->x[IDIR]; + IdefixArray4D Vc = hydro->Vc; + IdefixArray2D meanV = this->meanVelocity; + [[maybe_unused]] FargoType fargoType = type; + [[maybe_unused]] real sbS = hydro->sbS; + #if GEOMETRY == CARTESIAN || GEOMETRY == POLAR + constexpr int Vadv = VX2; + #elif GEOMETRY == SPHERICAL + constexpr int Vadv = VX3; + #endif + if constexpr (Phys::nvar > Vadv) { + idefix_for("FargoAddVelocity", + 0,data->np_tot[KDIR], + 0,data->np_tot[JDIR], + 0,data->np_tot[IDIR], + KOKKOS_LAMBDA(int k, int j, int i) { + #if GEOMETRY == CARTESIAN || GEOMETRY == POLAR + if(fargoType == userdef) { + Vc(VX2,k,j,i) += meanV(k,i); + } else if(fargoType == shearingbox) { + Vc(VX2,k,j,i) += sbS*x1(i); + } + #elif GEOMETRY == SPHERICAL + Vc(VX3,k,j,i) += meanV(j,i); + #endif + }); + } // if constexpr + idfx::popRegion(); +} + +template +void Fargo::SubstractVelocityFluid(const real t, Fluid* hydro) { + idfx::pushRegion("Fargo::SubstractVelocityFluid"); + if(type==userdef) { + GetFargoVelocity(t); + } + IdefixArray1D x1 = data->x[IDIR]; + IdefixArray4D Vc = hydro->Vc; + [[maybe_unused]] IdefixArray2D meanV = this->meanVelocity; + [[maybe_unused]] FargoType fargoType = type; + [[maybe_unused]] real sbS = hydro->sbS; + #if GEOMETRY == CARTESIAN || GEOMETRY == POLAR + constexpr int Vadv = VX2; + #elif GEOMETRY == SPHERICAL + constexpr int Vadv = VX3; + #endif + if constexpr (Phys::nvar > Vadv) { + idefix_for("FargoSubstractVelocity", + 0,data->np_tot[KDIR], + 0,data->np_tot[JDIR], + 0,data->np_tot[IDIR], + KOKKOS_LAMBDA(int k, int j, int i) { + #if GEOMETRY == CARTESIAN || GEOMETRY == POLAR + if(fargoType == userdef) { + Vc(VX2,k,j,i) -= meanV(k,i); + } else if(fargoType == shearingbox) { + Vc(VX2,k,j,i) -= sbS*x1(i); + } + #elif GEOMETRY == SPHERICAL + Vc(VX3,k,j,i) -= meanV(j,i); + #endif + }); + } + idfx::popRegion(); +} + +template +void Fargo::StoreToScratch(Fluid* hydro) { + IdefixArray4D Uc = hydro->Uc; + IdefixArray4D scrhUc = this->scrhUc; + bool haveDomainDecomposition = this->haveDomainDecomposition; + int maxShift = this->maxShift; + + idefix_for("Fargo:StoreUc", + 0,Phys::nvar, + data->beg[KDIR],data->end[KDIR], + data->beg[JDIR],data->end[JDIR], + data->beg[IDIR],data->end[IDIR], + KOKKOS_LAMBDA(int n, int k, int j, int i) { + if(!haveDomainDecomposition) { + scrhUc(n,k,j,i) = Uc(n,k,j,i); + } else { + #if GEOMETRY==POLAR || GEOMETRY==CARTESIAN + scrhUc(n,k,j+maxShift,i) = Uc(n,k,j,i); + #elif GEOMETRY == SPHERICAL + scrhUc(n,k+maxShift,j,i) = Uc(n,k,j,i); + #endif + } + }); + + if constexpr(Phys::mhd) { + #ifdef EVOLVE_VECTOR_POTENTIAL + // Update Vs to its latest + hydro->emf->ComputeMagFieldFromA(hydro->Ve,hydro->Vs); + #endif + // in MHD mode, we need to copy Vs only when there is domain decomposition, otherwise, + // we just make a reference (this is already done by init) + if(haveDomainDecomposition) { + IdefixArray4D Vs = hydro->Vs; + IdefixArray4D scrhVs = this->scrhVs; + idefix_for("Fargo:StoreVs", + 0,DIMENSIONS, + data->beg[KDIR],data->end[KDIR]+KOFFSET, + data->beg[JDIR],data->end[JDIR]+JOFFSET, + data->beg[IDIR],data->end[IDIR]+IOFFSET, + KOKKOS_LAMBDA(int n, int k, int j, int i) { + #if GEOMETRY==POLAR || GEOMETRY==CARTESIAN + scrhVs(n,k,j+maxShift,i) = Vs(n,k,j,i); + #elif GEOMETRY == SPHERICAL + scrhVs(n,k+maxShift,j,i) = Vs(n,k,j,i); + #endif + }); + } + } + #if WITH_MPI + if(haveDomainDecomposition) { + #if GEOMETRY == CARTESIAN || GEOMETRY == POLAR + this->mpi.ExchangeX2(scrhUc, scrhVs); + #elif GEOMETRY == SPHERICAL + this->mpi.ExchangeX3(scrhUc, scrhVs); + #endif + } + #endif +} + +template +void Fargo::ShiftFluid(const real t, const real dt, Fluid* hydro) { + idfx::pushRegion("Fargo::ShiftFluid"); + + #if GEOMETRY == CYLINDRICAL + IDEFIX_ERROR("Fargo is not compatible with cylindrical geometry " + "(which is intended to be 2D axisymmetric)"); + #else + + // Refresh the fargo velocity function + if(type==userdef) { + GetFargoVelocity(t); + } + if(haveDomainDecomposition && dt>dtMax) { + std::stringstream message; + message << "Your dt is too large with your domain decomposition and Fargo." << std::endl + << "Got dt=" << dt << " and Fargo:dtmax=" << dtMax << "." << std::endl + << "Try to increase [Fargo]:maxShift to a value larger than " + << static_cast(ceil(dt/(dtMax/maxShift))) << std::endl; + IDEFIX_ERROR(message); + } + + IdefixArray4D Uc = hydro->Uc; + IdefixArray4D scrh = this->scrhUc; + IdefixArray2D meanV = this->meanVelocity; + IdefixArray1D x1 = data->x[IDIR]; + IdefixArray1D x2 = data->x[JDIR]; + IdefixArray1D dx2 = data->dx[JDIR]; + IdefixArray1D dx3 = data->dx[KDIR]; + IdefixArray1D sinx2 = data->sinx2; + IdefixArray1D sinx2m = data->sinx2m; + [[maybe_unused]] FargoType fargoType = type; + [[maybe_unused]] real sbS = hydro->sbS; + bool haveDomainDecomposition = this->haveDomainDecomposition; + int maxShift = this->maxShift; + + real Lphi; + int sbeg, send; + #if GEOMETRY == CARTESIAN || GEOMETRY == POLAR + Lphi = data->mygrid->xend[JDIR] - data->mygrid->xbeg[JDIR]; + sbeg = data->beg[JDIR]; + send = data->end[JDIR]; + #elif GEOMETRY == SPHERICAL + Lphi = data->mygrid->xend[KDIR] - data->mygrid->xbeg[KDIR]; + sbeg = data->beg[KDIR]; + send = data->end[KDIR]; + #else + Lphi = 1.0; // Do nothing, but initialize this. + #endif + + // move Uc to scratch, and fill the ghost zones if required. + StoreToScratch(hydro); + + idefix_for("Fargo:ShiftVc", + 0,Phys::nvar, + data->beg[KDIR],data->end[KDIR], + data->beg[JDIR],data->end[JDIR], + data->beg[IDIR],data->end[IDIR], + KOKKOS_LAMBDA(int n, int k, int j, int i) { + real w,dphi; + int s; + #if GEOMETRY == CARTESIAN + if(fargoType==userdef) { + w = meanV(k,i); + } else if(fargoType==shearingbox) { + w = sbS*x1(i); + } + dphi = dx2(j); + s = j; + #elif GEOMETRY == POLAR + w = meanV(k,i)/x1(i); + dphi = dx2(j); + s = j; + #elif GEOMETRY == SPHERICAL + w = meanV(j,i)/(x1(i)*sinx2(j)); + dphi = dx3(k); + s = k; + #endif + + // Compute the offset in phi, modulo the full domain size + real dL = std::fmod(w*dt, Lphi); + + // Translate this into # of cells + int m = static_cast (std::floor(dL/dphi+HALF_F)); + + // get the remainding shift + real eps = dL/dphi - m; + + // origin index before the shift + // Note the trick to get a positive module i%%n = (i%n + n)%n; + int ds = send-sbeg; + + // so is the "origin" index + int so; + if(haveDomainDecomposition) { + so = s-m + maxShift; // maxshift corresponds to the offset between + // the indices in scrh and in Uc + } else { + so = sbeg + modPositive(s-m-sbeg, ds); + } + + // Define Left and right fluxes + // Fluxes are defined from slope-limited interpolation + // Using Van-leer slope limiter (consistently with the main advection scheme) + real Fl,Fr; + + if(eps>=ZERO_F) { + int som1 = so-1; + if(!haveDomainDecomposition && som1-sbeg< 0 ) som1 = som1+ds; + Fl = FargoFlux(scrh, n, k, j, i, som1, ds, sbeg, eps, haveDomainDecomposition); + Fr = FargoFlux(scrh, n, k, j, i, so, ds, sbeg, eps, haveDomainDecomposition); + } else { + int sop1 = so+1; + if(!haveDomainDecomposition && sop1-sbeg >= ds) sop1 = sop1-ds; + Fl = FargoFlux(scrh, n, k, j, i, so, ds, sbeg, eps, haveDomainDecomposition); + Fr = FargoFlux(scrh, n, k, j, i, sop1, ds, sbeg, eps, haveDomainDecomposition); + } + + #if GEOMETRY == CARTESIAN || GEOMETRY == POLAR + Uc(n,k,s,i) = scrh(n,k,so,i) - (Fr - Fl); + #elif GEOMETRY == SPHERICAL + Uc(n,s,j,i) = scrh(n,so,j,i) - (Fr - Fl); + #endif + }); + + if constexpr(Phys::mhd) { + IdefixArray4D scrhVs = this->scrhVs; + IdefixArray3D ex = hydro->emf->Ex1; + IdefixArray3D ey = hydro->emf->Ex2; + IdefixArray3D ez = hydro->emf->Ex3; + IdefixArray1D x1m = data->xl[IDIR]; + IdefixArray1D x2m = data->xl[JDIR]; + IdefixArray1D dmu = data->dmu; + IdefixArray1D dx1 = data->dx[IDIR]; + + + + #if GEOMETRY == CARTESIAN || GEOMETRY == POLAR + IdefixArray3D ek = ez; + #elif GEOMETRY == SPHERICAL + IdefixArray3D ek = ey; + #endif + + idefix_for("Fargo:ComputeEk", + data->beg[KDIR],data->end[KDIR]+KOFFSET, + data->beg[JDIR],data->end[JDIR]+JOFFSET, + data->beg[IDIR],data->end[IDIR]+IOFFSET, + KOKKOS_LAMBDA(int k, int j, int i) { + real w,dphi; + int s; + #if GEOMETRY == CARTESIAN + if(fargoType==userdef) { + w = 0.5*(meanV(k,i-1)+meanV(k,i)); + } else if(fargoType==shearingbox) { + w = sbS*x1m(i); + } + dphi = dx2(j); + s = j; + #elif GEOMETRY == POLAR + w = 0.5*(meanV(k,i-1)+meanV(k,i))/x1m(i); + dphi = dx2(j); + s = j; + #elif GEOMETRY == SPHERICAL + w = 0.5*(meanV(j,i-1)/x1(i-1)+meanV(j,i)/x1(i))/sinx2(j); + dphi = dx3(k); + s = k; + #endif + + // Compute the offset in phi, modulo the full domain size + real dL = std::fmod(w*dt, Lphi); + + // Translate this into # of cells + int m = static_cast (std::floor(dL/dphi+HALF_F)); + + // get the remainding shift + real eps = dL/dphi - m; + + // origin index before the shift + // Note the trick to get a positive module i%%n = (i%n + n)%n; + int n = send-sbeg; + + // so is the "origin" index + int so; + if(haveDomainDecomposition) { + so = s-m + maxShift; // maxshift corresponds to the offset between + // the indices in scrh and in Uc + } else { + so = sbeg + modPositive(s-m-sbeg,n); + } + + #if GEOMETRY == CARTESIAN || GEOMETRY == POLAR + if(eps>=ZERO_F) { + int som1; + if(haveDomainDecomposition) { + som1 = so - 1; + } else { + som1 = sbeg + modPositive(so-1-sbeg,n); + } + ek(k,s,i) = FargoFlux(scrhVs, BX1s, k, j, i, som1, + n, sbeg, eps, haveDomainDecomposition); + + } else { + ek(k,s,i) = FargoFlux(scrhVs, BX1s, k, j, i, so, + n, sbeg, eps, haveDomainDecomposition); + } + if(m>0) { + for(int ss = s-m ; ss < s ; ss++) { + int sc; + if(haveDomainDecomposition) { + sc = ss; + } else { + sc = sbeg + modPositive(ss-sbeg,n); + } + ek(k,s,i) += scrhVs(BX1s,k,sc,i); + } + } else { + for(int ss = s ; ss < s-m ; ss++) { + int sc; + if(haveDomainDecomposition) { + sc = ss; + } else { + sc = sbeg + modPositive(ss-sbeg,n); + } + ek(k,s,i) -= scrhVs(BX1s,k,sc,i); + } + } + #elif GEOMETRY == SPHERICAL + if(eps>=ZERO_F) { + int som1; + if(haveDomainDecomposition) { + som1 = so - 1; + } else { + som1 = sbeg + modPositive(so-1-sbeg,n); + } + ek(s,j,i) = FargoFlux(scrhVs, BX1s, k, j, i, som1, + n, sbeg, eps, haveDomainDecomposition); + + } else { + ek(s,j,i) = FargoFlux(scrhVs, BX1s, k, j, i, so, + n, sbeg, eps, haveDomainDecomposition); + } + if(m>0) { + for(int ss = s-m ; ss < s ; ss++) { + int sc; + if(haveDomainDecomposition) { + sc = ss; + } else { + sc = sbeg + modPositive(ss-sbeg,n); + } + ek(s,j,i) += scrhVs(BX1s,sc,j,i); + } + } else { + for(int ss = s ; ss < s-m ; ss++) { + int sc; + if(haveDomainDecomposition) { + sc = ss; + } else { + sc = sbeg + modPositive(ss-sbeg,n); + } + ek(s,j,i) -= scrhVs(BX1s,sc,j,i); + } + } + #endif // GEOMETRY + + ek(k,j,i) *= dphi; + }); + + #if DIMENSIONS == 3 + // In cartesian and polar coordinates, ei is actually -Ex + IdefixArray3D ei = ex; + + idefix_for("Fargo:ComputeEi", + data->beg[KDIR],data->end[KDIR]+KOFFSET, + data->beg[JDIR],data->end[JDIR]+JOFFSET, + data->beg[IDIR],data->end[IDIR]+IOFFSET, + KOKKOS_LAMBDA(int k, int j, int i) { + real w,dphi; + int s; + #if GEOMETRY == CARTESIAN + if(fargoType==userdef) { + w = 0.5*(meanV(k,i)+meanV(k-1,i)); + } else if(fargoType==shearingbox) { + w = sbS*x1(i); + } + + dphi = dx2(j); + s = j; + #elif GEOMETRY == POLAR + w = 0.5*(meanV(k-1,i)+meanV(k,i))/x1(i); + dphi = dx2(j); + s = j; + #elif GEOMETRY == SPHERICAL + w = 0.5*(meanV(j-1,i)/sinx2(j-1)+meanV(j,i)/sinx2(j))/(x1(i)); + dphi = dx3(k); + s = k; + #endif + + // Compute the offset in phi, modulo the full domain size + real dL = std::fmod(w*dt, Lphi); + + // Translate this into # of cells + int m = static_cast (std::floor(dL/dphi+HALF_F)); + + // get the remainding shift + real eps = dL/dphi - m; + + // origin index before the shift + // Note the trick to get a positive module i%%n = (i%n + n)%n; + int n = send-sbeg; + + // so is the "origin" index + int so; + if(haveDomainDecomposition) { + so = s-m + maxShift; // maxshift corresponds to the offset between + // the indices in scrh and in Uc + } else { + so = sbeg + modPositive(s-m-sbeg,n); + } + + // Compute EMF due to the shift via second order reconstruction + #if GEOMETRY == CARTESIAN || GEOMETRY == POLAR + if(eps>=ZERO_F) { + int som1; + if(haveDomainDecomposition) { + som1 = so - 1; + } else { + som1 = sbeg + modPositive(so-1-sbeg,n); + } + ei(k,s,i) = FargoFlux(scrhVs, BX3s, k, j, i, som1, + n, sbeg, eps, haveDomainDecomposition); + } else { + ei(k,s,i) = FargoFlux(scrhVs, BX3s, k, j, i, so, + n, sbeg, eps, haveDomainDecomposition); + } + if(m>0) { + for(int ss = s-m ; ss < s ; ss++) { + int sc; + if(haveDomainDecomposition) { + sc = ss; + } else { + sc = sbeg + modPositive(ss-sbeg,n); + } + ei(k,s,i) += scrhVs(BX3s,k,sc,i); + } + } else { + for(int ss = s ; ss < s-m ; ss++) { + int sc; + if(haveDomainDecomposition) { + sc = ss; + } else { + sc = sbeg + modPositive(ss-sbeg,n); + } + ei(k,s,i) -= scrhVs(BX3s,k,sc,i); + } + } + #elif GEOMETRY == SPHERICAL + if(eps>=ZERO_F) { + int som1; + if(haveDomainDecomposition) { + som1 = so - 1; + } else { + som1 = sbeg + modPositive(so-1-sbeg,n); + } + ei(s,j,i) = FargoFlux(scrhVs, BX2s, k, j, i, som1, + n, sbeg, eps, haveDomainDecomposition); + } else { + ei(s,j,i) = FargoFlux(scrhVs, BX2s, k, j, i, so, + n, sbeg, eps, haveDomainDecomposition); + } + if(m>0) { + for(int ss = s-m ; ss < s ; ss++) { + int sc; + if(haveDomainDecomposition) { + sc = ss; + } else { + sc = sbeg + modPositive(ss-sbeg,n); + } + ei(s,j,i) += scrhVs(BX2s,sc,j,i); + } + } else { + for(int ss = s ; ss < s-m ; ss++) { + int sc; + if(haveDomainDecomposition) { + sc = ss; + } else { + sc = sbeg + modPositive(ss-sbeg,n); + } + ei(s,j,i) -= scrhVs(BX2s,sc,j,i); + } + } + + #endif // GEOMETRY + + ei(k,j,i) *= dphi; + }); + #endif + + // Update field components according to the computed EMFS + #ifndef EVOLVE_VECTOR_POTENTIAL + IdefixArray4D Vs = hydro->Vs; + idefix_for("Fargo::EvolvMagField", + data->beg[KDIR],data->end[KDIR]+KOFFSET, + data->beg[JDIR],data->end[JDIR]+JOFFSET, + data->beg[IDIR],data->end[IDIR]+IOFFSET, + KOKKOS_LAMBDA (int k, int j, int i) { + #if GEOMETRY == CARTESIAN || GEOMETRY == POLAR + Vs(BX1s,k,j,i) += - (ez(k,j+1,i) - ez(k,j,i) ) / dx2(j); + + #elif GEOMETRY == SPHERICAL + Vs(BX1s,k,j,i) += - sinx2(j)*dx2(j)/dmu(j)*(ey(k+1,j,i) - ey(k,j,i) ) / dx3(k); + #endif + + #if GEOMETRY == CARTESIAN + Vs(BX2s,k,j,i) += D_EXPAND( 0.0 , + + (ez(k,j,i+1) - ez(k,j,i)) / dx1(i) , + + (ex(k+1,j,i) - ex(k,j,i)) / dx3(k) ); + #elif GEOMETRY == POLAR + Vs(BX2s,k,j,i) += D_EXPAND( 0.0 , + + (x1m(i+1) * ez(k,j,i+1) - x1m(i)*ez(k,j,i)) / dx1(i) , + + x1(i) * (ex(k+1,j,i) - ex(k,j,i)) / dx3(k) ); + #elif GEOMETRY == SPHERICAL + #if DIMENSIONS == 3 + Vs(BX2s,k,j,i) += - (ex(k+1,j,i) - ex(k,j,i)) / dx3(k); + #endif + #endif // GEOMETRY + + #if DIMENSIONS == 3 + #if GEOMETRY == CARTESIAN || GEOMETRY == POLAR + Vs(BX3s,k,j,i) -= (ex(k,j+1,i) - ex(k,j,i) ) / dx2(j); + #elif GEOMETRY == SPHERICAL + real A1p = x1m(i+1)*x1m(i+1); + real A1m = x1m(i)*x1m(i); + real A2m = FABS(sinx2m(j)); + real A2p = FABS(sinx2m(j+1)); + Vs(BX3s,k,j,i) += sinx2(j) * (A1p * ey(k,j,i+1) - A1m * ey(k,j,i))/(x1(i)*dx1(i)) + + (A2p * ex(k,j+1,i) - A2m * ex(k,j,i))/dx2(j); + #endif + #endif// DIMENSIONS + }); + + #else // EVOLVE_VECTOR_POTENTIAL + // evolve field using vector potential + IdefixArray4D Ve = hydro->Ve; + idefix_for("Fargo::EvolvMagField", + data->beg[KDIR],data->end[KDIR]+KOFFSET, + data->beg[JDIR],data->end[JDIR]+JOFFSET, + data->beg[IDIR],data->end[IDIR]+IOFFSET, + KOKKOS_LAMBDA (int k, int j, int i) { + #if GEOMETRY == CARTESIAN + #if DIMENSIONS == 3 + Ve(AX1e,k,j,i) += ex(k,j,i); + #endif + Ve(AX3e,k,j,i) += - ez(k,j,i); + #elif GEOMETRY == POLAR + #if DIMENSIONS == 3 + Ve(AX1e,k,j,i) += x1(i) * ex(k,j,i); + #endif + Ve(AX3e,k,j,i) += - x1m(i) * ez(k,j,i); + #elif GEOMETRY == SPHERICAL + #if DIMENSIONS == 3 + Ve(AX1e,k,j,i) += - x1(i) * sinx2m(j) * ex(k,j,i); + Ve(AX2e,k,j,i) += x1m(i) * sinx2(j) * ey(k,j,i); + #endif + #endif + }); + + #endif // EVOLVE_VECTOR_POTENTIAL + } + #endif // GEOMETRY==CYLINDRICAL + idfx::popRegion(); +} + #endif // DATABLOCK_FARGO_HPP_ diff --git a/src/dataBlock/makeGeometry.cpp b/src/dataBlock/makeGeometry.cpp index 9e5e37d1..ddb9a210 100644 --- a/src/dataBlock/makeGeometry.cpp +++ b/src/dataBlock/makeGeometry.cpp @@ -16,6 +16,33 @@ void DataBlock::MakeGeometry() { idfx::pushRegion("DataBlock::MakeGeometry()"); + + // Initialize grid coarsening if needed + if(mygrid->haveGridCoarsening != GridCoarsening::disabled) { + this->haveGridCoarsening = mygrid->haveGridCoarsening; + this->coarseningDirection = mygrid->coarseningDirection; + + for(int dir = 0 ; dir < 3 ; dir++) { + if(coarseningDirection[dir]) { + const int Xt = (dir == IDIR ? JDIR : IDIR); + const int Xb = (dir == KDIR ? JDIR : KDIR); + + // Allocate coarsening level arrays + coarseningLevel[dir] = IdefixArray2D( + "DataBlock_corseLevel", + np_tot[Xb], + np_tot[Xt]); + // Make a local reference + IdefixArray2D coarseInit = coarseningLevel[dir]; + // Init coarsening level array to one everywhere + idefix_for("init_coarsening", 0, np_tot[Xb], 0, np_tot[Xt], + KOKKOS_LAMBDA(int j, int i) { + coarseInit(j,i) = 1; + }); + } + } + } + // Compute Volumes IdefixArray3D dV = this->dV; IdefixArray1D dx1 = this->dx[IDIR]; @@ -188,3 +215,31 @@ void DataBlock::MakeGeometry() { idfx::popRegion(); } + +void DataBlock::ExtractSubdomain() { + idfx::pushRegion("DataBlock::ExtractSubdomain"); + // Copy the relevant part of the coordinate system to the datablock + for(int dir = 0 ; dir < 3 ; dir++) { + int offset=gbeg[dir]-beg[dir]; + + IdefixArray1D x_input = mygrid->x[dir]; + IdefixArray1D x_output= x[dir]; + IdefixArray1D xr_input = mygrid->xr[dir]; + IdefixArray1D xr_output= xr[dir]; + IdefixArray1D xl_input = mygrid->xl[dir]; + IdefixArray1D xl_output= xl[dir]; + IdefixArray1D dx_input = mygrid->dx[dir]; + IdefixArray1D dx_output= dx[dir]; + + idefix_for("coordinates",0,np_tot[dir], + KOKKOS_LAMBDA (int i) { + x_output(i) = x_input(i+offset); + xr_output(i) = xr_input(i+offset); + xl_output(i) = xl_input(i+offset); + dx_output(i) = dx_input(i+offset); + } + ); + } + + idfx::popRegion(); +} diff --git a/src/dataBlock/planetarySystem/CMakeLists.txt b/src/dataBlock/planetarySystem/CMakeLists.txt new file mode 100644 index 00000000..217cb705 --- /dev/null +++ b/src/dataBlock/planetarySystem/CMakeLists.txt @@ -0,0 +1,8 @@ +target_sources(idefix + PUBLIC ${CMAKE_CURRENT_LIST_DIR}/planet.cpp + PUBLIC ${CMAKE_CURRENT_LIST_DIR}/planet.hpp + PUBLIC ${CMAKE_CURRENT_LIST_DIR}/planetarySystem.cpp + PUBLIC ${CMAKE_CURRENT_LIST_DIR}/planetarySystem.hpp + PUBLIC ${CMAKE_CURRENT_LIST_DIR}/planetStructs.hpp + + ) diff --git a/src/dataBlock/planetarySystem/planet.cpp b/src/dataBlock/planetarySystem/planet.cpp new file mode 100644 index 00000000..ac2732c6 --- /dev/null +++ b/src/dataBlock/planetarySystem/planet.cpp @@ -0,0 +1,451 @@ +// *********************************************************************************** +// Idefix MHD astrophysical code +// Copyright(C) Geoffroy R. J. Lesur +// and other code contributors +// Licensed under CeCILL 2.1 License, see COPYING for more information +// *********************************************************************************** + +#include +#include +#include "planet.hpp" +#include "dataBlock.hpp" +#include "planetarySystem.hpp" +#include "fluid.hpp" +/* +Planet::Planet() : + m_vxp(state.vx), + m_vyp(state.vy), + m_vzp(state.vz), + m_xp(state.x), + m_yp(state.y), + m_zp(state.z) { +} +*/ + +Planet::Planet(const Planet& p) : + m_vxp(state.vx), + m_vyp(state.vy), + m_vzp(state.vz), + m_xp(state.x), + m_yp(state.y), + m_zp(state.z) { + idfx::pushRegion("Planet::Planet(Planet)"); + this->operator=(p); + idfx::popRegion(); +} + +// Assignement operator (required since we internally use references) +Planet& Planet::operator=(const Planet &p) { + idfx::pushRegion("Planet::operator="); + this->state = p.state; + this->m_qp = p.m_qp; + this->m_qpIni = p.m_qpIni; + this->m_force = p.m_force; + this->m_tOffset = p.m_tOffset; + this->m_isActive = p.m_isActive; + this->m_ip = p.m_ip; + this->pSys = p.pSys; + this->data = p.data; + idfx::popRegion(); + return *this; +} + +Planet::Planet(int ip, Input &input, DataBlock *datain, PlanetarySystem *pSys): +m_vxp(state.vx), m_vyp(state.vy), m_vzp(state.vz), m_xp(state.x), m_yp(state.y), m_zp(state.z) { + idfx::pushRegion("Planet::Planet"); + + // Save the datablock to which we are attached from now on + this->data = datain; + this->pSys = pSys; + + this->m_ip = ip; + + real eccentricity{input.GetOrSet("Planet","initialEccentricity",ip, ZERO_F)}; + real inclination{input.GetOrSet("Planet","initialInclination",ip, ZERO_F)}; + real distIni{input.Get("Planet","initialDistance",ip)}; + real massTaper = pSys->massTaper; + + std::string pintegratorString = input.Get("Planet","integrator",0); + if (pintegratorString.compare("analytical") == 0) { + if (eccentricity != ZERO_F) { + IDEFIX_ERROR( + "planet "+std::to_string(ip) + +": no planet eccentricity yet\n\ + if analytical orbit. Try initialEccentricity = 0.0.\n\ + You can also change integrator."); + } + if (inclination != ZERO_F) { + IDEFIX_ERROR("planet "+std::to_string(ip) + +": no planet inclination yet\n\ + if analytical orbit. Try initialInclination = 0.0.\n\ + You can also change integrator."); + } + } + + this->m_qpIni = input.Get("Planet","planetToPrimary",ip); + this->m_tOffset = input.GetOrSet("Planet","tOffset",ip, ZERO_F); + + if (this->m_tOffset == ZERO_F) { + this->m_isActive = true; + } else { + this->m_isActive = false; + } + + // @GL: We need to discuss with GI Jonah in order to be able to add the gas indirect term if SG + + this->m_force = { + {ZERO_F,ZERO_F,ZERO_F}, + {ZERO_F,ZERO_F,ZERO_F}, + {ZERO_F,ZERO_F,ZERO_F}, + {ZERO_F,ZERO_F,ZERO_F} + }; + + if (massTaper == ZERO_F) { + this->m_qp = this->m_qpIni; + } else { + this->m_qp = ZERO_F; + } + // We initialize the quantities required by the planet solver + this->m_xp = distIni*(ONE_F+eccentricity); + this->m_yp = ZERO_F; + this->m_zp = ZERO_F; + this->m_vxp = ZERO_F; + if (distIni>=ZERO_F) { + this->m_vyp = sqrt((ONE_F+this->m_qp)/FABS(distIni))* + sqrt((ONE_F-eccentricity)/(ONE_F+eccentricity))*cos(inclination); + } else { + this->m_vyp = -sqrt((ONE_F+this->m_qp)/FABS(distIni))* + sqrt((ONE_F-eccentricity)/(ONE_F+eccentricity))*cos(inclination); + } + // Remove inertial rotating frame velocity if present + if(this->data->hydro->haveRotation) { + this->m_vyp += -this->data->hydro->OmegaZ * this->m_xp; + } + this->m_vzp = this->m_vyp*sin(inclination)/cos(inclination); + + idfx::popRegion(); +} + +void Planet::RegisterInDump() { + idfx::pushRegion("Planet::RegisterInDump"); + // Register variables for dump read/write + data->dump->RegisterVariable(&m_xp,std::string("x_p_")+std::to_string(m_ip)); + data->dump->RegisterVariable(&m_yp,std::string("y_p_")+std::to_string(m_ip)); + data->dump->RegisterVariable(&m_zp,std::string("z_p_")+std::to_string(m_ip)); + data->dump->RegisterVariable(&m_vxp,std::string("vx_p_")+std::to_string(m_ip)); + data->dump->RegisterVariable(&m_vyp,std::string("vy_p_")+std::to_string(m_ip)); + data->dump->RegisterVariable(&m_vzp,std::string("vz_p_")+std::to_string(m_ip)); + data->dump->RegisterVariable(&m_qp,std::string("q_p_")+std::to_string(m_ip)); + + idfx::popRegion(); +} + +void Planet::ShowConfig() { + idfx::cout << "Planet[" << this->m_ip << "]: mass qp=" << this->m_qpIni <m_ip << "]: initial location dp=" + << sqrt(pow(this->m_xp,2)+pow(this->m_yp,2)+pow(this->m_zp,2)) < not arguments of the class Planet... +} + +void Planet::displayPlanet() const { + idfx::cout << "Planet: Init orbital parameters. " << std::endl; + idfx::cout << "xp = " << this->m_xp << std::endl; + idfx::cout << "yp = " << this->m_yp << std::endl; + idfx::cout << "zp = " << this->m_zp << std::endl; + idfx::cout << "vxp = " << this->m_vxp << std::endl; + idfx::cout << "vyp = " << this->m_vyp << std::endl; + idfx::cout << "vzp = " << this->m_vzp << std::endl; + idfx::cout << "qp = " << this->m_qp << std::endl; +} + +real Planet::getMp() const { + return this->m_qp; +} + +real Planet::getXp() const { + return this->m_xp; +} + +real Planet::getYp() const { + return this->m_yp; +} + +real Planet::getZp() const { + return this->m_zp; +} + +real Planet::getVxp() const { + return this->m_vxp; +} + +real Planet::getVyp() const { + return this->m_vyp; +} + +real Planet::getVzp() const { + return this->m_vzp; +} + +bool Planet::getIsActive() const { + return this->m_isActive; +} + +int Planet::getIndex() const { + return(this->m_ip); +} + +void Planet::setMp(real qp) { + this->m_qp = qp; +} + +void Planet::setXp(real xp) { + this->m_xp = xp; +} + +void Planet::setYp(real yp) { + this->m_yp = yp; +} + +void Planet::setZp(real zp) { + this->m_zp = zp; +} + +void Planet::setVxp(real vxp) { + this->m_vxp = vxp; +} + +void Planet::setVyp(real vyp) { + this->m_vyp = vyp; +} + +void Planet::setVzp(real vzp) { + this->m_vzp = vzp; +} + + +void Planet::activatePlanet(const real t) { + if (t >= this->m_tOffset) { + this->m_isActive = true; + } else { + this->m_isActive = false; + } +} + + +void Planet::updateMp(const real t) { + // idfx::cout << "tOffset = " << this->m_tOffset << std::endl; + real mtaper = pSys->massTaper; + real factor_mtaper = ONE_F; + if (mtaper > ZERO_F) { + if (t >= this->m_tOffset) { + // changed t to t-tOffset + factor_mtaper = ( + (t-this->m_tOffset) >= mtaper ? ONE_F : .5*(ONE_F-cos(M_PI*(t-this->m_tOffset)/mtaper)) + ); + } else { + factor_mtaper = ZERO_F; + } + } + this->m_qp = this->m_qpIni*factor_mtaper; +} + +Point Planet::computeAccel(DataBlock& data, bool& isPlanet) { + Point acceleration; + Force &force = this->m_force; +// Force &force = data.planet[0].force; + computeForce(data,isPlanet); + bool excludeHill = pSys->excludeHill; + if (excludeHill) { + acceleration.x = force.f_ex_inner[0]+force.f_ex_outer[0]; + acceleration.y = force.f_ex_inner[1]+force.f_ex_outer[1]; + acceleration.z = force.f_ex_inner[2]+force.f_ex_outer[2]; + } else { + acceleration.x = force.f_inner[0]+force.f_outer[0]; + acceleration.y = force.f_inner[1]+force.f_outer[1]; + acceleration.z = force.f_inner[2]+force.f_outer[2]; + } + return acceleration; +} + +/* +Be careful: you need to substract +the azimuthally averaged density +prior to the torque evaluation (BM08 trick) +*/ +void Planet::computeForce(DataBlock& data, bool& isPlanet) { + PlanetarySystem::SmoothingFunction smoothingFunction = pSys->myPlanetarySmoothing; + real smoothingValue = pSys->smoothingValue; + real smoothingExponent = pSys->smoothingExponent; + + IdefixArray1D x1 = data.x[IDIR]; + IdefixArray1D x2 = data.x[JDIR]; + IdefixArray1D x3 = data.x[KDIR]; + + IdefixArray4D Vc = data.hydro->Vc; + IdefixArray3D dV = data.dV; + + real xp; + real yp; + real zp; + real qp; + + if(isPlanet) { + xp = this->m_xp; + yp = this->m_yp; + zp = this->m_zp; + qp = this->m_qp; + } else { + xp = ZERO_F; + yp = ZERO_F; + zp = ZERO_F; + qp = ZERO_F; + } + real distPlanet = sqrt(xp*xp+yp*yp+zp*zp); + real smoothing = smoothingValue * pow(distPlanet,ONE_F+smoothingExponent); + real rh = pow(qp/3., 1./3.)*distPlanet; + bool excludeHill = pSys->excludeHill; + + // since we cannot throw an error in kokkos kernel, with throw this one before the kernel. + #if GEOMETRY == CYLINDRICAL + IDEFIX_ERROR("Planet::ComputeForce is not compatible with the GEOMETRY you intend to use"); + #endif + + Kokkos::parallel_reduce("ComputeForce", + Kokkos::MDRangePolicy> + ({data.beg[KDIR],data.beg[JDIR],data.beg[IDIR]}, + {data.end[KDIR], data.end[JDIR], data.end[IDIR]}), + KOKKOS_LAMBDA (int k, int j, int i, Force &forceProc) { + real cellMass = dV(k,j,i)*Vc(RHO,k,j,i); + real xc, yc, zc; + #if GEOMETRY == CARTESIAN + xc = x1(i); + yc = x2(j); + zc = x3(k); + #elif GEOMETRY == POLAR + xc = x1(i)*cos(x2(j)); + yc = x1(i)*sin(x2(j)); + zc = x3(k); + #elif GEOMETRY == SPHERICAL + xc = x1(i)*sin(x2(j))*cos(x3(k)); + yc = x1(i)*sin(x2(j))*sin(x3(k)); + zc = x1(i)*cos(x2(j)); + #endif + + real distc = sqrt(xc*xc+yc*yc+zc*zc); + real dist2 = ((xc-xp)*(xc-xp) + (yc-yp)*(yc-yp) + (zc-zp)*(zc-zp)); + real hillcut; + + if(excludeHill) { + real squaredist2 = sqrt(dist2); + if (squaredist2/rh < 0.5) { + hillcut = ZERO_F; + } else { + if (squaredist2 > rh) { + hillcut = ONE_F; + } else { + hillcut = pow(sin((squaredist2/rh-.5)*M_PI),2.); + } + } + } + + real forceCell; + switch(smoothingFunction) { + case PlanetarySystem::SmoothingFunction::PLUMMER: + { + dist2 += smoothing*smoothing; // if default potential + real distance = sqrt(dist2); // if default potential + real InvDist3 = ONE_F/(dist2*distance); // if default potential + forceCell = cellMass * InvDist3; // if default potential + break; + } + case PlanetarySystem::SmoothingFunction::POLYNOMIAL: + { + real rmrp = sqrt(dist2); // if other potential + if (rmrp/smoothing < 1) { + forceCell = -cellMass*(3.0*rmrp/smoothing - 4.0)/smoothing/smoothing/smoothing; + } else { + forceCell = cellMass/rmrp/rmrp/rmrp; + } + break; + } + default: // do nothing + break; + } + if (distc < distPlanet) { + // INNER FORCE + forceProc.f_inner[0] += (xc-xp)*forceCell; + forceProc.f_inner[1] += (yc-yp)*forceCell; + forceProc.f_inner[2] += (zc-zp)*forceCell; + if(excludeHill) { + forceProc.f_ex_inner[0] += (xc-xp)*forceCell*hillcut; + forceProc.f_ex_inner[1] += (yc-yp)*forceCell*hillcut; + forceProc.f_ex_inner[2] += (zc-zp)*forceCell*hillcut; + } else { + forceProc.f_ex_inner[0] += ZERO_F; + forceProc.f_ex_inner[1] += ZERO_F; + forceProc.f_ex_inner[2] += ZERO_F; + } + } else { + // OUTER FORCE + forceProc.f_outer[0] += (xc-xp)*forceCell; + forceProc.f_outer[1] += (yc-yp)*forceCell; + forceProc.f_outer[2] += (zc-zp)*forceCell; + if(excludeHill) { + forceProc.f_ex_outer[0] += (xc-xp)*forceCell*hillcut; + forceProc.f_ex_outer[1] += (yc-yp)*forceCell*hillcut; + forceProc.f_ex_outer[2] += (zc-zp)*forceCell*hillcut; + } else { + forceProc.f_ex_outer[0] += ZERO_F; + forceProc.f_ex_outer[1] += ZERO_F; + forceProc.f_ex_outer[2] += ZERO_F; + } + } + }, this->m_force ); + + if(pSys->halfdisk) { + // Cancel vertical component + m_force.f_inner[2] = 0; + m_force.f_ex_inner[2] = 0; + m_force.f_outer[2] = 0; + m_force.f_ex_outer[2] = 0; + + // Multiply by 2 the remaining components + for(int i = 0 ; i < 2 ; i++) { + m_force.f_inner[i] *= 2; + m_force.f_ex_inner[i] *= 2; + m_force.f_outer[i] *= 2; + m_force.f_ex_outer[i] *= 2; + } + } + + #ifdef WITH_MPI + real forceGlob[12]; + real forceLoc[12]; + forceLoc[0] = this->m_force.f_inner[0]; + forceLoc[1] = this->m_force.f_inner[1]; + forceLoc[2] = this->m_force.f_inner[2]; + forceLoc[3] = this->m_force.f_ex_inner[0]; + forceLoc[4] = this->m_force.f_ex_inner[1]; + forceLoc[5] = this->m_force.f_ex_inner[2]; + forceLoc[6] = this->m_force.f_outer[0]; + forceLoc[7] = this->m_force.f_outer[1]; + forceLoc[8] = this->m_force.f_outer[2]; + forceLoc[9] = this->m_force.f_ex_outer[0]; + forceLoc[10] = this->m_force.f_ex_outer[1]; + forceLoc[11] = this->m_force.f_ex_outer[2]; + MPI_SAFE_CALL(MPI_Allreduce(&forceLoc, &forceGlob, 12, realMPI, MPI_SUM, MPI_COMM_WORLD)); + this->m_force.f_inner[0] = forceGlob[0]; + this->m_force.f_inner[1] = forceGlob[1]; + this->m_force.f_inner[2] = forceGlob[2]; + this->m_force.f_ex_inner[0] = forceGlob[3]; + this->m_force.f_ex_inner[1] = forceGlob[4]; + this->m_force.f_ex_inner[2] = forceGlob[5]; + this->m_force.f_outer[0] = forceGlob[6]; + this->m_force.f_outer[1] = forceGlob[7]; + this->m_force.f_outer[2] = forceGlob[8]; + this->m_force.f_ex_outer[0] = forceGlob[9]; + this->m_force.f_ex_outer[1] = forceGlob[10]; + this->m_force.f_ex_outer[2] = forceGlob[11]; + #endif +} diff --git a/src/dataBlock/planetarySystem/planet.hpp b/src/dataBlock/planetarySystem/planet.hpp new file mode 100644 index 00000000..46b70188 --- /dev/null +++ b/src/dataBlock/planetarySystem/planet.hpp @@ -0,0 +1,71 @@ +// *********************************************************************************** +// Idefix MHD astrophysical code +// Copyright(C) Geoffroy R. J. Lesur +// and other code contributors +// Licensed under CeCILL 2.1 License, see COPYING for more information +// *********************************************************************************** + +#ifndef DATABLOCK_PLANETARYSYSTEM_PLANET_HPP_ +#define DATABLOCK_PLANETARYSYSTEM_PLANET_HPP_ + +#include "idefix.hpp" +#include "input.hpp" +#include "planetStructs.hpp" + +// forward class declaration +class DataBlock; +class PlanetarySystem; + + +class Planet { + public: + Force m_force; + Planet(int, Input &, DataBlock *, PlanetarySystem *); + Planet(const Planet&); + Planet& operator=(const Planet &p); + //void Init(int &, Input &, DataBlock *, PlanetarySystem *); + void displayPlanet() const; + void RegisterInDump(); + void ShowConfig(); + real getMp() const; + real getXp() const; + real getYp() const; + real getZp() const; + real getVxp() const; + real getVyp() const; + real getVzp() const; + bool getIsActive() const; + int getIndex() const; + void setMp(real qp); + void setXp(real xp); + void setYp(real yp); + void setZp(real zp); + void setVxp(real vxp); + void setVyp(real vyp); + void setVzp(real vzp); + void updateMp(const real); + void activatePlanet(const real); + // refresh the force + Point computeAccel(DataBlock&, bool&); + void computeForce(DataBlock&, bool&); + + protected: + friend class PlanetarySystem; + DataBlock *data; + PointSpeed state; + + real &m_xp; + real &m_yp; + real &m_zp; + real &m_vxp; + real &m_vyp; + real &m_vzp; + bool m_isActive; + real m_qp; + real m_qpIni; + real m_tOffset; + int m_ip; + PlanetarySystem *pSys; +}; + +#endif // DATABLOCK_PLANETARYSYSTEM_PLANET_HPP_ diff --git a/src/dataBlock/planetarySystem/planetStructs.hpp b/src/dataBlock/planetarySystem/planetStructs.hpp new file mode 100644 index 00000000..3749a34d --- /dev/null +++ b/src/dataBlock/planetarySystem/planetStructs.hpp @@ -0,0 +1,103 @@ +// *********************************************************************************** +// Idefix MHD astrophysical code +// Copyright(C) Geoffroy R. J. Lesur +// and other code contributors +// Licensed under CeCILL 2.1 License, see COPYING for more information +// *********************************************************************************** + +#ifndef DATABLOCK_PLANETARYSYSTEM_PLANETSTRUCTS_HPP_ +#define DATABLOCK_PLANETARYSYSTEM_PLANETSTRUCTS_HPP_ + +#include "idefix.hpp" +#include "input.hpp" + + +// Structs used in Planetary system class + +struct Force { + real f_inner[3]; + real f_ex_inner[3]; + real f_outer[3]; + real f_ex_outer[3]; + KOKKOS_FUNCTION void operator+=(Force const volatile& f) volatile { + for (int i = 0; i < 3; ++i) { + f_inner[i] += f.f_inner[i]; + f_ex_inner[i] += f.f_ex_inner[i]; + f_outer[i] += f.f_outer[i]; + f_ex_outer[i] += f.f_ex_outer[i]; + } + } +}; +struct Point { + real x; + real y; + real z; +}; + +struct PointSpeed { + real x; + real y; + real z; + real vx; + real vy; + real vz; +}; + +// arithmetics of pointspeed +// Addition +static PointSpeed operator+(PointSpeed a, PointSpeed const &b ) { + a.x += b.x; + a.y += b.y; + a.z += b.z; + a.vx += b.vx; + a.vy += b.vy; + a.vz += b.vz; + return(a); + } + +// Substraction +static PointSpeed operator-(PointSpeed a, PointSpeed const &b ) { + a.x -= b.x; + a.y -= b.y; + a.z -= b.z; + a.vx -= b.vx; + a.vy -= b.vy; + a.vz -= b.vz; + return(a); + } + +// product +template +static PointSpeed operator*(PointSpeed a, T const &b ) { + a.x *= b; + a.y *= b; + a.z *= b; + a.vx *= b; + a.vy *= b; + a.vz *= b; + return(a); +} +template +static PointSpeed operator*(T const &b, PointSpeed a) { + a.x *= b; + a.y *= b; + a.z *= b; + a.vx *= b; + a.vy *= b; + a.vz *= b; + return(a); +} + +// division +template +static PointSpeed operator/(PointSpeed a, T const &b ) { + a.x /= b; + a.y /= b; + a.z /= b; + a.vx /= b; + a.vy /= b; + a.vz /= b; + return(a); +} + +#endif //DATABLOCK_PLANETARYSYSTEM_PLANETSTRUCTS_HPP_ diff --git a/src/dataBlock/planetarySystem/planetarySystem.cpp b/src/dataBlock/planetarySystem/planetarySystem.cpp new file mode 100644 index 00000000..b08cb0cf --- /dev/null +++ b/src/dataBlock/planetarySystem/planetarySystem.cpp @@ -0,0 +1,492 @@ +// *********************************************************************************** +// Idefix MHD astrophysical code +// Copyright(C) Geoffroy R. J. Lesur +// and other code contributors +// Licensed under CeCILL 2.1 License, see COPYING for more information +// *********************************************************************************** + +#include +#include +#include "planetarySystem.hpp" +#include "planet.hpp" +#include "planetStructs.hpp" +#include "dataBlock.hpp" +#include "fluid.hpp" +#include "gravity.hpp" + + +PlanetarySystem::PlanetarySystem(Input &input, DataBlock *datain) { + idfx::pushRegion("PlanetarySystem::Init"); + this->data = datain; + this->feelDisk = input.Get("Planet","feelDisk",0); + this->feelPlanets = input.Get("Planet","feelPlanets",0); + // read Solver from input file + std::string pintegratorString = input.GetOrSet("Planet","integrator",0,"rk4"); + if (pintegratorString.compare("analytical") == 0) { + this->myPlanetaryIntegrator = Integrator::ANALYTICAL; + if ((this->feelDisk) || (this->feelPlanets)) { + IDEFIX_ERROR("No feelDisk nor feelPlanets if analytical orbit.\n\ + Change integrator. You can also change feelDisk/feelPlanets."); + } + } else if (pintegratorString.compare("rk4") == 0) { + this->myPlanetaryIntegrator = Integrator::RK4; + } else if (pintegratorString.compare("rk5") == 0) { + this->myPlanetaryIntegrator = Integrator::RK5; + } else { + std::stringstream msg; + msg << "Unknown planet integrator type " << pintegratorString; + IDEFIX_ERROR(msg); + } + + std::string sfunctionString = input.Get("Planet","smoothing",0); + if (sfunctionString.compare("plummer") == 0) { + this->myPlanetarySmoothing = PLUMMER; + } else if (sfunctionString.compare("polynomial") == 0) { + this->myPlanetarySmoothing = POLYNOMIAL; + } else { + std::stringstream msg; + msg << "Unknown smoothing function type " << sfunctionString; + IDEFIX_ERROR(msg); + } + + this->torqueNormalization = input.GetOrSet("Planet","torqueNormalization",0, 1.0); + this->massTaper = input.GetOrSet("Planet","masstaper",0, ZERO_F); + this->excludeHill = input.GetOrSet("Planet","hillCut",0, false); + this->indirectPlanetsTerm = input.GetOrSet("Planet","indirectPlanets",0, true); + this->smoothingValue = input.Get("Planet","smoothing",1); + this->smoothingExponent = input.Get("Planet","smoothing",2); + + // Initialize the planet object attached to this datablock + this->nbp = input.CheckEntry("Planet","planetToPrimary"); + if (this->nbp<0) this->nbp=0; + + if ((this->nbp>1) + && ((this->myPlanetaryIntegrator == Integrator::RK4) + || (this->myPlanetaryIntegrator == Integrator::RK5)) + && (!(this->feelPlanets)) + && (this->indirectPlanetsTerm)) { + IDEFIX_WARNING("Careful, the results are unphysical if Runge-Kutta with\n\ +indirect term, with multiple planets that don't feel each others."); + } + + if(this->nbp>0) { + for(int ip = 0 ; ip < this->nbp ; ip++) { + this->planet.emplace_back(ip, input, this->data, this); + } + // Now that we have initialised our planets, we register the variables for dump output + // We cannot do this in the first loop since emplace_back might do some hidden copies + // which then messes up the pointers used by dump i/O + for(int ip = 0 ; ip < this->nbp ; ip++) { + this->planet[ip].RegisterInDump(); + } + } else { + IDEFIX_ERROR("need to define a planet-to-primary mass ratio via planetToPrimary"); + } +#if GEOMETRY == POLAR || GEOMETRY == CARTESIAN + if ((this->data->mygrid->xbeg[KDIR] == 0) + || (this->data->mygrid->xend[KDIR] == 0)) { + this->halfdisk = true; + } else { + this->halfdisk = false; + } +#elif GEOMETRY == SPHERICAL + if ( (this->data->mygrid->xbeg[JDIR] == M_PI/2.0) + || (this->data->mygrid->xend[JDIR] == M_PI/2.0)) { + this->halfdisk = true; + } else { + this->halfdisk = false; + } +#endif + + idfx::popRegion(); +} + +void PlanetarySystem::ShowConfig() { + idfx::pushRegion("PlanetarySystem::ShowConfig"); + idfx::cout << "PlanetarySystem: have " << this->nbp << " planets." << std::endl; + switch(this->myPlanetaryIntegrator) { + case ANALYTICAL: + idfx::cout << "PlanetarySystem: uses analytical integration for planet location." + << std::endl; + break; + case RK4: + idfx::cout << "PlanetarySystem: uses RK4 integration for planet location." << std::endl; + break; + case RK5: + idfx::cout << "PlanetarySystem: uses RK5 integration for planet location." << std::endl; + break; + default: + IDEFIX_ERROR("Unknown time integrator for planets"); + idfx::cout << "PlanetarySystem: feelDisk: " << this->feelDisk <feelPlanets <myPlanetarySmoothing) { + case PLUMMER: + idfx::cout << "PlanetarySystem: uses plummer expression for planet potential." + << std::endl; + break; + case POLYNOMIAL: + idfx::cout << "PlanetarySystem: uses polynomial expression for planet potential." + << std::endl; + break; + default: + IDEFIX_ERROR("Unknown smoothing function for planet potential"); + } + + if (this->halfdisk) { + idfx::cout << "PlanetarySystem: half disk is detected, planet torques are computed accordingly." + << std::endl; + } + + // walk and show the planets + for(Planet p : this->planet) { + p.ShowConfig(); + } + + idfx::popRegion(); +} + +real PlanetarySystem::GetSmoothingValue() const { + return this->smoothingValue; +} + +real PlanetarySystem::GetSmoothingExponent() const { + return this->smoothingExponent; +} + +void PlanetarySystem::EvolveSystem(DataBlock& data, const real& dt) { + idfx::pushRegion("PlanetarySystem::EvolveSystem"); + if(this->feelDisk) { + this->AdvancePlanetFromDisk(data, data.dt); + } + this->IntegratePlanets(data, data.dt); + idfx::popRegion(); +} + +void PlanetarySystem::AdvancePlanetFromDisk(DataBlock& data, const real& dt) { + idfx::pushRegion("PlanetarySystem::AdvancePlanetFromDisk"); + for(int ip=0; ip< this->nbp ; ip++) { + if (!(planet[ip].m_isActive)) continue; + Point gamma; + bool isp = true; + + real qp = planet[ip].m_qp; + real xp = planet[ip].m_xp; + real yp = planet[ip].m_yp; + real zp = planet[ip].m_zp; + real vxp = planet[ip].m_vxp; + real vyp = planet[ip].m_vyp; + real vzp = planet[ip].m_vzp; + real r = sqrt(xp*xp + yp*yp + zp*zp); + + gamma = planet[ip].computeAccel (data, isp); + + planet[ip].m_vxp += dt * gamma.x*this->torqueNormalization; + planet[ip].m_vyp += dt * gamma.y*this->torqueNormalization; + planet[ip].m_vzp += dt * gamma.z*this->torqueNormalization; + } + idfx::popRegion(); +} + +void PlanetarySystem::IntegratePlanets(DataBlock& data, const real& dt) { + switch(this->myPlanetaryIntegrator) { + case ANALYTICAL: + IntegrateAnalytically(data, dt); + break; + case RK4: + { + int subcycling = 1; + int i; + for (i = 0; i < subcycling; i++) + IntegrateRK4(data, 1.0/(static_cast(subcycling))*dt); + break; + } + case RK5: + { + int subcycling = 1; + int i; + for (i = 0; i < subcycling; i++) + IntegrateRK5(data, 1.0/(static_cast(subcycling))*dt); + break; + } + default: // do nothing + break; + } +} + +void PlanetarySystem::IntegrateAnalytically(DataBlock& data, const real& dt) { + idfx::pushRegion("PlanetarySystem::IntegrateAnalytically"); + for(int ip=0; ipnbp ; ip++) { + if (!(planet[ip].m_isActive)) continue; + + real qp = planet[ip].m_qp; + real xp = planet[ip].m_xp; + real yp = planet[ip].m_yp; + + // from cartesian to polar + real Rp{sqrt(xp*xp+yp*yp)}; + real phip{atan2(yp,xp)}; + + real omegap{sqrt((ONE_F+qp)/Rp/Rp/Rp)}; + + real RpNew{Rp}; +// real phipNew{phip + dt*omegap}; + real phipNew; + + if(planet[ip].data->hydro->haveRotation) { + real omegaframe = planet[ip].data->hydro->OmegaZ; + phipNew = phip + dt*(omegap-omegaframe); + } else { + phipNew = phip + dt*omegap; + } + + real vxp_tmp{ZERO_F}; + real vyp_tmp{RpNew*sqrt((ONE_F+qp)/RpNew/RpNew/RpNew)}; + + planet[ip].m_xp = RpNew*cos(phipNew); + planet[ip].m_yp = RpNew*sin(phipNew); + planet[ip].m_zp = ZERO_F; + planet[ip].m_vxp = vxp_tmp*cos(phipNew)-vyp_tmp*sin(phipNew); + planet[ip].m_vyp = vxp_tmp*sin(phipNew)+vyp_tmp*cos(phipNew); + planet[ip].m_vzp = ZERO_F; + } + idfx::popRegion(); +} + +void PlanetarySystem::IntegrateRK5(DataBlock& data, const real& dt) { + idfx::pushRegion("PlanetarySystem::IntegrateRK5"); + std::vector &ki = planet; + + real t1, t2, t3, t4; + std::vector k0 = ki; + std::vector k1 = ki; + std::vector k2 = ki; + std::vector k3 = ki; + std::vector k4 = ki; + std::vector kf = ki; + + // No need to copy since we already copied when instantiating k0 + + std::vector f0 = ComputeRHS(data.t, k0); + t1 = data.t + dt/4.0; + for(int ip=0; ipnbp; ip++) { + k1[ip].state = ki[ip].state + dt*f0[ip]/4; + } + + std::vector f1 = ComputeRHS(t1, k1); + t2 = data.t + 3*dt/8.0; + for(int ip=0; ipnbp; ip++) { + k2[ip].state = ki[ip].state + 3*dt*f0[ip]/32 + 9*dt*f1[ip]/32; + } + + std::vector f2 = ComputeRHS(t2, k2); + t3 = data.t + 12*dt/13.0; + for(int ip=0; ipnbp; ip++) { + k3[ip].state = ki[ip].state + 1932*dt*f0[ip]/2197 - 7200*dt*f1[ip]/2197 + 7296*dt*f2[ip]/2197; + } + + std::vector f3 = ComputeRHS(t3, k3); + t4 = data.t + dt; + for(int ip=0; ipnbp; ip++) { + k4[ip].state = ki[ip].state + 439*dt*f0[ip]/216 - 8*dt*f1[ip] + + 3680*dt*f2[ip]/513 - 845*dt*f3[ip]/4104; + } + + std::vector f4 = ComputeRHS(t4, k4); + for(int ip=0; ipnbp; ip++) { + kf[ip].state = ki[ip].state + dt * ( 25*f0[ip]/216 + 1408*f2[ip]/2565 + + 2197*f3[ip]/4104 - f4[ip]/5); + } + + planet = kf; + idfx::popRegion(); +} + +void PlanetarySystem::IntegrateRK4(DataBlock& data, const real& dt) { + idfx::pushRegion("PlanetarySystem::IntegrateRK4"); + std::vector &ki = planet; + + real t1, t2, t3; + std::vector k0 = ki; + std::vector k1 = ki; + std::vector k2 = ki; + std::vector k3 = ki; + std::vector kf = ki; + + // No need to copy since we already copied when instantiating k0 + + std::vector f0 = ComputeRHS(data.t, k0); + t1 = data.t + dt/2.0; + for(int ip=0; ipnbp; ip++) { + k1[ip].state = ki[ip].state + dt*f0[ip]/2; + } + + std::vector f1 = ComputeRHS(t1, k1); + t2 = data.t + dt/2.0; + for(int ip=0; ipnbp; ip++) { + k2[ip].state = ki[ip].state + dt*f1[ip]/2; + } + + std::vector f2 = ComputeRHS(t2, k2); + t3 = data.t + dt; + for(int ip=0; ipnbp; ip++) { + k3[ip].state = ki[ip].state + dt*f2[ip]; + } + + std::vector f3 = ComputeRHS(t3, k3); + for(int ip=0; ipnbp; ip++) { + kf[ip].state = ki[ip].state + dt * ( f0[ip] + 2*f1[ip] + 2*f2[ip] + f3[ip]) / 6.0; + } + + planet = kf; + idfx::popRegion(); +} + +std::vector PlanetarySystem::ComputeRHS(real& t, std::vector planet) { + std::vector planet_update(this->nbp); + + for(int ip=0; ipnbp; ip++) { + planet_update[ip].x = ZERO_F; + planet_update[ip].y = ZERO_F; + planet_update[ip].z = ZERO_F; + planet_update[ip].vx = ZERO_F; + planet_update[ip].vy = ZERO_F; + planet_update[ip].vz = ZERO_F; + + if (!(planet[ip].m_isActive)) continue; + + real dist; + real coef; + dist = sqrt( + planet[ip].m_xp*planet[ip].m_xp + + planet[ip].m_yp*planet[ip].m_yp + + planet[ip].m_zp*planet[ip].m_zp + ); + planet_update[ip].x = planet[ip].m_vxp; + planet_update[ip].y = planet[ip].m_vyp; + planet_update[ip].z = planet[ip].m_vzp; + planet_update[ip].vx = -planet[ip].m_xp/dist/dist/dist; + planet_update[ip].vy = -planet[ip].m_yp/dist/dist/dist; + planet_update[ip].vz = -planet[ip].m_zp/dist/dist/dist; + + for(int jp=0; jpnbp; jp++) { + if(this->indirectPlanetsTerm && planet[jp].m_isActive) { + dist = sqrt( + planet[jp].m_xp*planet[jp].m_xp + + planet[jp].m_yp*planet[jp].m_yp + + planet[jp].m_zp*planet[jp].m_zp + ); + coef = planet[jp].m_qp/dist/dist/dist; + planet_update[ip].vx -= coef*planet[jp].m_xp; + planet_update[ip].vy -= coef*planet[jp].m_yp; + planet_update[ip].vz -= coef*planet[jp].m_zp; + } + + if((jp != ip) && this->feelPlanets && planet[jp].m_isActive) { + dist = sqrt( + (planet[ip].m_xp-planet[jp].m_xp)*(planet[ip].m_xp-planet[jp].m_xp) + + (planet[ip].m_yp-planet[jp].m_yp)*(planet[ip].m_yp-planet[jp].m_yp) + + (planet[ip].m_zp-planet[jp].m_zp)*(planet[ip].m_zp-planet[jp].m_zp) + ); + coef = planet[jp].m_qp/dist/dist/dist; + planet_update[ip].vx += coef*(planet[jp].m_xp-planet[ip].m_xp); + planet_update[ip].vy += coef*(planet[jp].m_yp-planet[ip].m_yp); + planet_update[ip].vz += coef*(planet[jp].m_zp-planet[ip].m_zp); + } + } + if(planet[ip].data->hydro->haveRotation) { + real omega = planet[ip].data->hydro->OmegaZ; + + + // Add Coriolis and centrifugal forces + planet_update[ip].vx += 2*omega* planet[ip].m_vyp + omega*omega*planet[ip].m_xp; + planet_update[ip].vy += -2*omega* planet[ip].m_vxp + omega*omega*planet[ip].m_yp; + } + } + return planet_update; +} + +void PlanetarySystem::AddPlanetsPotential(IdefixArray3D &phiP, real t) { + idfx::pushRegion("PlanetarySystem::AddPlanetsPotential"); + bool indirectPlanetsTerm = this->indirectPlanetsTerm; + real smoothingValue = this->smoothingValue; + real smoothingExponent = this->smoothingExponent; + SmoothingFunction myPlanetarySmoothing = this->myPlanetarySmoothing; + + IdefixArray1D x1 = this->data->x[IDIR]; + IdefixArray1D x2 = this->data->x[JDIR]; + IdefixArray1D x3 = this->data->x[KDIR]; + + for(Planet& p : this->planet) { + // update mass according to mass taper + p.updateMp(t); + p.activatePlanet(t); + + bool isActive = p.getIsActive(); + if (!(isActive)) continue; + + real qp = p.getMp(); + real xp = p.getXp(); + real yp = p.getYp(); + real zp = p.getZp(); + + real distPlanet = sqrt(xp*xp+yp*yp+zp*zp); + real smoothing = smoothingValue * pow(distPlanet,1.0+smoothingExponent); + real Mcentral = this->data->gravity->centralMass; + + idefix_for("PlanetPotential", + 0,this->data->np_tot[KDIR], + 0, this->data->np_tot[JDIR], + 0, this->data->np_tot[IDIR], + KOKKOS_LAMBDA (int k, int j, int i) { + real xc, yc, zc; + #if GEOMETRY == CARTESIAN + xc = x1(i); + yc = x2(j); + zc = x3(k); + #elif GEOMETRY == POLAR + xc = x1(i)*cos(x2(j)); + yc = x1(i)*sin(x2(j)); + zc = x3(k); + #elif GEOMETRY == SPHERICAL + xc = x1(i)*sin(x2(j))*cos(x3(k)); + yc = x1(i)*sin(x2(j))*sin(x3(k)); + zc = x1(i)*cos(x2(j)); + #endif + + real dist = ((xc-xp)*(xc-xp)+ + (yc-yp)*(yc-yp)+ + (zc-zp)*(zc-zp)); + + // term due to planet + switch(myPlanetarySmoothing) { + case PLUMMER: + { + phiP(k,j,i) += -Mcentral*qp/sqrt(dist+smoothing*smoothing); + break; + } + case POLYNOMIAL: + { + real rmrp = sqrt(dist); + if (rmrp/smoothing < 1) { + phiP(k,j,i) += -(Mcentral*qp/rmrp)*(pow(rmrp/smoothing,4.0) - + 2.0*pow(rmrp/smoothing,3.0)+ + 2.0*rmrp/smoothing); + } else { + phiP(k,j,i) += -(Mcentral*qp/rmrp); + } + break; + } + default: // do nothing + break; + } + // indirect term due to planet + if (indirectPlanetsTerm) { + phiP(k,j,i) += Mcentral*qp*(xc*xp+yc*yp+zc*zp)/(distPlanet*distPlanet*distPlanet); + } + }); + } + + idfx::popRegion(); +} diff --git a/src/dataBlock/planetarySystem/planetarySystem.hpp b/src/dataBlock/planetarySystem/planetarySystem.hpp new file mode 100644 index 00000000..436ad3d4 --- /dev/null +++ b/src/dataBlock/planetarySystem/planetarySystem.hpp @@ -0,0 +1,66 @@ +// *********************************************************************************** +// Idefix MHD astrophysical code +// Copyright(C) Geoffroy R. J. Lesur +// and other code contributors +// Licensed under CeCILL 2.1 License, see COPYING for more information +// *********************************************************************************** + +// *********************************************************************************** +// Idefix MHD astrophysical code +// Copyright(C) 2020-2021 Geoffroy R. J. Lesur +// and other code contributors +// Licensed under CeCILL 2.1 License, see COPYING for more information +// *********************************************************************************** + +#ifndef DATABLOCK_PLANETARYSYSTEM_PLANETARYSYSTEM_HPP_ +#define DATABLOCK_PLANETARYSYSTEM_PLANETARYSYSTEM_HPP_ + +#include +#include "idefix.hpp" +#include "input.hpp" +#include "planet.hpp" + +// forward class declaration +class DataBlock; + + + +class PlanetarySystem { + public: + enum Integrator {RK4=1, ANALYTICAL, RK5}; + enum SmoothingFunction {PLUMMER=1, POLYNOMIAL}; + + PlanetarySystem(Input&, DataBlock*); + void EvolveSystem(DataBlock&, const real& ); + void IntegrateAnalytically(DataBlock&, const real&); + void IntegrateRK4(DataBlock&, const real&); + void IntegrateRK5(DataBlock&, const real&); + void ShowConfig(); + void AddPlanetsPotential(IdefixArray3D &, real); + std::vector ComputeRHS(real&, std::vector); + + // number of planets + int nbp{0}; + real torqueNormalization{ONE_F}; + std::vector planet; + real GetSmoothingValue() const; + real GetSmoothingExponent() const; + + protected: + void AdvancePlanetFromDisk(DataBlock&, const real&); + void IntegratePlanets(DataBlock&, const real&); + friend class Planet; + real massTaper{ZERO_F}; + real smoothingValue; + real smoothingExponent; + bool excludeHill{false}; + bool indirectPlanetsTerm{true}; + bool feelDisk; + bool feelPlanets; + bool halfdisk; + Integrator myPlanetaryIntegrator; + SmoothingFunction myPlanetarySmoothing; + DataBlock *data; +}; + +#endif // DATABLOCK_PLANETARYSYSTEM_PLANETARYSYSTEM_HPP_ diff --git a/src/dataBlock/validation.cpp b/src/dataBlock/validation.cpp index a8976220..9ee53e60 100644 --- a/src/dataBlock/validation.cpp +++ b/src/dataBlock/validation.cpp @@ -6,118 +6,24 @@ // *********************************************************************************** #include "dataBlock.hpp" -#include "dataBlockHost.hpp" - -// Check if current datablock has nans - -int DataBlock::CheckNan() { - int nanVs=0; - int nanVc=0; - - idfx::pushRegion("DataBlock::CheckNan"); - IdefixArray4D Vc=this->hydro.Vc; - - idefix_reduce("checkNanVc", - 0, NVAR, - beg[KDIR], end[KDIR], - beg[JDIR], end[JDIR], - beg[IDIR], end[IDIR], - KOKKOS_LAMBDA (int n, int k, int j, int i, int &nnan) { - if(std::isnan(Vc(n,k,j,i))) nnan++; - }, Kokkos::Sum(nanVc) // reduction variable - ); - - #if MHD == YES - IdefixArray4D Vs=this->hydro.Vs; - idefix_reduce("checkNanVs", - 0, DIMENSIONS, - beg[KDIR], end[KDIR]+KOFFSET, - beg[JDIR], end[JDIR]+JOFFSET, - beg[IDIR], end[IDIR]+IOFFSET, - KOKKOS_LAMBDA (int n, int k, int j, int i, int &nnan) { - if(std::isnan(Vs(n,k,j,i))) nnan++; - }, Kokkos::Sum(nanVs) // reduction variable - ); - #endif - - int nanTot = nanVc+nanVs; - #ifdef WITH_MPI - MPI_Allreduce(MPI_IN_PLACE, &nanTot,1,MPI_INT, MPI_SUM, MPI_COMM_WORLD); - #endif - if(nanTot>0) { - idfx::cout << "DataBlock: Nans were found in the current calculation" << std::endl; - if(nanVc+nanVs>0) { - // Each datablock shows its findings to stdout - std::cout << "DataBlock: rank " << idfx::prank << " found " << nanVc << " Nans in Vc" - #if MHD == YES - << " and " << nanVs << " Nans in Vs" - #endif - << " in the current datablock. Details will be in corresponding process log file." - << std::endl; - - // We need to make copies to find exactly where the thing is wrong - DataBlockHost dataHost(*this); - dataHost.SyncFromDevice(); - - int nerrormax=10; - - for(int k = beg[KDIR] ; k < end[KDIR] ; k++) { - for(int j = beg[JDIR] ; j < end[JDIR] ; j++) { - for(int i = beg[IDIR] ; i < end[IDIR] ; i++) { - for(int n = 0 ; n < NVAR ; n ++) { - if(std::isnan(dataHost.Vc(n,k,j,i)) && nerrormax>0) { - nerrormax--; - idfx::cout << "rank " << idfx::prank << ": Nan found in variable " - << this->hydro.VcName[n] << std::endl; - - idfx::cout << " global (i,j,k) = (" << i-beg[IDIR]+gbeg[IDIR]-nghost[IDIR] - << ", " << j-beg[JDIR]+gbeg[JDIR]-nghost[JDIR] << ", " - << k-beg[KDIR]+gbeg[KDIR]-nghost[KDIR] << ")" << std::endl; - - idfx::cout << " global (x,y,z) = (" << dataHost.x[IDIR](i) << ", " - << dataHost.x[JDIR](j) << ", " << dataHost.x[KDIR](k) << ")" << std::endl; - } - } - } - } - } - - #if MHD == YES - for(int k = beg[KDIR] ; k < end[KDIR]+KOFFSET ; k++) { - for(int j = beg[JDIR] ; j < end[JDIR]+JOFFSET ; j++) { - for(int i = beg[IDIR] ; i < end[IDIR]+IOFFSET ; i++) { - for(int n = 0 ; n < DIMENSIONS ; n ++) { - if(std::isnan(dataHost.Vs(n,k,j,i)) && nerrormax>0) { - nerrormax--; - idfx::cout << "rank " << idfx::prank << ": Nan found in variable " - << this->hydro.VsName[n] << std::endl; - idfx::cout << " global (i,j,k) = (" << i-beg[IDIR]+gbeg[IDIR]-nghost[IDIR] - << ", " << j-beg[JDIR]+gbeg[JDIR]-nghost[JDIR] << ", " - << k-beg[KDIR]+gbeg[KDIR]-nghost[KDIR] << ")" << std::endl; - - idfx::cout << " global (x,y,z) = (" << dataHost.x[IDIR](i) << ", " - << dataHost.x[JDIR](j) << ", " << dataHost.x[KDIR](k) << ")" << std::endl; - } - } - } - } - } - #endif - if(nerrormax<=0) { - idfx::cout << "... " << std::endl << "*** More Nans have been found in current dataBlock. " - << "Only showing the first 10." << std::endl; - } +#include "fluid.hpp" + +int DataBlock::CheckNan() { + idfx::pushRegion("DataBlock::Check"); + int nNans = hydro->CheckNan(); + if(haveDust) { + for(int n = 0 ; n < dust.size() ; n++) { + nNans += dust[n]->CheckNan(); } } idfx::popRegion(); - return(nanTot); + return(nNans); } - void DataBlock::Validate() { idfx::pushRegion("DataBlock::Validate"); - if(CheckNan()) { + if(this->CheckNan()) { IDEFIX_ERROR("Nans were found in your initial conditions."); } diff --git a/src/error.cpp b/src/error.cpp index 63740015..5ea3006c 100644 --- a/src/error.cpp +++ b/src/error.cpp @@ -10,6 +10,8 @@ #include "idefix.hpp" #include "error.hpp" + + void ErrorHandler(const int ErrorType, std::stringstream &ErrorMessage, std::string ErrorFunction, @@ -24,6 +26,10 @@ void ErrorHandler(const int ErrorType, idfx::cerr << ErrorMessage.str() << std::endl; idfx::cerr << "------------------------------------------------------------------------------" << std::endl; + if(idfx::warningsAreErrors) { + idfx::cerr << "Warnings are considered as errors" << std::endl; + idfx::safeExit(1); + } } else if (ErrorType == ERROR_DEPRECATED) { idfx::cerr << std::endl << "------------------------------------------------------------------------------" @@ -34,6 +40,10 @@ void ErrorHandler(const int ErrorType, idfx::cerr << ErrorMessage.str() << std::endl; idfx::cerr << "------------------------------------------------------------------------------" << std::endl; + if(idfx::warningsAreErrors) { + idfx::cerr << "Warnings are considered as errors" << std::endl; + idfx::safeExit(1); + } } else { idfx::cerr << std::endl << "------------------------------------------------------------------------------" @@ -43,10 +53,7 @@ void ErrorHandler(const int ErrorType, idfx::cerr << ErrorMessage.str() << std::endl; idfx::cerr << "------------------------------------------------------------------------------" << std::endl; - #ifdef WITH_MPI - MPI_Abort(MPI_COMM_WORLD,1); - #endif - exit(1); + idfx::safeExit(1); } } @@ -66,10 +73,7 @@ void ErrorHandler(const int ErrorType, << std::endl; if(idfx::warningsAreErrors) { idfx::cerr << "Warnings are considered as errors" << std::endl; - #ifdef WITH_MPI - MPI_Abort(MPI_COMM_WORLD,1); - #endif - exit(1); + idfx::safeExit(1); } } else if (ErrorType == ERROR_DEPRECATED) { idfx::cerr << std::endl @@ -83,10 +87,7 @@ void ErrorHandler(const int ErrorType, << std::endl; if(idfx::warningsAreErrors) { idfx::cerr << "Warnings are considered as errors" << std::endl; - #ifdef WITH_MPI - MPI_Abort(MPI_COMM_WORLD,1); - #endif - exit(1); + idfx::safeExit(1); } } else { idfx::cerr << std::endl @@ -97,9 +98,6 @@ void ErrorHandler(const int ErrorType, idfx::cerr << ErrorMessage << std::endl; idfx::cerr << "------------------------------------------------------------------------------" << std::endl; - #ifdef WITH_MPI - MPI_Abort(MPI_COMM_WORLD,1); - #endif - exit(1); + idfx::safeExit(1); } } diff --git a/src/fluid/CMakeLists.txt b/src/fluid/CMakeLists.txt new file mode 100644 index 00000000..f849b22f --- /dev/null +++ b/src/fluid/CMakeLists.txt @@ -0,0 +1,29 @@ +add_subdirectory(boundary) +add_subdirectory(braginskii) +add_subdirectory(constrainedTransport) +add_subdirectory(eos) +add_subdirectory(RiemannSolver) +add_subdirectory(tracer) + + +target_sources(idefix + PUBLIC ${CMAKE_CURRENT_LIST_DIR}/addNonIdealMHDFlux.hpp + PUBLIC ${CMAKE_CURRENT_LIST_DIR}/addSourceTerms.hpp + PUBLIC ${CMAKE_CURRENT_LIST_DIR}/calcCurrent.hpp + PUBLIC ${CMAKE_CURRENT_LIST_DIR}/calcParabolicFlux.hpp + PUBLIC ${CMAKE_CURRENT_LIST_DIR}/calcRightHandSide.hpp + PUBLIC ${CMAKE_CURRENT_LIST_DIR}/checkNan.hpp + PUBLIC ${CMAKE_CURRENT_LIST_DIR}/checkDivB.hpp + PUBLIC ${CMAKE_CURRENT_LIST_DIR}/coarsenFlow.hpp + PUBLIC ${CMAKE_CURRENT_LIST_DIR}/convertConsToPrim.hpp + PUBLIC ${CMAKE_CURRENT_LIST_DIR}/drag.hpp + PUBLIC ${CMAKE_CURRENT_LIST_DIR}/drag.cpp + PUBLIC ${CMAKE_CURRENT_LIST_DIR}/evolveStage.hpp + PUBLIC ${CMAKE_CURRENT_LIST_DIR}/fluid_defs.hpp + PUBLIC ${CMAKE_CURRENT_LIST_DIR}/enroll.hpp + PUBLIC ${CMAKE_CURRENT_LIST_DIR}/fluid.hpp + PUBLIC ${CMAKE_CURRENT_LIST_DIR}/viscosity.hpp + PUBLIC ${CMAKE_CURRENT_LIST_DIR}/viscosity.cpp + PUBLIC ${CMAKE_CURRENT_LIST_DIR}/thermalDiffusion.hpp + PUBLIC ${CMAKE_CURRENT_LIST_DIR}/thermalDiffusion.cpp + ) diff --git a/src/fluid/RiemannSolver/CMakeLists.txt b/src/fluid/RiemannSolver/CMakeLists.txt new file mode 100644 index 00000000..f9a01873 --- /dev/null +++ b/src/fluid/RiemannSolver/CMakeLists.txt @@ -0,0 +1,14 @@ + +add_subdirectory(MHDsolvers) +add_subdirectory(HDsolvers) +add_subdirectory(Dustsolvers) + + +target_sources(idefix + PUBLIC ${CMAKE_CURRENT_LIST_DIR}/calcFlux.hpp + PUBLIC ${CMAKE_CURRENT_LIST_DIR}/extrapolateToFaces.hpp + PUBLIC ${CMAKE_CURRENT_LIST_DIR}/flux.hpp + PUBLIC ${CMAKE_CURRENT_LIST_DIR}/riemannSolver.hpp + PUBLIC ${CMAKE_CURRENT_LIST_DIR}/shockFlattening.hpp + PUBLIC ${CMAKE_CURRENT_LIST_DIR}/slopeLimiter.hpp + ) diff --git a/src/fluid/RiemannSolver/Dustsolvers/CMakeLists.txt b/src/fluid/RiemannSolver/Dustsolvers/CMakeLists.txt new file mode 100644 index 00000000..0cafdc8d --- /dev/null +++ b/src/fluid/RiemannSolver/Dustsolvers/CMakeLists.txt @@ -0,0 +1,3 @@ +target_sources(idefix + PUBLIC ${CMAKE_CURRENT_LIST_DIR}/hllDust.hpp + ) diff --git a/src/fluid/RiemannSolver/Dustsolvers/hllDust.hpp b/src/fluid/RiemannSolver/Dustsolvers/hllDust.hpp new file mode 100644 index 00000000..d55f0ceb --- /dev/null +++ b/src/fluid/RiemannSolver/Dustsolvers/hllDust.hpp @@ -0,0 +1,106 @@ +// *********************************************************************************** +// Idefix MHD astrophysical code +// Copyright(C) Geoffroy R. J. Lesur +// and other code contributors +// Licensed under CeCILL 2.1 License, see COPYING for more information +// *********************************************************************************** + +#ifndef FLUID_RIEMANNSOLVER_DUSTSOLVERS_HLLDUST_HPP_ +#define FLUID_RIEMANNSOLVER_DUSTSOLVERS_HLLDUST_HPP_ + +#include "../idefix.hpp" +#include "fluid.hpp" +#include "extrapolateToFaces.hpp" +#include "flux.hpp" +#include "convertConsToPrim.hpp" + +// Compute Riemann fluxes from states using HLL solver +template +template +void RiemannSolver::HllDust(IdefixArray4D &Flux) { + idfx::pushRegion("RiemannSolver::HLL_Dust"); + + constexpr int ioffset = (DIR==IDIR) ? 1 : 0; + constexpr int joffset = (DIR==JDIR) ? 1 : 0; + constexpr int koffset = (DIR==KDIR) ? 1 : 0; + + IdefixArray4D Vc = this->Vc; + IdefixArray3D cMax = this->cMax; + + // Required for high order interpolations + IdefixArray1D dx = this->data->dx[DIR]; + + + ExtrapolateToFaces extrapol = *this->GetExtrapolator(); + + idefix_for("HLL_Kernel", + data->beg[KDIR],data->end[KDIR]+koffset, + data->beg[JDIR],data->end[JDIR]+joffset, + data->beg[IDIR],data->end[IDIR]+ioffset, + KOKKOS_LAMBDA (int k, int j, int i) { + // Init the directions (should be in the kernel for proper optimisation by the compilers) + constexpr int Xn = DIR+MX1; + + // Primitive variables + real vL[Phys::nvar]; + real vR[Phys::nvar]; + + // Conservative variables + real uL[Phys::nvar]; + real uR[Phys::nvar]; + + // Flux (left and right) + real fluxL[Phys::nvar]; + real fluxR[Phys::nvar]; + + + // 1-- Store the primitive variables on the left, right, and averaged states + extrapol.ExtrapolatePrimVar(i, j, k, vL, vR); + + // 2-- Get the wave speed + + real SL = vL[Xn]; + real SR = vR[Xn]; + + real cmax = FMAX(FABS(SL), FABS(SR)); + + // 3-- Compute the conservative variables: do this by extrapolation + K_PrimToCons(uL, vL, NULL); // Set gamma to 0 implicitly + K_PrimToCons(uR, vR, NULL); + + // 4-- Compute the left and right fluxes (wave speed is null) + K_Flux(fluxL, vL, uL, 0); + K_Flux(fluxR, vR, uR, 0); + + // 5-- Compute the flux from the left and right states + if (SL > 0) { +#pragma unroll + for (int nv = 0 ; nv < Phys::nvar; nv++) { + Flux(nv,k,j,i) = fluxL[nv]; + } + } else if (SR < 0) { +#pragma unroll + for (int nv = 0 ; nv < Phys::nvar; nv++) { + Flux(nv,k,j,i) = fluxR[nv]; + } + } else { + real dS = SR-SL; + if(std::abs(dS) < SMALL_NUMBER) { + dS = SMALL_NUMBER; + } +#pragma unroll + for(int nv = 0 ; nv < Phys::nvar; nv++) { + Flux(nv,k,j,i) = SL*SR*uR[nv] - SL*SR*uL[nv] + SR*fluxL[nv] - SL*fluxR[nv]; + Flux(nv,k,j,i) /= dS; + } + } + + //6-- Compute maximum wave speed for this sweep + cMax(k,j,i) = cmax; + } + ); + + idfx::popRegion(); +} + +#endif // FLUID_RIEMANNSOLVER_DUSTSOLVERS_HLLDUST_HPP_ diff --git a/src/hydro/HDsolvers/CMakeLists.txt b/src/fluid/RiemannSolver/HDsolvers/CMakeLists.txt similarity index 100% rename from src/hydro/HDsolvers/CMakeLists.txt rename to src/fluid/RiemannSolver/HDsolvers/CMakeLists.txt diff --git a/src/hydro/HDsolvers/hllHD.hpp b/src/fluid/RiemannSolver/HDsolvers/hllHD.hpp similarity index 61% rename from src/hydro/HDsolvers/hllHD.hpp rename to src/fluid/RiemannSolver/HDsolvers/hllHD.hpp index ce5a9ab7..89d72d33 100644 --- a/src/hydro/HDsolvers/hllHD.hpp +++ b/src/fluid/RiemannSolver/HDsolvers/hllHD.hpp @@ -5,19 +5,20 @@ // Licensed under CeCILL 2.1 License, see COPYING for more information // *********************************************************************************** -#ifndef HYDRO_HDSOLVERS_HLLHD_HPP_ -#define HYDRO_HDSOLVERS_HLLHD_HPP_ +#ifndef FLUID_RIEMANNSOLVER_HDSOLVERS_HLLHD_HPP_ +#define FLUID_RIEMANNSOLVER_HDSOLVERS_HLLHD_HPP_ #include "../idefix.hpp" -#include "hydro.hpp" -#include "slopeLimiter.hpp" -#include "fluxHD.hpp" -#include "convertConsToPrimHD.hpp" +#include "fluid.hpp" +#include "extrapolateToFaces.hpp" +#include "flux.hpp" +#include "convertConsToPrim.hpp" // Compute Riemann fluxes from states using HLL solver +template template -void Hydro::HllHD() { - idfx::pushRegion("Hydro::HLL_Solver"); +void RiemannSolver::HllHD(IdefixArray4D &Flux) { + idfx::pushRegion("RiemannSolver::HLL_Solver"); constexpr int ioffset = (DIR==IDIR) ? 1 : 0; constexpr int joffset = (DIR==JDIR) ? 1 : 0; @@ -25,20 +26,13 @@ void Hydro::HllHD() { IdefixArray4D Vc = this->Vc; IdefixArray4D Vs = this->Vs; - IdefixArray4D Flux = this->FluxRiemann; IdefixArray3D cMax = this->cMax; - IdefixArray3D csIsoArr = this->isoSoundSpeedArray; + EquationOfState eos = *(hydro->eos.get()); // Required for high order interpolations IdefixArray1D dx = this->data->dx[DIR]; - [[maybe_unused]] real gamma = this->gamma; - [[maybe_unused]] real gamma_m1 = gamma - ONE_F; - [[maybe_unused]] real csIso = this->isoSoundSpeed; - [[maybe_unused]] HydroModuleStatus haveIsoCs = this->haveIsoSoundSpeed; - - SlopeLimiter slopeLim(Vc,data->dx[DIR],shockFlattening); - + ExtrapolateToFaces extrapol = *this->GetExtrapolator(); idefix_for("HLL_Kernel", data->beg[KDIR],data->end[KDIR]+koffset, data->beg[JDIR],data->end[JDIR]+joffset, @@ -48,35 +42,32 @@ void Hydro::HllHD() { constexpr int Xn = DIR+MX1; // Primitive variables - real vL[NVAR]; - real vR[NVAR]; + real vL[Phys::nvar]; + real vR[Phys::nvar]; // Conservative variables - real uL[NVAR]; - real uR[NVAR]; + real uL[Phys::nvar]; + real uR[Phys::nvar]; // Flux (left and right) - real fluxL[NVAR]; - real fluxR[NVAR]; + real fluxL[Phys::nvar]; + real fluxR[Phys::nvar]; // Signal speeds real cL, cR, cmax; // 1-- Store the primitive variables on the left, right, and averaged states - slopeLim.ExtrapolatePrimVar(i, j, k, vL, vR); + extrapol.ExtrapolatePrimVar(i, j, k, vL, vR); // 2-- Get the wave speed -#if HAVE_ENERGY - cL = std::sqrt( gamma *(vL[PRS]/vL[RHO]) ); - cR = std::sqrt( gamma *(vR[PRS]/vR[RHO]) ); -#else - if(haveIsoCs == UserDefFunction) { - cL = HALF_F*(csIsoArr(k,j,i)+csIsoArr(k-koffset,j-joffset,i-ioffset)); - } else { - cL = csIso; - } - cR = cL; -#endif + #if HAVE_ENERGY + cL = std::sqrt(eos.GetGamma(vL[PRS],vL[RHO])*(vL[PRS]/vL[RHO])); + cR = std::sqrt(eos.GetGamma(vR[PRS],vR[RHO])*(vR[PRS]/vR[RHO])); + #else + cL = HALF_F*(eos.GetWaveSpeed(k,j,i) + +eos.GetWaveSpeed(k-koffset,j-joffset,i-ioffset)); + cR = cL; + #endif // 4.1 real cminL = vL[Xn] - cL; @@ -91,27 +82,27 @@ void Hydro::HllHD() { cmax = FMAX(FABS(SL), FABS(SR)); // 2-- Compute the conservative variables: do this by extrapolation - K_PrimToCons(uL, vL, gamma_m1); - K_PrimToCons(uR, vR, gamma_m1); + K_PrimToCons(uL, vL, &eos); + K_PrimToCons(uR, vR, &eos); // 3-- Compute the left and right fluxes - K_Flux(fluxL, vL, uL, cL*cL, Xn); - K_Flux(fluxR, vR, uR, cR*cR, Xn); + K_Flux(fluxL, vL, uL, cL*cL); + K_Flux(fluxR, vR, uR, cR*cR); // 5-- Compute the flux from the left and right states if (SL > 0) { #pragma unroll - for (int nv = 0 ; nv < NFLX; nv++) { + for (int nv = 0 ; nv < Phys::nvar; nv++) { Flux(nv,k,j,i) = fluxL[nv]; } } else if (SR < 0) { #pragma unroll - for (int nv = 0 ; nv < NFLX; nv++) { + for (int nv = 0 ; nv < Phys::nvar; nv++) { Flux(nv,k,j,i) = fluxR[nv]; } } else { #pragma unroll - for(int nv = 0 ; nv < NFLX; nv++) { + for(int nv = 0 ; nv < Phys::nvar; nv++) { Flux(nv,k,j,i) = SL*SR*uR[nv] - SL*SR*uL[nv] + SR*fluxL[nv] - SL*fluxR[nv]; Flux(nv,k,j,i) /= (SR - SL); } @@ -125,4 +116,4 @@ void Hydro::HllHD() { idfx::popRegion(); } -#endif // HYDRO_HDSOLVERS_HLLHD_HPP_ +#endif // FLUID_RIEMANNSOLVER_HDSOLVERS_HLLHD_HPP_ diff --git a/src/hydro/HDsolvers/hllcHD.hpp b/src/fluid/RiemannSolver/HDsolvers/hllcHD.hpp similarity index 72% rename from src/hydro/HDsolvers/hllcHD.hpp rename to src/fluid/RiemannSolver/HDsolvers/hllcHD.hpp index 2a2aba40..e79aac35 100644 --- a/src/hydro/HDsolvers/hllcHD.hpp +++ b/src/fluid/RiemannSolver/HDsolvers/hllcHD.hpp @@ -5,19 +5,20 @@ // Licensed under CeCILL 2.1 License, see COPYING for more information // *********************************************************************************** -#ifndef HYDRO_HDSOLVERS_HLLCHD_HPP_ -#define HYDRO_HDSOLVERS_HLLCHD_HPP_ +#ifndef FLUID_RIEMANNSOLVER_HDSOLVERS_HLLCHD_HPP_ +#define FLUID_RIEMANNSOLVER_HDSOLVERS_HLLCHD_HPP_ #include "../idefix.hpp" -#include "hydro.hpp" -#include "slopeLimiter.hpp" -#include "fluxHD.hpp" -#include "convertConsToPrimHD.hpp" +#include "fluid.hpp" +#include "extrapolateToFaces.hpp" +#include "flux.hpp" +#include "convertConsToPrim.hpp" // Compute Riemann fluxes from states using HLLC solver +template template -void Hydro::HllcHD() { - idfx::pushRegion("Hydro::HLLC_Solver"); +void RiemannSolver::HllcHD(IdefixArray4D &Flux) { + idfx::pushRegion("RiemannSolver::HLLC_Solver"); constexpr int ioffset = (DIR==IDIR) ? 1 : 0; constexpr int joffset = (DIR==JDIR) ? 1 : 0; @@ -25,16 +26,11 @@ void Hydro::HllcHD() { IdefixArray4D Vc = this->Vc; IdefixArray4D Vs = this->Vs; - IdefixArray4D Flux = this->FluxRiemann; IdefixArray3D cMax = this->cMax; - IdefixArray3D csIsoArr = this->isoSoundSpeedArray; - [[maybe_unused]] real gamma = this->gamma; - [[maybe_unused]] real gamma_m1 = this->gamma - ONE_F; - [[maybe_unused]] real csIso = this->isoSoundSpeed; - [[maybe_unused]] HydroModuleStatus haveIsoCs = this->haveIsoSoundSpeed; + EquationOfState eos = *(hydro->eos.get()); - SlopeLimiter slopeLim(Vc,data->dx[DIR],shockFlattening); + ExtrapolateToFaces extrapol = *this->GetExtrapolator(); idefix_for("HLLC_Kernel", data->beg[KDIR],data->end[KDIR]+koffset, @@ -47,35 +43,32 @@ void Hydro::HllcHD() { constexpr int Xb = (DIR == KDIR ? MX2 : MX3); ) // Primitive variables - real vL[NVAR]; - real vR[NVAR]; + real vL[Phys::nvar]; + real vR[Phys::nvar]; // Conservative variables - real uL[NVAR]; - real uR[NVAR]; + real uL[Phys::nvar]; + real uR[Phys::nvar]; // Flux (left and right) - real fluxL[NVAR]; - real fluxR[NVAR]; + real fluxL[Phys::nvar]; + real fluxR[Phys::nvar]; // Signal speeds real cL, cR, cmax; // 1-- Store the primitive variables on the left, right, and averaged states - slopeLim.ExtrapolatePrimVar(i, j, k, vL, vR); + extrapol.ExtrapolatePrimVar(i, j, k, vL, vR); // 2-- Get the wave speed -#if HAVE_ENERGY - cL = std::sqrt( gamma *(vL[PRS]/vL[RHO]) ); - cR = std::sqrt( gamma *(vR[PRS]/vR[RHO]) ); -#else - if(haveIsoCs == UserDefFunction) { - cL = HALF_F*(csIsoArr(k,j,i)+csIsoArr(k-koffset,j-joffset,i-ioffset)); - } else { - cL = csIso; - } - cR = cL; -#endif + #if HAVE_ENERGY + cL = std::sqrt(eos.GetGamma(vL[PRS],vL[RHO])*(vL[PRS]/vL[RHO])); + cR = std::sqrt(eos.GetGamma(vR[PRS],vR[RHO])*(vR[PRS]/vR[RHO])); + #else + cL = HALF_F*(eos.GetWaveSpeed(k,j,i) + +eos.GetWaveSpeed(k-koffset,j-joffset,i-ioffset)); + cR = cL; + #endif real cminL = vL[Xn] - cL; real cmaxL = vL[Xn] + cL; @@ -89,27 +82,27 @@ void Hydro::HllcHD() { cmax = FMAX(FABS(SL), FABS(SR)); // 3-- Compute the conservative variables - K_PrimToCons(uL, vL, gamma_m1); - K_PrimToCons(uR, vR, gamma_m1); + K_PrimToCons(uL, vL, &eos); + K_PrimToCons(uR, vR, &eos); // 4-- Compute the left and right fluxes - K_Flux(fluxL, vL, uL, cL*cL, Xn); - K_Flux(fluxR, vR, uR, cR*cR, Xn); + K_Flux(fluxL, vL, uL, cL*cL); + K_Flux(fluxR, vR, uR, cR*cR); // 5-- Compute the flux from the left and right states if (SL > 0) { #pragma unroll - for (int nv = 0 ; nv < NFLX; nv++) { + for (int nv = 0 ; nv < Phys::nvar; nv++) { Flux(nv,k,j,i) = fluxL[nv]; } } else if (SR < 0) { #pragma unroll - for (int nv = 0 ; nv < NFLX; nv++) { + for (int nv = 0 ; nv < Phys::nvar; nv++) { Flux(nv,k,j,i) = fluxR[nv]; } } else { - real usL[NVAR]; - real usR[NVAR]; + real usL[Phys::nvar]; + real usR[Phys::nvar]; real vs; #if HAVE_ENERGY @@ -154,12 +147,12 @@ void Hydro::HllcHD() { // Compute the flux from the left and right states if (vs >= 0.0) { #pragma unroll - for(int nv = 0 ; nv < NVAR; nv++) { + for(int nv = 0 ; nv < Phys::nvar; nv++) { Flux(nv,k,j,i) = fluxL[nv] + SL*(usL[nv] - uL[nv]); } } else { #pragma unroll - for(int nv = 0 ; nv < NVAR; nv++) { + for(int nv = 0 ; nv < Phys::nvar; nv++) { Flux(nv,k,j,i) = fluxR[nv] + SR*(usR[nv] - uR[nv]); } } @@ -172,4 +165,4 @@ void Hydro::HllcHD() { idfx::popRegion(); } -#endif // HYDRO_HDSOLVERS_HLLCHD_HPP_ +#endif // FLUID_RIEMANNSOLVER_HDSOLVERS_HLLCHD_HPP_ diff --git a/src/hydro/HDsolvers/roeHD.hpp b/src/fluid/RiemannSolver/HDsolvers/roeHD.hpp similarity index 78% rename from src/hydro/HDsolvers/roeHD.hpp rename to src/fluid/RiemannSolver/HDsolvers/roeHD.hpp index b6249ae6..416f9984 100644 --- a/src/hydro/HDsolvers/roeHD.hpp +++ b/src/fluid/RiemannSolver/HDsolvers/roeHD.hpp @@ -5,16 +5,17 @@ // Licensed under CeCILL 2.1 License, see COPYING for more information // *********************************************************************************** -#ifndef HYDRO_HDSOLVERS_ROEHD_HPP_ -#define HYDRO_HDSOLVERS_ROEHD_HPP_ +#ifndef FLUID_RIEMANNSOLVER_HDSOLVERS_ROEHD_HPP_ +#define FLUID_RIEMANNSOLVER_HDSOLVERS_ROEHD_HPP_ #include "../idefix.hpp" -#include "hydro.hpp" -#include "slopeLimiter.hpp" -#include "fluxHD.hpp" -#include "convertConsToPrimHD.hpp" +#include "fluid.hpp" +#include "extrapolateToFaces.hpp" +#include "flux.hpp" +#include "convertConsToPrim.hpp" #define ROE_AVERAGE 0 +#undef NMODES #if HAVE_ENERGY #define I0 0 @@ -35,9 +36,10 @@ #endif // Compute Riemann fluxes from states using ROE solver +template template -void Hydro::RoeHD() { - idfx::pushRegion("Hydro::ROE_Solver"); +void RiemannSolver::RoeHD(IdefixArray4D &Flux) { + idfx::pushRegion("RiemannSolver::ROE_Solver"); constexpr int ioffset = (DIR==IDIR) ? 1 : 0; constexpr int joffset = (DIR==JDIR) ? 1 : 0; @@ -45,16 +47,11 @@ void Hydro::RoeHD() { IdefixArray4D Vc = this->Vc; IdefixArray4D Vs = this->Vs; - IdefixArray4D Flux = this->FluxRiemann; IdefixArray3D cMax = this->cMax; - IdefixArray3D csIsoArr = this->isoSoundSpeedArray; - [[maybe_unused]] real gamma = this->gamma; - [[maybe_unused]] real gamma_m1 = this->gamma - ONE_F; - [[maybe_unused]] real csIso = this->isoSoundSpeed; - [[maybe_unused]] HydroModuleStatus haveIsoCs = this->haveIsoSoundSpeed; + EquationOfState eos = *(hydro->eos.get()); - SlopeLimiter slopeLim(Vc,data->dx[DIR],shockFlattening); + ExtrapolateToFaces extrapol = *this->GetExtrapolator(); idefix_for("ROE_Kernel", data->beg[KDIR],data->end[KDIR]+koffset, @@ -66,53 +63,58 @@ void Hydro::RoeHD() { const int Xt = (DIR == IDIR ? MX2 : MX1); , const int Xb = (DIR == KDIR ? MX2 : MX3); ) // Primitive variables - real vL[NVAR]; - real vR[NVAR]; - real dv[NVAR]; + real vL[Phys::nvar]; + real vR[Phys::nvar]; + real dv[Phys::nvar]; // Conservative variables - real uL[NVAR]; - real uR[NVAR]; + real uL[Phys::nvar]; + real uR[Phys::nvar]; // Flux (left and right) - real fluxL[NVAR]; - real fluxR[NVAR]; + real fluxL[Phys::nvar]; + real fluxR[Phys::nvar]; // Roe - real Rc[NVAR][NVAR]; - real um[NVAR]; + real Rc[Phys::nvar][Phys::nvar]; + real um[Phys::nvar]; // 1-- Store the primitive variables on the left, right, and averaged states - slopeLim.ExtrapolatePrimVar(i, j, k, vL, vR); + extrapol.ExtrapolatePrimVar(i, j, k, vL, vR); #pragma unroll - for(int nv = 0 ; nv < NVAR; nv++) { + for(int nv = 0 ; nv < Phys::nvar; nv++) { dv[nv] = vR[nv] - vL[nv]; } // --- Compute the square of the sound speed real a, a2, a2L, a2R; #if HAVE_ENERGY - a2L = gamma * vL[PRS] / vL[RHO]; - a2R = gamma * vR[PRS] / vR[RHO]; + a2L = std::sqrt(eos.GetGamma(vL[PRS],vL[RHO])*(vL[PRS]/vL[RHO])); + a2R = std::sqrt(eos.GetGamma(vR[PRS],vR[RHO])*(vR[PRS]/vR[RHO])); real h, vel2; #else - if(haveIsoCs == UserDefFunction) { - a2L = HALF_F*(csIsoArr(k,j,i)+csIsoArr(k-koffset,j-joffset,i-ioffset)); - } else { - a2L = csIso; - } - // Take the square - a2L = a2L*a2L; + a2L = HALF_F*(eos.GetWaveSpeed(k,j,i) + +eos.GetWaveSpeed(k-koffset,j-joffset,i-ioffset)); a2R = a2L; #endif + // Take the square + a2L = a2L*a2L; + a2R = a2R*a2R; // 2-- Compute the conservative variables - K_PrimToCons(uL, vL, gamma_m1); - K_PrimToCons(uR, vR, gamma_m1); + K_PrimToCons(uL, vL, &eos); + K_PrimToCons(uR, vR, &eos); // 3-- Compute the left and right fluxes - K_Flux(fluxL, vL, uL, a2L, Xn); - K_Flux(fluxR, vR, uR, a2R, Xn); + K_Flux(fluxL, vL, uL, a2L); + K_Flux(fluxR, vR, uR, a2R); + + // Compute gamma of this interface + // todo(glesur): check that it's not the internal energy that should be used there instead + #if HAVE_ENERGY + real gamma = eos.GetGamma(0.5*(vL[PRS]+vR[PRS]), 0.5*(vL[RHO]+vR[RHO])); + real gamma_m1 = gamma-1; + #endif // ---- Define Wave Jumps ---- #if ROE_AVERAGE == YES @@ -161,7 +163,7 @@ void Hydro::RoeHD() { #endif // HAVE_ENERGY #else #pragma unroll - for(int nv = 0 ; nv < NVAR; nv++) { + for(int nv = 0 ; nv < Phys::nvar; nv++) { um[nv] = HALF_F*(vR[nv]+vL[nv]); } #if HAVE_ENERGY @@ -186,9 +188,9 @@ void Hydro::RoeHD() { real eta[NMODES]; #pragma unroll - for(int nv1 = 0 ; nv1 < NVAR; nv1++) { + for(int nv1 = 0 ; nv1 < Phys::nvar; nv1++) { #pragma unroll - for(int nv2 = 0 ; nv2 < NVAR; nv2++) { + for(int nv2 = 0 ; nv2 < Phys::nvar; nv2++) { Rc[nv1][nv2] = 0; } } @@ -292,15 +294,15 @@ void Hydro::RoeHD() { #endif /*#if CHECK_ROE_MATRIX == YES - for(int nv = 0 ; nv < NVAR; nv++) { + for(int nv = 0 ; nv < Phys::nvar; nv++) { um[nv] = ZERO_F; - for(int nv1 = 0 ; nv1 < NVAR; nv1++) { - for(int nv2 = 0 ; nv2 < NVAR; nv2++) { + for(int nv1 = 0 ; nv1 < Phys::nvar; nv1++) { + for(int nv2 = 0 ; nv2 < Phys::nvar; nv2++) { um[nv] += Rc[nv][k]*(k==j)*lambda[k]*eta[j]; } } } - for(int nv = 0 ; nv < NVAR; nv++) { + for(int nv = 0 ; nv < Phys::nvar; nv++) { scrh = fluxR[nv] - fluxL[nv] - um[nv]; if (nv == Xn) scrh += pR - pL; if (FABS(scrh) > 1.e-6){ @@ -318,7 +320,7 @@ void Hydro::RoeHD() { bmax = FMAX(ZERO_F, lambda[1]); scrh1 = ONE_F/(bmax - bmin); #pragma unroll - for(int nv = 0 ; nv < NVAR; nv++) { + for(int nv = 0 ; nv < Phys::nvar; nv++) { Flux(nv,k,j,i) = bmin*bmax*(uR[nv] - uL[nv]) + bmax*fluxL[nv] - bmin*fluxR[nv]; Flux(nv,k,j,i) *= scrh1; @@ -330,7 +332,7 @@ void Hydro::RoeHD() { ----------------------------------------------------------- */ #pragma unroll - for(int nv = 0 ; nv < NVAR; nv++) { + for(int nv = 0 ; nv < Phys::nvar; nv++) { alambda[nv] = fabs(lambda[nv]); } @@ -344,10 +346,10 @@ void Hydro::RoeHD() { } #pragma unroll - for(int nv = 0 ; nv < NVAR; nv++) { + for(int nv = 0 ; nv < Phys::nvar; nv++) { Flux(nv,k,j,i) = fluxL[nv] + fluxR[nv]; #pragma unroll - for(int nv2 = 0 ; nv2 < NVAR; nv2++) { + for(int nv2 = 0 ; nv2 < Phys::nvar; nv2++) { Flux(nv,k,j,i) -= alambda[nv2]*eta[nv2]*Rc[nv][nv2]; } Flux(nv,k,j,i) *= HALF_F; @@ -362,4 +364,4 @@ void Hydro::RoeHD() { idfx::popRegion(); } -#endif // HYDRO_HDSOLVERS_ROEHD_HPP_ +#endif // FLUID_RIEMANNSOLVER_HDSOLVERS_ROEHD_HPP_ diff --git a/src/hydro/HDsolvers/tvdlfHD.hpp b/src/fluid/RiemannSolver/HDsolvers/tvdlfHD.hpp similarity index 59% rename from src/hydro/HDsolvers/tvdlfHD.hpp rename to src/fluid/RiemannSolver/HDsolvers/tvdlfHD.hpp index e4523c72..597dd0d1 100644 --- a/src/hydro/HDsolvers/tvdlfHD.hpp +++ b/src/fluid/RiemannSolver/HDsolvers/tvdlfHD.hpp @@ -5,19 +5,20 @@ // Licensed under CeCILL 2.1 License, see COPYING for more information // *********************************************************************************** -#ifndef HYDRO_HDSOLVERS_TVDLFHD_HPP_ -#define HYDRO_HDSOLVERS_TVDLFHD_HPP_ +#ifndef FLUID_RIEMANNSOLVER_HDSOLVERS_TVDLFHD_HPP_ +#define FLUID_RIEMANNSOLVER_HDSOLVERS_TVDLFHD_HPP_ #include "../idefix.hpp" -#include "hydro.hpp" -#include "slopeLimiter.hpp" -#include "fluxHD.hpp" -#include "convertConsToPrimHD.hpp" +#include "fluid.hpp" +#include "extrapolateToFaces.hpp" +#include "flux.hpp" +#include "convertConsToPrim.hpp" // Compute Riemann fluxes from states using TVDLF solver +template template -void Hydro::TvdlfHD() { - idfx::pushRegion("Hydro::TVDLF_Solver"); +void RiemannSolver::TvdlfHD(IdefixArray4D &Flux) { + idfx::pushRegion("RiemannSolver::TVDLF_Solver"); constexpr int ioffset = (DIR==IDIR) ? 1 : 0; constexpr int joffset = (DIR==JDIR) ? 1 : 0; @@ -25,19 +26,13 @@ void Hydro::TvdlfHD() { IdefixArray4D Vc = this->Vc; IdefixArray4D Vs = this->Vs; - IdefixArray4D Flux = this->FluxRiemann; IdefixArray3D cMax = this->cMax; - IdefixArray3D csIsoArr = this->isoSoundSpeedArray; + EquationOfState eos = *(hydro->eos.get()); // Required for high order interpolations IdefixArray1D dx = this->data->dx[DIR]; - real gamma = this->gamma; - [[maybe_unused]] real gamma_m1=gamma-ONE_F; - [[maybe_unused]] real csIso = this->isoSoundSpeed; - [[maybe_unused]] HydroModuleStatus haveIsoCs = this->haveIsoSoundSpeed; - - SlopeLimiter slopeLim(Vc,data->dx[DIR],shockFlattening); + ExtrapolateToFaces extrapol = *this->GetExtrapolator(); idefix_for("TVDLF_Kernel", data->beg[KDIR],data->end[KDIR]+koffset, @@ -48,53 +43,50 @@ void Hydro::TvdlfHD() { constexpr int Xn = DIR+MX1; // Primitive variables - real vL[NVAR]; - real vR[NVAR]; - real vRL[NVAR]; + real vL[Phys::nvar]; + real vR[Phys::nvar]; + real vRL[Phys::nvar]; // Conservative variables - real uL[NVAR]; - real uR[NVAR]; + real uL[Phys::nvar]; + real uR[Phys::nvar]; // Flux (left and right) - real fluxL[NVAR]; - real fluxR[NVAR]; + real fluxL[Phys::nvar]; + real fluxR[Phys::nvar]; // Signal speeds real cRL, cmax; // 1-- Read primitive variables - slopeLim.ExtrapolatePrimVar(i, j, k, vL, vR); + extrapol.ExtrapolatePrimVar(i, j, k, vL, vR); #pragma unroll - for(int nv = 0 ; nv < NVAR; nv++) { + for(int nv = 0 ; nv < Phys::nvar; nv++) { vRL[nv] = HALF_F*(vL[nv]+vR[nv]); } // 2-- Get the wave speed #if HAVE_ENERGY - cRL = std::sqrt( (gamma_m1+ONE_F)*(vRL[PRS]/vRL[RHO]) ); + cRL = std::sqrt(eos.GetGamma(vRL[PRS],vRL[RHO])*(vRL[PRS]/vRL[RHO])); #else - if(haveIsoCs == UserDefFunction) { - cRL = HALF_F*(csIsoArr(k,j,i)+csIsoArr(k-koffset,j-joffset,i-ioffset)); - } else { - cRL = csIso; - } + cRL = HALF_F*(eos.GetWaveSpeed(k,j,i) + +eos.GetWaveSpeed(k-koffset,j-joffset,i-ioffset)); #endif cmax = FMAX(FABS(vRL[Xn]+cRL),FABS(vRL[Xn]-cRL)); // 3-- Compute the conservative variables - K_PrimToCons(uL, vL, gamma_m1); - K_PrimToCons(uR, vR, gamma_m1); + K_PrimToCons(uL, vL, &eos); + K_PrimToCons(uR, vR, &eos); // 4-- Compute the left and right fluxes - K_Flux(fluxL, vL, uL, cRL*cRL, Xn); - K_Flux(fluxR, vR, uR, cRL*cRL, Xn); + K_Flux(fluxL, vL, uL, cRL*cRL); + K_Flux(fluxR, vR, uR, cRL*cRL); // 5-- Compute the flux from the left and right states #pragma unroll - for(int nv = 0 ; nv < NVAR; nv++) { + for(int nv = 0 ; nv < Phys::nvar; nv++) { Flux(nv,k,j,i) = HALF_F*(fluxL[nv]+fluxR[nv] - cmax*(uR[nv]-uL[nv])); } @@ -106,4 +98,4 @@ void Hydro::TvdlfHD() { idfx::popRegion(); } -#endif // HYDRO_HDSOLVERS_TVDLFHD_HPP_ +#endif // FLUID_RIEMANNSOLVER_HDSOLVERS_TVDLFHD_HPP_ diff --git a/src/hydro/MHDsolvers/CMakeLists.txt b/src/fluid/RiemannSolver/MHDsolvers/CMakeLists.txt similarity index 100% rename from src/hydro/MHDsolvers/CMakeLists.txt rename to src/fluid/RiemannSolver/MHDsolvers/CMakeLists.txt diff --git a/src/hydro/MHDsolvers/hllMHD.hpp b/src/fluid/RiemannSolver/MHDsolvers/hllMHD.hpp similarity index 78% rename from src/hydro/MHDsolvers/hllMHD.hpp rename to src/fluid/RiemannSolver/MHDsolvers/hllMHD.hpp index 6be3014a..77253b72 100644 --- a/src/hydro/MHDsolvers/hllMHD.hpp +++ b/src/fluid/RiemannSolver/MHDsolvers/hllMHD.hpp @@ -5,30 +5,33 @@ // Licensed under CeCILL 2.1 License, see COPYING for more information // *********************************************************************************** -#ifndef HYDRO_MHDSOLVERS_HLLMHD_HPP_ -#define HYDRO_MHDSOLVERS_HLLMHD_HPP_ +#ifndef FLUID_RIEMANNSOLVER_MHDSOLVERS_HLLMHD_HPP_ +#define FLUID_RIEMANNSOLVER_MHDSOLVERS_HLLMHD_HPP_ #include "../idefix.hpp" -#include "slopeLimiter.hpp" -#include "fluxMHD.hpp" -#include "convertConsToPrimMHD.hpp" +#include "extrapolateToFaces.hpp" +#include "flux.hpp" +#include "convertConsToPrim.hpp" #include "storeFlux.hpp" -#include "electroMotiveForce.hpp" +#include "constrainedTransport.hpp" // Compute Riemann fluxes from states using HLL solver +template template -void Hydro::HllMHD() { - idfx::pushRegion("Hydro::HLL_MHD"); +void RiemannSolver::HllMHD(IdefixArray4D &Flux) { + idfx::pushRegion("RiemannSolver::HLL_MHD"); + + using EMF = ConstrainedTransport; constexpr int ioffset = (DIR==IDIR) ? 1 : 0; constexpr int joffset = (DIR==JDIR) ? 1 : 0; constexpr int koffset = (DIR==KDIR) ? 1 : 0; int perpExtension=1; - if (emf.averaging == ElectroMotiveForce::uct_hll - || emf.averaging == ElectroMotiveForce::uct_hlld) { + if (hydro->emf->averaging == EMF::uct_hll + || hydro->emf->averaging == EMF::uct_hlld) { // Need two cells in the perp direction for these schemes - perpExtension=2; + perpExtension= data->nghost[DIR]; } // extension in perp to the direction of integration, as required by CT. const int iextend = (DIR==IDIR) ? 0 : perpExtension; @@ -45,13 +48,11 @@ void Hydro::HllMHD() { IdefixArray4D Vc = this->Vc; IdefixArray4D Vs = this->Vs; - IdefixArray4D Flux = this->FluxRiemann; IdefixArray3D cMax = this->cMax; - IdefixArray3D csIsoArr = this->isoSoundSpeedArray; - HydroModuleStatus haveHall = this->hallStatus.status; - IdefixArray4D J = this->J; - IdefixArray3D xHallArr = this->xHall; + HydroModuleStatus haveHall = hydro->hallStatus.status; + IdefixArray4D J = hydro->J; + IdefixArray3D xHallArr = hydro->xHall; IdefixArray1D dx = data->dx[DIR]; IdefixArray1D dx2 = data->dx[JDIR]; IdefixArray1D x1 = data->x[IDIR]; @@ -62,7 +63,7 @@ void Hydro::HllMHD() { IdefixArray3D Eb; IdefixArray3D Et; - const ElectroMotiveForce::AveragingType emfAverage = emf.averaging; + const typename EMF::AveragingType emfAverage = hydro->emf->averaging; // Required by UCT_Contact IdefixArray3D SV; @@ -73,13 +74,11 @@ void Hydro::HllMHD() { IdefixArray3D dL; IdefixArray3D dR; - real gamma = this->gamma; - [[maybe_unused]] real xHConstant = this->xH; - [[maybe_unused]] real gamma_m1=gamma-ONE_F; - [[maybe_unused]] real csIso = this->isoSoundSpeed; - [[maybe_unused]] HydroModuleStatus haveIsoCs = this->haveIsoSoundSpeed; + EquationOfState eos = *(hydro->eos.get()); + + [[maybe_unused]] real xHConstant = hydro->xH; - SlopeLimiter slopeLim(Vc,data->dx[DIR],shockFlattening); + ExtrapolateToFaces extrapol = *this->GetExtrapolator(); // Define normal, tangent and bi-tanget indices // st and sb will be useful only when Hall is included @@ -93,16 +92,16 @@ void Hydro::HllMHD() { sb = +ONE_F; ) - Et = this->emf.ezi; - Eb = this->emf.eyi; + Et = hydro->emf->ezi; + Eb = hydro->emf->eyi; - SV = this->emf.svx; + SV = hydro->emf->svx; - aL = this->emf.axL; - aR = this->emf.axR; + aL = hydro->emf->axL; + aR = hydro->emf->axR; - dL = this->emf.dxL; - dR = this->emf.dxR; + dL = hydro->emf->dxL; + dR = hydro->emf->dxR; break; #if DIMENSIONS >= 2 @@ -113,16 +112,16 @@ void Hydro::HllMHD() { sb = -ONE_F; ) - Et = this->emf.ezj; - Eb = this->emf.exj; + Et = hydro->emf->ezj; + Eb = hydro->emf->exj; - SV = this->emf.svy; + SV = hydro->emf->svy; - aL = this->emf.ayL; - aR = this->emf.ayR; + aL = hydro->emf->ayL; + aR = hydro->emf->ayR; - dL = this->emf.dyL; - dR = this->emf.dyR; + dL = hydro->emf->dyL; + dR = hydro->emf->dyR; break; #endif @@ -135,16 +134,16 @@ void Hydro::HllMHD() { , sb = +ONE_F; ) - Et = this->emf.eyk; - Eb = this->emf.exk; + Et = hydro->emf->eyk; + Eb = hydro->emf->exk; - SV = this->emf.svz; + SV = hydro->emf->svz; - aL = this->emf.azL; - aR = this->emf.azR; + aL = hydro->emf->azL; + aR = hydro->emf->azR; - dL = this->emf.dzL; - dR = this->emf.dzR; + dL = hydro->emf->dzL; + dR = hydro->emf->dzR; break; #endif default: @@ -158,25 +157,22 @@ void Hydro::HllMHD() { data->beg[IDIR]-iextend,data->end[IDIR]+ioffset+iextend, KOKKOS_LAMBDA (int k, int j, int i) { // Init the directions (should be in the kernel for proper optimisation by the compilers) - EXPAND( const int Xn = DIR+MX1; , - const int Xt = (DIR == IDIR ? MX2 : MX1); , - const int Xb = (DIR == KDIR ? MX2 : MX3); ) - + const int Xn = DIR+MX1; EXPAND( const int BXn = DIR+BX1; , const int BXt = (DIR == IDIR ? BX2 : BX1); , const int BXb = (DIR == KDIR ? BX2 : BX3); ) // Primitive variables - real vL[NVAR]; - real vR[NVAR]; + real vL[Phys::nvar]; + real vR[Phys::nvar]; // Conservative variables - real uL[NVAR]; - real uR[NVAR]; + real uL[Phys::nvar]; + real uR[Phys::nvar]; // Flux (left and right) - real fluxL[NVAR]; - real fluxR[NVAR]; + real fluxL[Phys::nvar]; + real fluxR[Phys::nvar]; // Signal speeds real cL, cR, cmax, c2Iso; @@ -184,7 +180,7 @@ void Hydro::HllMHD() { c2Iso = ZERO_F; // 1-- Store the primitive variables on the left, right, and averaged states - slopeLim.ExtrapolatePrimVar(i, j, k, vL, vR); + extrapol.ExtrapolatePrimVar(i, j, k, vL, vR); vL[BXn] = Vs(DIR,k,j,i); vR[BXn] = vL[BXn]; @@ -192,14 +188,12 @@ void Hydro::HllMHD() { real gpr, b1, b2, b3, Btmag2, Bmag2; real xH; #if HAVE_ENERGY + real gamma = eos.GetGamma(0.5*(vL[PRS]+vR[PRS]),0.5*(vL[RHO]+vR[RHO])); gpr = gamma*vL[PRS]; #else - if(haveIsoCs == UserDefFunction) { - c2Iso = HALF_F*(csIsoArr(k,j,i)+csIsoArr(k-koffset,j-joffset,i-ioffset)); - c2Iso = c2Iso*c2Iso; - } else { - c2Iso = csIso*csIso; - } + c2Iso = HALF_F*(eos.GetWaveSpeed(k,j,i) + +eos.GetWaveSpeed(k-koffset,j-joffset,i-ioffset)); + c2Iso *= c2Iso; gpr = c2Iso*vL[RHO]; #endif @@ -283,13 +277,13 @@ void Hydro::HllMHD() { cmax = FMAX(FABS(SLb), FABS(SRb)); // 2-- Compute the conservative variables - K_PrimToCons(uL, vL, gamma_m1); - K_PrimToCons(uR, vR, gamma_m1); + K_PrimToCons(uL, vL, &eos); + K_PrimToCons(uR, vR, &eos); #pragma unroll - for(int nv = 0 ; nv < NVAR; nv++) { + for(int nv = 0 ; nv < Phys::nvar; nv++) { fluxL[nv] = uL[nv]; fluxR[nv] = uR[nv]; } @@ -297,8 +291,8 @@ void Hydro::HllMHD() { // 3-- Compute the left and right fluxes - K_Flux(fluxL, vL, fluxL, c2Iso, ARG_EXPAND(Xn, Xt, Xb), ARG_EXPAND(BXn, BXt, BXb)); - K_Flux(fluxR, vR, fluxR, c2Iso, ARG_EXPAND(Xn, Xt, Xb), ARG_EXPAND(BXn, BXt, BXb)); + K_Flux(fluxL, vL, fluxL, c2Iso); + K_Flux(fluxR, vR, fluxR, c2Iso); // 4-- Compute the Hall flux if(haveHall) { @@ -413,20 +407,30 @@ void Hydro::HllMHD() { if (SLb > 0) { #pragma unroll - for (int nv = BX1 ; nv < NFLX; nv++) { + for (int nv = BX1 ; nv < BX1+COMPONENTS; nv++) { Flux(nv,k,j,i) = fluxL[nv]; } + if constexpr(Phys::pressure) { + Flux(ENG,k,j,i) = fluxL[ENG]; + } } else if (SRb < 0) { #pragma unroll - for (int nv = BX1 ; nv < NFLX; nv++) { + for (int nv = BX1 ; nv < BX1+COMPONENTS; nv++) { Flux(nv,k,j,i) = fluxR[nv]; } + if constexpr(Phys::pressure) { + Flux(ENG,k,j,i) = fluxR[ENG]; + } } else { #pragma unroll - for(int nv = BX1 ; nv < NFLX; nv++) { + for(int nv = BX1 ; nv < BX1+COMPONENTS; nv++) { Flux(nv,k,j,i) = SLb*SRb*uR[nv] - SLb*SRb*uL[nv] + SRb*fluxL[nv] - SLb*fluxR[nv]; Flux(nv,k,j,i) *= (1.0 / (SRb - SLb)); } + if constexpr(Phys::pressure) { + Flux(ENG,k,j,i) = SLb*SRb*uR[ENG] - SLb*SRb*uL[ENG] + SRb*fluxL[ENG] - SLb*fluxR[ENG]; + Flux(ENG,k,j,i) *= (1.0 / (SRb - SLb)); + } } @@ -434,15 +438,15 @@ void Hydro::HllMHD() { cMax(k,j,i) = cmax; // 7-- Store the flux in the emf components - if (emfAverage==ElectroMotiveForce::arithmetic - || emfAverage==ElectroMotiveForce::uct0) { + if (emfAverage==EMF::arithmetic + || emfAverage==EMF::uct0) { K_StoreEMF(i,j,k,st,sb,Flux,Et,Eb); - } else if (emfAverage==ElectroMotiveForce::uct_contact) { + } else if (emfAverage==EMF::uct_contact) { K_StoreContact(i,j,k,st,sb,Flux,Et,Eb,SV); - } else if (emfAverage==ElectroMotiveForce::uct_hll) { + } else if (emfAverage==EMF::uct_hll) { K_StoreHLL(i,j,k,st,sb,sl,sr,vL,vR,Et,Eb,aL,aR,dL,dR); } - /* else if (emfAverage==ElectroMotiveForce::uct_hlld) { + /* else if (emfAverage==EMF::uct_hlld) { // We do not have the Alfven speed in the HLL solver K_StoreHLLD(i,j,k,st,sb,c2Iso,SLb,SRb,vL,vR,uL,uR,Et,Eb,aL,aR,dL,dR); }*/ @@ -451,4 +455,4 @@ void Hydro::HllMHD() { idfx::popRegion(); } -#endif // HYDRO_MHDSOLVERS_HLLMHD_HPP_ +#endif // FLUID_RIEMANNSOLVER_MHDSOLVERS_HLLMHD_HPP_ diff --git a/src/hydro/MHDsolvers/hlldMHD.hpp b/src/fluid/RiemannSolver/MHDsolvers/hlldMHD.hpp similarity index 83% rename from src/hydro/MHDsolvers/hlldMHD.hpp rename to src/fluid/RiemannSolver/MHDsolvers/hlldMHD.hpp index 3852db46..fa2491c5 100644 --- a/src/hydro/MHDsolvers/hlldMHD.hpp +++ b/src/fluid/RiemannSolver/MHDsolvers/hlldMHD.hpp @@ -5,30 +5,35 @@ // Licensed under CeCILL 2.1 License, see COPYING for more information // *********************************************************************************** -#ifndef HYDRO_MHDSOLVERS_HLLDMHD_HPP_ -#define HYDRO_MHDSOLVERS_HLLDMHD_HPP_ +#ifndef FLUID_RIEMANNSOLVER_MHDSOLVERS_HLLDMHD_HPP_ +#define FLUID_RIEMANNSOLVER_MHDSOLVERS_HLLDMHD_HPP_ #include "../idefix.hpp" -#include "slopeLimiter.hpp" -#include "fluxMHD.hpp" -#include "convertConsToPrimMHD.hpp" +#include "extrapolateToFaces.hpp" +#include "flux.hpp" +#include "convertConsToPrim.hpp" #include "storeFlux.hpp" -#include "electroMotiveForce.hpp" +#include "constrainedTransport.hpp" + + // Compute Riemann fluxes from states using HLLD solver +template template -void Hydro::HlldMHD() { - idfx::pushRegion("Hydro::HLLD_MHD"); +void RiemannSolver::HlldMHD(IdefixArray4D &Flux) { + idfx::pushRegion("RiemannSolver::HLLD_MHD"); + + using EMF = ConstrainedTransport; constexpr int ioffset = (DIR==IDIR) ? 1 : 0; constexpr int joffset = (DIR==JDIR) ? 1 : 0; constexpr int koffset = (DIR==KDIR) ? 1 : 0; int perpExtension=1; - if (emf.averaging == ElectroMotiveForce::uct_hll - || emf.averaging == ElectroMotiveForce::uct_hlld) { + if (hydro->emf->averaging == EMF::uct_hll + || hydro->emf->averaging == EMF::uct_hlld) { // Need two cells in the perp direction for these schemes - perpExtension=2; + perpExtension= data->nghost[DIR]; } // extension in perp to the direction of integration, as required by CT. const int iextend = (DIR==IDIR) ? 0 : perpExtension; @@ -45,9 +50,7 @@ void Hydro::HlldMHD() { IdefixArray4D Vc = this->Vc; IdefixArray4D Vs = this->Vs; - IdefixArray4D Flux = this->FluxRiemann; IdefixArray3D cMax = this->cMax; - IdefixArray3D csIsoArr = this->isoSoundSpeedArray; // Required for high order interpolations IdefixArray1D dx = this->data->dx[DIR]; @@ -56,7 +59,8 @@ void Hydro::HlldMHD() { IdefixArray3D Eb; IdefixArray3D Et; - const ElectroMotiveForce::AveragingType emfAverage = emf.averaging; + + const typename EMF::AveragingType emfAverage = hydro->emf->averaging; // Required by UCT_Contact IdefixArray3D SV; @@ -67,12 +71,9 @@ void Hydro::HlldMHD() { IdefixArray3D dL; IdefixArray3D dR; - real gamma = this->gamma; - [[maybe_unused]] real gamma_m1 = gamma-ONE_F; - [[maybe_unused]] real csIso = this->isoSoundSpeed; - [[maybe_unused]] HydroModuleStatus haveIsoCs = this->haveIsoSoundSpeed; + EquationOfState eos = *(hydro->eos.get()); - SlopeLimiter slopeLim(Vc,data->dx[DIR],shockFlattening); + ExtrapolateToFaces extrapol = *this->GetExtrapolator(); // st and sb will be useful only when Hall is included real st = ONE_F, sb = ONE_F; @@ -85,16 +86,16 @@ void Hydro::HlldMHD() { sb = +ONE_F; ) - Et = this->emf.ezi; - Eb = this->emf.eyi; + Et = hydro->emf->ezi; + Eb = hydro->emf->eyi; - SV = this->emf.svx; + SV = hydro->emf->svx; - aL = this->emf.axL; - aR = this->emf.axR; + aL = hydro->emf->axL; + aR = hydro->emf->axR; - dL = this->emf.dxL; - dR = this->emf.dxR; + dL = hydro->emf->dxL; + dR = hydro->emf->dxR; break; #if DIMENSIONS >= 2 @@ -105,16 +106,16 @@ void Hydro::HlldMHD() { sb = -ONE_F; ) - Et = this->emf.ezj; - Eb = this->emf.exj; + Et = hydro->emf->ezj; + Eb = hydro->emf->exj; - SV = this->emf.svy; + SV = hydro->emf->svy; - aL = this->emf.ayL; - aR = this->emf.ayR; + aL = hydro->emf->ayL; + aR = hydro->emf->ayR; - dL = this->emf.dyL; - dR = this->emf.dyR; + dL = hydro->emf->dyL; + dR = hydro->emf->dyR; break; #endif @@ -127,16 +128,16 @@ void Hydro::HlldMHD() { , sb = +ONE_F; ) - Et = this->emf.eyk; - Eb = this->emf.exk; + Et = hydro->emf->eyk; + Eb = hydro->emf->exk; - SV = this->emf.svz; + SV = hydro->emf->svz; - aL = this->emf.azL; - aR = this->emf.azR; + aL = hydro->emf->azL; + aR = hydro->emf->azR; - dL = this->emf.dzL; - dR = this->emf.dzR; + dL = hydro->emf->dzL; + dR = hydro->emf->dzR; break; #endif default: @@ -149,29 +150,29 @@ void Hydro::HlldMHD() { data->beg[IDIR]-iextend,data->end[IDIR]+ioffset+iextend, KOKKOS_LAMBDA (int k, int j, int i) { // Init the directions (should be in the kernel for proper optimisation by the compilers) - EXPAND( const int Xn = DIR+MX1; , - const int Xt = (DIR == IDIR ? MX2 : MX1); , - const int Xb = (DIR == KDIR ? MX2 : MX3); ) + EXPAND( constexpr int Xn = DIR+MX1; , + constexpr int Xt = (DIR == IDIR ? MX2 : MX1); , + constexpr int Xb = (DIR == KDIR ? MX2 : MX3); ) - EXPAND( const int BXn = DIR+BX1; , - const int BXt = (DIR == IDIR ? BX2 : BX1); , - const int BXb = (DIR == KDIR ? BX2 : BX3); ) + EXPAND( constexpr int BXn = DIR+BX1; , + constexpr int BXt = (DIR == IDIR ? BX2 : BX1); , + constexpr int BXb = (DIR == KDIR ? BX2 : BX3); ) // Primitive variables - real vL[NVAR]; - real vR[NVAR]; + real vL[Phys::nvar]; + real vR[Phys::nvar]; - slopeLim.ExtrapolatePrimVar(i, j, k, vL, vR); + extrapol.ExtrapolatePrimVar(i, j, k, vL, vR); vL[BXn] = Vs(DIR,k,j,i); vR[BXn] = vL[BXn]; // Conservative variables - real uL[NVAR]; - real uR[NVAR]; + real uL[Phys::nvar]; + real uR[Phys::nvar]; // Flux (left and right) - real fluxL[NVAR]; - real fluxR[NVAR]; + real fluxL[Phys::nvar]; + real fluxR[Phys::nvar]; // Signal speeds real cL, cR, cmax, c2Iso; @@ -182,14 +183,12 @@ void Hydro::HlldMHD() { // 2-- Get the wave speed real gpr, b1, b2, b3, Btmag2, Bmag2; #if HAVE_ENERGY + real gamma = eos.GetGamma(0.5*(vL[PRS]+vR[PRS]),0.5*(vL[RHO]+vR[RHO])); gpr = gamma*vL[PRS]; #else - if(haveIsoCs == UserDefFunction) { - c2Iso = HALF_F*(csIsoArr(k,j,i)+csIsoArr(k-koffset,j-joffset,i-ioffset)); - c2Iso = c2Iso*c2Iso; - } else { - c2Iso = csIso*csIso; - } + c2Iso = HALF_F*(eos.GetWaveSpeed(k,j,i) + +eos.GetWaveSpeed(k-koffset,j-joffset,i-ioffset)); + c2Iso *= c2Iso; gpr = c2Iso*vL[RHO]; #endif @@ -239,18 +238,18 @@ void Hydro::HlldMHD() { cmax = std::fmax(FABS(sl), FABS(sr)); // 2-- Compute the conservative variables - K_PrimToCons(uL, vL, gamma_m1); - K_PrimToCons(uR, vR, gamma_m1); + K_PrimToCons(uL, vL, &eos); + K_PrimToCons(uR, vR, &eos); // 3-- Compute the left and right fluxes #pragma unroll - for(int nv = 0 ; nv < NVAR; nv++) { + for(int nv = 0 ; nv < Phys::nvar; nv++) { fluxL[nv] = uL[nv]; fluxR[nv] = uR[nv]; } - K_Flux(fluxL, vL, fluxL, c2Iso, ARG_EXPAND(Xn, Xt, Xb), ARG_EXPAND(BXn, BXt, BXb)); - K_Flux(fluxR, vR, fluxR, c2Iso, ARG_EXPAND(Xn, Xt, Xb), ARG_EXPAND(BXn, BXt, BXb)); + K_Flux(fluxL, vL, fluxL, c2Iso); + K_Flux(fluxR, vR, fluxR, c2Iso); [[maybe_unused]] int revert_to_hll = 0, revert_to_hllc = 0; @@ -266,22 +265,22 @@ void Hydro::HlldMHD() { // 5-- Compute the flux from the left and right states if (sl > 0) { #pragma unroll - for (int nv = 0 ; nv < NFLX; nv++) { + for (int nv = 0 ; nv < Phys::nvar; nv++) { Flux(nv,k,j,i) = fluxL[nv]; } } else if (sr < 0) { #pragma unroll - for (int nv = 0 ; nv < NFLX; nv++) { + for (int nv = 0 ; nv < Phys::nvar; nv++) { Flux(nv,k,j,i) = fluxR[nv]; } } else { - real usL[NVAR]; - real usR[NVAR]; + real usL[Phys::nvar]; + real usR[Phys::nvar]; real scrh, scrhL, scrhR, duL, duR, sBx, Bx, SM, S1L, S1R; #if HAVE_ENERGY - real Uhll[NVAR]; + real Uhll[Phys::nvar]; real pts, sqrL, sqrR; [[maybe_unused]] real vsL, vsR, wsL, wsR; @@ -327,7 +326,7 @@ void Hydro::HlldMHD() { if (revert_to_hllc) { scrh = ONE_F/(sr - sl); #pragma unroll - for(int nv = 0 ; nv < NVAR; nv++) { + for(int nv = 0 ; nv < Phys::nvar; nv++) { Uhll[nv] = sr*uR[nv] - sl*uL[nv] + fluxL[nv] - fluxR[nv]; Uhll[nv] *= scrh; } @@ -388,19 +387,19 @@ void Hydro::HlldMHD() { if (S1L >= 0.0) { // ---- Region L* #pragma unroll - for(int nv = 0 ; nv < NVAR; nv++) { + for(int nv = 0 ; nv < Phys::nvar; nv++) { Flux(nv,k,j,i) = fluxL[nv] + sl*(usL[nv] - uL[nv]); } } else if (S1R <= 0.0) { // ---- Region R* #pragma unroll - for(int nv = 0 ; nv < NVAR; nv++) { + for(int nv = 0 ; nv < Phys::nvar; nv++) { Flux(nv,k,j,i) = fluxR[nv] + sr*(usR[nv] - uR[nv]); } } else { // -- This state exists only if B_x != 0 // Compute U** [[maybe_unused]]real vss, wss; - real ussl[NVAR]; - real ussr[NVAR]; + real ussl[Phys::nvar]; + real ussr[Phys::nvar]; ussl[RHO] = usL[RHO]; ussr[RHO] = usR[RHO]; @@ -445,20 +444,20 @@ void Hydro::HlldMHD() { if (SM >= 0.0) { // ---- Region L** #pragma unroll - for(int nv = 0 ; nv < NVAR; nv++) { + for(int nv = 0 ; nv < Phys::nvar; nv++) { Flux(nv,k,j,i) = fluxL[nv] + S1L*(ussl[nv] - usL[nv]) + sl*(usL[nv] - uL[nv]); } } else { // ---- Region R** #pragma unroll - for(int nv = 0 ; nv < NVAR; nv++) { + for(int nv = 0 ; nv < Phys::nvar; nv++) { Flux(nv,k,j,i) = fluxR[nv] + S1R*(ussr[nv] - usR[nv]) + sr*(usR[nv] - uR[nv]); } } } // end if (S1L < 0 S1R > 0) #else // No ENERGY - real usc[NVAR]; + real usc[Phys::nvar]; real rho, sqrho; scrh = ONE_F/(sr - sl); @@ -491,7 +490,7 @@ void Hydro::HlldMHD() { if (revert_to_hll) { scrh = ONE_F/(sr - sl); #pragma unroll - for(int nv = 0 ; nv < NVAR; nv++) { + for(int nv = 0 ; nv < Phys::nvar; nv++) { Flux(nv,k,j,i) = sl*sr*(uR[nv] - uL[nv]) + sr*fluxL[nv] - sl*fluxR[nv]; Flux(nv,k,j,i) *= scrh; @@ -577,18 +576,18 @@ void Hydro::HlldMHD() { cMax(k,j,i) = cmax; // 7-- Store the flux in the emf components - if (emfAverage==ElectroMotiveForce::arithmetic - || emfAverage==ElectroMotiveForce::uct0) { + if (emfAverage==EMF::arithmetic + || emfAverage==EMF::uct0) { K_StoreEMF(i,j,k,st,sb,Flux,Et,Eb); - } else if (emfAverage==ElectroMotiveForce::uct_contact) { + } else if (emfAverage==EMF::uct_contact) { K_StoreContact(i,j,k,st,sb,Flux,Et,Eb,SV); - } else if (emfAverage==ElectroMotiveForce::uct_hll) { + } else if (emfAverage==EMF::uct_hll) { K_StoreHLL(i,j,k,st,sb,sl,sr,vL,vR,Et,Eb,aL,aR,dL,dR); - } else if (emfAverage==ElectroMotiveForce::uct_hlld) { + } else if (emfAverage==EMF::uct_hlld) { K_StoreHLLD(i,j,k,st,sb,c2Iso,sl,sr,vL,vR,uL,uR,Et,Eb,aL,aR,dL,dR); } }); idfx::popRegion(); } -#endif // HYDRO_MHDSOLVERS_HLLDMHD_HPP_ +#endif // FLUID_RIEMANNSOLVER_MHDSOLVERS_HLLDMHD_HPP_ diff --git a/src/hydro/MHDsolvers/roeMHD.hpp b/src/fluid/RiemannSolver/MHDsolvers/roeMHD.hpp similarity index 85% rename from src/hydro/MHDsolvers/roeMHD.hpp rename to src/fluid/RiemannSolver/MHDsolvers/roeMHD.hpp index 086e4474..633c3ebd 100644 --- a/src/hydro/MHDsolvers/roeMHD.hpp +++ b/src/fluid/RiemannSolver/MHDsolvers/roeMHD.hpp @@ -5,17 +5,18 @@ // Licensed under CeCILL 2.1 License, see COPYING for more information // *********************************************************************************** -#ifndef HYDRO_MHDSOLVERS_ROEMHD_HPP_ -#define HYDRO_MHDSOLVERS_ROEMHD_HPP_ +#ifndef FLUID_RIEMANNSOLVER_MHDSOLVERS_ROEMHD_HPP_ +#define FLUID_RIEMANNSOLVER_MHDSOLVERS_ROEMHD_HPP_ #include "../idefix.hpp" -#include "slopeLimiter.hpp" -#include "fluxMHD.hpp" -#include "convertConsToPrimMHD.hpp" +#include "extrapolateToFaces.hpp" +#include "flux.hpp" +#include "convertConsToPrim.hpp" #include "storeFlux.hpp" -#include "electroMotiveForce.hpp" +#include "constrainedTransport.hpp" #define ROE_AVERAGE 0 +#undef NMODES #define KFASTM 0 #define KFASTP 1 @@ -47,9 +48,12 @@ #define DSIGN(x) ( (x) >= 0.0 ? (1.0) : (-1.0)) // Compute Riemann fluxes from states using ROE solver +template template -void Hydro::RoeMHD() { - idfx::pushRegion("Hydro::ROE_MHD"); +void RiemannSolver::RoeMHD(IdefixArray4D &Flux) { + idfx::pushRegion("RiemannSolver::ROE_MHD"); + + using EMF = ConstrainedTransport; constexpr int ioffset = (DIR==IDIR) ? 1 : 0; constexpr int joffset = (DIR==JDIR) ? 1 : 0; @@ -70,9 +74,7 @@ void Hydro::RoeMHD() { IdefixArray4D Vc = this->Vc; IdefixArray4D Vs = this->Vs; - IdefixArray4D Flux = this->FluxRiemann; IdefixArray3D cMax = this->cMax; - IdefixArray3D csIsoArr = this->isoSoundSpeedArray; // Required for high order interpolations IdefixArray1D dx = this->data->dx[DIR]; @@ -81,7 +83,7 @@ void Hydro::RoeMHD() { IdefixArray3D Eb; IdefixArray3D Et; - const ElectroMotiveForce::AveragingType emfAverage = emf.averaging; + const typename EMF::AveragingType emfAverage = hydro->emf->averaging; // Required by UCT_Contact @@ -93,10 +95,7 @@ void Hydro::RoeMHD() { IdefixArray3D dL; IdefixArray3D dR; - real gamma = this->gamma; - [[maybe_unused]] real gamma_m1=gamma-ONE_F; - [[maybe_unused]] real csIso = this->isoSoundSpeed; - [[maybe_unused]] HydroModuleStatus haveIsoCs = this->haveIsoSoundSpeed; + EquationOfState eos = *(hydro->eos.get()); // TODO(baghdads) what is this delta? real delta = 1.e-6; @@ -105,7 +104,7 @@ void Hydro::RoeMHD() { // st and sb will be useful only when Hall is included real st = ONE_F, sb = ONE_F; - SlopeLimiter slopeLim(Vc,data->dx[DIR],shockFlattening); + ExtrapolateToFaces extrapol = *this->GetExtrapolator(); switch(DIR) { case(IDIR): @@ -115,16 +114,16 @@ void Hydro::RoeMHD() { sb = +ONE_F; ) - Et = this->emf.ezi; - Eb = this->emf.eyi; + Et = hydro->emf->ezi; + Eb = hydro->emf->eyi; - SV = this->emf.svx; + SV = hydro->emf->svx; - aL = this->emf.axL; - aR = this->emf.axR; + aL = hydro->emf->axL; + aR = hydro->emf->axR; - dL = this->emf.dxL; - dR = this->emf.dxR; + dL = hydro->emf->dxL; + dR = hydro->emf->dxR; break; #if DIMENSIONS >= 2 @@ -135,16 +134,16 @@ void Hydro::RoeMHD() { sb = -ONE_F; ) - Et = this->emf.ezj; - Eb = this->emf.exj; + Et = hydro->emf->ezj; + Eb = hydro->emf->exj; - SV = this->emf.svy; + SV = hydro->emf->svy; - aL = this->emf.ayL; - aR = this->emf.ayR; + aL = hydro->emf->ayL; + aR = hydro->emf->ayR; - dL = this->emf.dyL; - dR = this->emf.dyR; + dL = hydro->emf->dyL; + dR = hydro->emf->dyR; break; #endif @@ -157,16 +156,16 @@ void Hydro::RoeMHD() { , sb = +ONE_F; ) - Et = this->emf.eyk; - Eb = this->emf.exk; + Et = hydro->emf->eyk; + Eb = hydro->emf->exk; - SV = this->emf.svz; + SV = hydro->emf->svz; - aL = this->emf.azL; - aR = this->emf.azR; + aL = hydro->emf->azL; + aR = hydro->emf->azR; - dL = this->emf.dzL; - dR = this->emf.dzR; + dL = hydro->emf->dzL; + dR = hydro->emf->dzR; break; #endif default: @@ -188,36 +187,36 @@ void Hydro::RoeMHD() { const int BXb = (DIR == KDIR ? BX2 : BX3); ) // Primitive variables - real vL[NVAR]; - real vR[NVAR]; - real dV[NVAR]; + real vL[Phys::nvar]; + real vR[Phys::nvar]; + real dV[Phys::nvar]; // Conservative variables - real uL[NVAR]; - real uR[NVAR]; - [[maybe_unused]] real dU[NVAR]; + real uL[Phys::nvar]; + real uR[Phys::nvar]; + [[maybe_unused]] real dU[Phys::nvar]; // Flux (left and right) - real fluxL[NVAR]; - real fluxR[NVAR]; + real fluxL[Phys::nvar]; + real fluxR[Phys::nvar]; // Roe - real Rc[NVAR][NVAR]; + real Rc[Phys::nvar][Phys::nvar]; // 1-- Store the primitive variables on the left, right, and averaged states - slopeLim.ExtrapolatePrimVar(i, j, k, vL, vR); + extrapol.ExtrapolatePrimVar(i, j, k, vL, vR); vL[BXn] = Vs(DIR,k,j,i); vR[BXn] = vL[BXn]; #pragma unroll - for(int nv = 0 ; nv < NVAR; nv++) { + for(int nv = 0 ; nv < Phys::nvar; nv++) { dV[nv] = vR[nv] - vL[nv]; } // 2-- Compute the conservative variables - K_PrimToCons(uL, vL, gamma_m1); - K_PrimToCons(uR, vR, gamma_m1); + K_PrimToCons(uL, vL, &eos); + K_PrimToCons(uR, vR, &eos); // --- Compute the square of the sound speed real a, a2, a2L, a2R; @@ -225,31 +224,29 @@ void Hydro::RoeMHD() { // These are actually not used, but are initialised to avoid warnings a2L = ONE_F; a2R = ONE_F; + real gamma = eos.GetGamma(0.5*(vL[RHO]+vR[RHO]),0.5*(vL[PRS]+vR[PRS])); #else - if(haveIsoCs == UserDefFunction) { - a2L = HALF_F*(csIsoArr(k,j,i)+csIsoArr(k-koffset,j-joffset,i-ioffset)); - } else { - a2L = csIso; - } - a2L = a2L*a2L; - a2R = a2L; + a2L = HALF_F*(eos.GetWaveSpeed(k,j,i) + +eos.GetWaveSpeed(k-koffset,j-joffset,i-ioffset)); + a2L = a2L*a2L; + a2R = a2L; #endif // 3-- Compute the left and right fluxes #pragma unroll - for(int nv = 0 ; nv < NVAR; nv++) { + for(int nv = 0 ; nv < Phys::nvar; nv++) { fluxL[nv] = uL[nv]; fluxR[nv] = uR[nv]; dU[nv] = uR[nv] - uL[nv]; } - K_Flux(fluxL, vL, fluxL, a2L, ARG_EXPAND(Xn, Xt, Xb), ARG_EXPAND(BXn, BXt, BXb)); - K_Flux(fluxR, vR, fluxR, a2R, ARG_EXPAND(Xn, Xt, Xb), ARG_EXPAND(BXn, BXt, BXb)); + K_Flux(fluxL, vL, fluxL, a2L); + K_Flux(fluxR, vR, fluxR, a2R); // 5. Set eigenvectors components Rc = 0 initially #pragma unroll - for(int nv1 = 0 ; nv1 < NVAR; nv1++) { + for(int nv1 = 0 ; nv1 < Phys::nvar; nv1++) { #pragma unroll - for(int nv2 = 0 ; nv2 < NVAR; nv2++) { + for(int nv2 = 0 ; nv2 < Phys::nvar; nv2++) { Rc[nv1][nv2] = 0; } } @@ -313,7 +310,7 @@ void Hydro::RoeMHD() { BdB = EXPAND(Bx*dU[BXn], + By*dU[BXt], + Bz*dU[BXb]); vel2 = EXPAND(u*u, + v*v, + w*w); - dV[PRS] = gamma_m1*((0.5*vel2 - X)*dV[RHO] - vdm + dU[ENG] - BdB); + dV[PRS] = (gamma-1.0)*((0.5*vel2 - X)*dV[RHO] - vdm + dU[ENG] - BdB); HL = (uL[ENG] + pL)/vL[RHO]; HR = (uR[ENG] + pR)/vR[RHO]; @@ -321,7 +318,7 @@ void Hydro::RoeMHD() { Hgas = H - b2; // gas enthalpy - a2 = (2.0 - gamma)*X + gamma_m1*(Hgas - 0.5*vel2); + a2 = (2.0 - gamma)*X + (gamma-1.0)*(Hgas - 0.5*vel2); if (a2 < 0.0) { //IDEFIX_ERROR("! Roe_Solver(): a2 < 0.0 !! \n"); } @@ -477,7 +474,7 @@ void Hydro::RoeMHD() { EXPAND( Rc[Xn][kk] = u; , Rc[Xt][kk] = v; , Rc[Xb][kk] = w; ) - Rc[ENG][kk] = 0.5*vel2 + (gamma - 2.0)/gamma_m1*X; + Rc[ENG][kk] = 0.5*vel2 + (gamma - 2.0)/(gamma-1.0)*X; eta[kk] = ((a2 - X)*dV[RHO] - dV[PRS])/a2; #endif @@ -604,7 +601,7 @@ void Hydro::RoeMHD() { sr = lambda[KFASTP]; #pragma unroll - for(int nv = 0 ; nv < NVAR; nv++) { + for(int nv = 0 ; nv < Phys::nvar; nv++) { alambda[nv] = fabs(lambda[nv]); } @@ -627,10 +624,10 @@ void Hydro::RoeMHD() { // 6j. Compute Roe numerical flux #pragma unroll - for(int nv1 = 0 ; nv1 < NFLX; nv1++) { + for(int nv1 = 0 ; nv1 < Phys::nvar; nv1++) { scrh = 0.0; #pragma unroll - for(int nv2 = 0 ; nv2 < NFLX; nv2++) { + for(int nv2 = 0 ; nv2 < Phys::nvar; nv2++) { scrh += alambda[nv2]*eta[nv2]*Rc[nv1][nv2]; } Flux(nv1,k,j,i) = 0.5*(fluxL[nv1] + fluxR[nv1] - scrh); @@ -640,14 +637,14 @@ void Hydro::RoeMHD() { cMax(k,j,i) = cmax; // 7-- Store the flux in the emf components - if (emfAverage==ElectroMotiveForce::arithmetic - || emfAverage==ElectroMotiveForce::uct0) { + if (emfAverage==EMF::arithmetic + || emfAverage==EMF::uct0) { K_StoreEMF(i,j,k,st,sb,Flux,Et,Eb); - } else if (emfAverage==ElectroMotiveForce::uct_contact) { + } else if (emfAverage==EMF::uct_contact) { K_StoreContact(i,j,k,st,sb,Flux,Et,Eb,SV); - } else if (emfAverage==ElectroMotiveForce::uct_hll) { + } else if (emfAverage==EMF::uct_hll) { K_StoreHLL(i,j,k,st,sb,sl,sr,vL,vR,Et,Eb,aL,aR,dL,dR); - } else if (emfAverage==ElectroMotiveForce::uct_hlld) { + } else if (emfAverage==EMF::uct_hlld) { K_StoreHLLD(i,j,k,st,sb,a2L,sl,sr, vL,vR,uL,uR,Et,Eb,aL,aR,dL,dR); } @@ -655,4 +652,4 @@ void Hydro::RoeMHD() { idfx::popRegion(); } -#endif // HYDRO_MHDSOLVERS_ROEMHD_HPP_ +#endif // FLUID_RIEMANNSOLVER_MHDSOLVERS_ROEMHD_HPP_ diff --git a/src/hydro/MHDsolvers/storeFlux.hpp b/src/fluid/RiemannSolver/MHDsolvers/storeFlux.hpp similarity index 97% rename from src/hydro/MHDsolvers/storeFlux.hpp rename to src/fluid/RiemannSolver/MHDsolvers/storeFlux.hpp index e719ee5b..c50cadb8 100644 --- a/src/hydro/MHDsolvers/storeFlux.hpp +++ b/src/fluid/RiemannSolver/MHDsolvers/storeFlux.hpp @@ -5,11 +5,11 @@ // Licensed under CeCILL 2.1 License, see COPYING for more information // *********************************************************************************** -#ifndef HYDRO_MHDSOLVERS_STOREFLUX_HPP_ -#define HYDRO_MHDSOLVERS_STOREFLUX_HPP_ +#ifndef FLUID_RIEMANNSOLVER_MHDSOLVERS_STOREFLUX_HPP_ +#define FLUID_RIEMANNSOLVER_MHDSOLVERS_STOREFLUX_HPP_ #include "idefix.hpp" -#include "hydro.hpp" +#include "fluid.hpp" template KOKKOS_FORCEINLINE_FUNCTION void K_StoreEMF( const int i, const int j, const int k, @@ -207,4 +207,4 @@ KOKKOS_FORCEINLINE_FUNCTION void K_StoreHLLD( const int i, const int j, const in #endif } -#endif //HYDRO_MHDSOLVERS_STOREFLUX_HPP_ +#endif //FLUID_RIEMANNSOLVER_MHDSOLVERS_STOREFLUX_HPP_ diff --git a/src/hydro/MHDsolvers/tvdlfMHD.hpp b/src/fluid/RiemannSolver/MHDsolvers/tvdlfMHD.hpp similarity index 63% rename from src/hydro/MHDsolvers/tvdlfMHD.hpp rename to src/fluid/RiemannSolver/MHDsolvers/tvdlfMHD.hpp index 27058b4e..b8435d3c 100644 --- a/src/hydro/MHDsolvers/tvdlfMHD.hpp +++ b/src/fluid/RiemannSolver/MHDsolvers/tvdlfMHD.hpp @@ -5,20 +5,23 @@ // Licensed under CeCILL 2.1 License, see COPYING for more information // *********************************************************************************** -#ifndef HYDRO_MHDSOLVERS_TVDLFMHD_HPP_ -#define HYDRO_MHDSOLVERS_TVDLFMHD_HPP_ +#ifndef FLUID_RIEMANNSOLVER_MHDSOLVERS_TVDLFMHD_HPP_ +#define FLUID_RIEMANNSOLVER_MHDSOLVERS_TVDLFMHD_HPP_ #include "../idefix.hpp" -#include "slopeLimiter.hpp" -#include "fluxMHD.hpp" -#include "convertConsToPrimMHD.hpp" +#include "extrapolateToFaces.hpp" +#include "flux.hpp" +#include "convertConsToPrim.hpp" #include "storeFlux.hpp" -#include "electroMotiveForce.hpp" +#include "constrainedTransport.hpp" // Compute Riemann fluxes from states using TVDLF solver +template template -void Hydro::TvdlfMHD() { - idfx::pushRegion("Hydro::TVDLF_MHD"); +void RiemannSolver::TvdlfMHD(IdefixArray4D &Flux) { + idfx::pushRegion("RiemannSolver::TVDLF_MHD"); + + using EMF = ConstrainedTransport; constexpr int ioffset = (DIR==IDIR) ? 1 : 0; constexpr int joffset = (DIR==JDIR) ? 1 : 0; @@ -39,9 +42,7 @@ void Hydro::TvdlfMHD() { IdefixArray4D Vc = this->Vc; IdefixArray4D Vs = this->Vs; - IdefixArray4D Flux = this->FluxRiemann; IdefixArray3D cMax = this->cMax; - IdefixArray3D csIsoArr = this->isoSoundSpeedArray; // Required for high order interpolations IdefixArray1D dx = this->data->dx[DIR]; @@ -50,7 +51,7 @@ void Hydro::TvdlfMHD() { IdefixArray3D Eb; IdefixArray3D Et; - const ElectroMotiveForce::AveragingType emfAverage = emf.averaging; + const typename EMF::AveragingType emfAverage = hydro->emf->averaging; // Required by UCT_Contact IdefixArray3D SV; @@ -61,12 +62,9 @@ void Hydro::TvdlfMHD() { IdefixArray3D dL; IdefixArray3D dR; - real gamma = this->gamma; - [[maybe_unused]] real gamma_m1=gamma-ONE_F; - [[maybe_unused]] real csIso = this->isoSoundSpeed; - [[maybe_unused]] HydroModuleStatus haveIsoCs = this->haveIsoSoundSpeed; + EquationOfState eos = *(hydro->eos.get()); - SlopeLimiter slopeLim(Vc,data->dx[DIR],shockFlattening); + ExtrapolateToFaces extrapol = *this->GetExtrapolator(); // Define normal, tangent and bi-tanget indices // st and sb will be useful only when Hall is included real st = ONE_F, sb = ONE_F; @@ -79,16 +77,16 @@ void Hydro::TvdlfMHD() { sb = +ONE_F; ) - Et = this->emf.ezi; - Eb = this->emf.eyi; + Et = hydro->emf->ezi; + Eb = hydro->emf->eyi; - SV = this->emf.svx; + SV = hydro->emf->svx; - aL = this->emf.axL; - aR = this->emf.axR; + aL = hydro->emf->axL; + aR = hydro->emf->axR; - dL = this->emf.dxL; - dR = this->emf.dxR; + dL = hydro->emf->dxL; + dR = hydro->emf->dxR; break; #if DIMENSIONS >= 2 @@ -99,16 +97,16 @@ void Hydro::TvdlfMHD() { sb = -ONE_F; ) - Et = this->emf.ezj; - Eb = this->emf.exj; + Et = hydro->emf->ezj; + Eb = hydro->emf->exj; - SV = this->emf.svy; + SV = hydro->emf->svy; - aL = this->emf.ayL; - aR = this->emf.ayR; + aL = hydro->emf->ayL; + aR = hydro->emf->ayR; - dL = this->emf.dyL; - dR = this->emf.dyR; + dL = hydro->emf->dyL; + dR = hydro->emf->dyR; break; #endif @@ -121,16 +119,16 @@ void Hydro::TvdlfMHD() { , sb = +ONE_F; ) - Et = this->emf.eyk; - Eb = this->emf.exk; + Et = hydro->emf->eyk; + Eb = hydro->emf->exk; - SV = this->emf.svz; + SV = hydro->emf->svz; - aL = this->emf.azL; - aR = this->emf.azR; + aL = hydro->emf->azL; + aR = hydro->emf->azR; - dL = this->emf.dzL; - dR = this->emf.dzR; + dL = hydro->emf->dzL; + dR = hydro->emf->dzR; break; #endif default: @@ -143,30 +141,27 @@ void Hydro::TvdlfMHD() { data->beg[IDIR]-iextend,data->end[IDIR]+ioffset+iextend, KOKKOS_LAMBDA (int k, int j, int i) { // Init the directions (should be in the kernel for proper optimisation by the compilers) - EXPAND( const int Xn = DIR+MX1; , - const int Xt = (DIR == IDIR ? MX2 : MX1); , - const int Xb = (DIR == KDIR ? MX2 : MX3); ) - + const int Xn = DIR+MX1; EXPAND( const int BXn = DIR+BX1; , const int BXt = (DIR == IDIR ? BX2 : BX1); , const int BXb = (DIR == KDIR ? BX2 : BX3); ) // Primitive variables - real vL[NVAR]; - real vR[NVAR]; - real v[NVAR]; + real vL[Phys::nvar]; + real vR[Phys::nvar]; + real v[Phys::nvar]; - real uL[NVAR]; - real uR[NVAR]; + real uL[Phys::nvar]; + real uR[Phys::nvar]; - real fluxL[NVAR]; - real fluxR[NVAR]; + real fluxL[Phys::nvar]; + real fluxR[Phys::nvar]; // Load primitive variables - slopeLim.ExtrapolatePrimVar(i, j, k, vL, vR); + extrapol.ExtrapolatePrimVar(i, j, k, vL, vR); vL[BXn] = Vs(DIR,k,j,i); vR[BXn] = vL[BXn]; #pragma unroll - for(int nv = 0 ; nv < NVAR; nv++) { + for(int nv = 0 ; nv < Phys::nvar; nv++) { v[nv] = HALF_F*(vL[nv] + vR[nv]); } @@ -179,14 +174,12 @@ void Hydro::TvdlfMHD() { c2Iso = ZERO_F; #if HAVE_ENERGY + real gamma = eos.GetGamma(v[PRS],v[RHO]); gpr=gamma*v[PRS]; #else - if(haveIsoCs == UserDefFunction) { - c2Iso = HALF_F*(csIsoArr(k,j,i)+csIsoArr(k-koffset,j-joffset,i-ioffset)); - c2Iso = c2Iso*c2Iso; - } else { - c2Iso = csIso*csIso; - } + c2Iso = HALF_F*(eos.GetWaveSpeed(k,j,i) + +eos.GetWaveSpeed(k-koffset,j-joffset,i-ioffset)); + c2Iso *= c2Iso; gpr = c2Iso*v[RHO]; #endif @@ -208,17 +201,17 @@ void Hydro::TvdlfMHD() { // 2-- Compute the conservative variables - K_PrimToCons(uL, vL, gamma_m1); - K_PrimToCons(uR, vR, gamma_m1); + K_PrimToCons(uL, vL, &eos); + K_PrimToCons(uR, vR, &eos); // 3-- Compute the left and right fluxes - K_Flux(fluxL, vL, uL, c2Iso, ARG_EXPAND(Xn, Xt, Xb), ARG_EXPAND(BXn, BXt, BXb)); - K_Flux(fluxR, vR, uR, c2Iso, ARG_EXPAND(Xn, Xt, Xb), ARG_EXPAND(BXn, BXt, BXb)); + K_Flux(fluxL, vL, uL, c2Iso); + K_Flux(fluxR, vR, uR, c2Iso); // 5-- Compute the flux from the left and right states #pragma unroll - for(int nv = 0 ; nv < NVAR; nv++) { + for(int nv = 0 ; nv < Phys::nvar; nv++) { Flux(nv,k,j,i) = HALF_F*(fluxL[nv] + fluxR[nv] + cmax*(uL[nv] - uR[nv])); } @@ -226,15 +219,15 @@ void Hydro::TvdlfMHD() { cMax(k,j,i) = cmax; // 7-- Store the flux in the emf components - if (emfAverage==ElectroMotiveForce::arithmetic - || emfAverage==ElectroMotiveForce::uct0) { + if (emfAverage==EMF::arithmetic + || emfAverage==EMF::uct0) { K_StoreEMF(i,j,k,st,sb,Flux,Et,Eb); - } else if (emfAverage==ElectroMotiveForce::uct_contact) { + } else if (emfAverage==EMF::uct_contact) { K_StoreContact(i,j,k,st,sb,Flux,Et,Eb,SV); - } else if (emfAverage==ElectroMotiveForce::uct_hll) { + } else if (emfAverage==EMF::uct_hll) { K_StoreHLL(i,j,k,st,sb,sl,sr,vL,vR,Et,Eb,aL,aR,dL,dR); } - /*else if (emfAverage==ElectroMotiveForce::uct_hlld) { + /*else if (emfAverage==EMF::uct_hlld) { // We do not have the Alfven speed in the HLL solver K_StoreHLLD(i,j,k,st,sb,c2Iso,sl,sr,vL,vR,uL,uR,Et,Eb,aL,aR,dL,dR); } */ @@ -243,4 +236,4 @@ void Hydro::TvdlfMHD() { idfx::popRegion(); } -#endif // HYDRO_MHDSOLVERS_TVDLFMHD_HPP_ +#endif // FLUID_RIEMANNSOLVER_MHDSOLVERS_TVDLFMHD_HPP_ diff --git a/src/fluid/RiemannSolver/calcFlux.hpp b/src/fluid/RiemannSolver/calcFlux.hpp new file mode 100644 index 00000000..ea5b1874 --- /dev/null +++ b/src/fluid/RiemannSolver/calcFlux.hpp @@ -0,0 +1,86 @@ +// *********************************************************************************** +// Idefix MHD astrophysical code +// Copyright(C) Geoffroy R. J. Lesur +// and other code contributors +// Licensed under CeCILL 2.1 License, see COPYING for more information +// *********************************************************************************** + +#ifndef FLUID_RIEMANNSOLVER_CALCFLUX_HPP_ +#define FLUID_RIEMANNSOLVER_CALCFLUX_HPP_ + +#if MHD == YES +#include "hlldMHD.hpp" +#include "hllMHD.hpp" +#include "roeMHD.hpp" +#include "tvdlfMHD.hpp" +#endif + +#include "hllcHD.hpp" +#include "hllHD.hpp" +#include "tvdlfHD.hpp" +#include "roeHD.hpp" +#include "hllDust.hpp" + +#include "shockFlattening.hpp" + +// Compute Riemann fluxes from states +template +template +void RiemannSolver::CalcFlux(IdefixArray4D &flux) { + idfx::pushRegion("RiemannSolver::CalcFlux"); + if constexpr(dir == IDIR) { + // enable shock flattening + if(haveShockFlattening) shockFlattening->FindShock(); + } + + if constexpr(Phys::mhd) { + switch (mySolver) { + case TVDLF_MHD: + TvdlfMHD(flux); + break; + case HLL_MHD: + HllMHD(flux); + break; + case HLLD_MHD: + HlldMHD(flux); + break; + case ROE_MHD: + RoeMHD(flux); + break; + default: + break; + } + } else { + if constexpr(Phys::dust) { + switch (mySolver) { + case HLL_DUST: + HllDust(flux); + break; + default: // do nothing + IDEFIX_ERROR("Internal error: Unknown solver"); + break; + } + } else { + // Default hydro solvers + switch (mySolver) { + case TVDLF: + TvdlfHD(flux); + break; + case HLL: + HllHD(flux); + break; + case HLLC: + HllcHD(flux); + break; + case ROE: + RoeHD(flux); + break; + default: // do nothing + IDEFIX_ERROR("Internal error: Unknown solver"); + break; + } + }// Dust + } + idfx::popRegion(); +} +#endif // FLUID_RIEMANNSOLVER_CALCFLUX_HPP_ diff --git a/src/fluid/RiemannSolver/extrapolateToFaces.hpp b/src/fluid/RiemannSolver/extrapolateToFaces.hpp new file mode 100644 index 00000000..bb3c9958 --- /dev/null +++ b/src/fluid/RiemannSolver/extrapolateToFaces.hpp @@ -0,0 +1,327 @@ +// *********************************************************************************** +// Idefix MHD astrophysical code +// Copyright(C) Geoffroy R. J. Lesur +// and other code contributors +// Licensed under CeCILL 2.1 License, see COPYING for more information +// *********************************************************************************** +#ifndef FLUID_RIEMANNSOLVER_EXTRAPOLATETOFACES_HPP_ +#define FLUID_RIEMANNSOLVER_EXTRAPOLATETOFACES_HPP_ + +#include "fluid.hpp" +#include "dataBlock.hpp" +#include "shockFlattening.hpp" +#include "slopeLimiter.hpp" + +// Build a left and right extrapolation of the primitive variables along direction dir + +// These functions extrapolate the cell prim vars to the faces. Definitions are as followed +// +// | cell i-1 interface i cell i +// |-----------------------------------|------------------------------------|| +// Vc(i-1) PrimL(i) PrimR(i) Vc(i) +template +class ExtrapolateToFaces { + using SL = SlopeLimiter; + + public: + explicit ExtrapolateToFaces(RiemannSolver *rSolver): + Vc(rSolver->hydro->Vc), + dx(rSolver->hydro->data->dx[dir]), + shockFlattening(rSolver->haveShockFlattening), + isRegularGrid(rSolver->hydro->data->mygrid->isRegularCartesian) { + if(shockFlattening) { + flags = rSolver->shockFlattening->flagArray; + } + if(!isRegularGrid) { + ComputePLMweights(rSolver->hydro->data); + } + } + + void ComputePLMweights(DataBlock *data) { + // Allocate weight arrays + cpArray = IdefixArray1D("ExtrapolateToFaces_cp",data->np_tot[dir]); + cmArray = IdefixArray1D("ExtrapolateToFaces_cm",data->np_tot[dir]); + dpArray = IdefixArray1D("ExtrapolateToFaces_dp",data->np_tot[dir]); + dmArray = IdefixArray1D("ExtrapolateToFaces_dm",data->np_tot[dir]); + wpArray = IdefixArray1D("ExtrapolateToFaces_wp",data->np_tot[dir]); + wmArray = IdefixArray1D("ExtrapolateToFaces_wm",data->np_tot[dir]); + + auto dx = data->dx[dir]; + auto xgc = data->xgc[dir]; + auto xr = data->xr[dir]; + auto wp = wpArray; + auto wm = wmArray; + auto cp = cpArray; + auto cm = cmArray; + auto dp = dpArray; + auto dm = dmArray; + + idefix_for("ComputePLMweights",1,data->np_tot[dir]-1, + KOKKOS_LAMBDA(const int i) { + wp(i) = dx(i) / (xgc(i+1) - xgc(i)); + wm(i) = dx(i) / (xgc(i) - xgc(i-1)); + cp(i) = (xgc(i+1) - xgc(i)) / (xr(i) - xgc(i)); + cm(i) = (xgc(i) - xgc(i-1)) / (xgc(i) - xr(i-1)); + dp(i) = (xr(i) - xgc(i)) / dx(i); + dm(i) = (xgc(i) - xr(i-1)) / dx(i); + }); + } + + + + KOKKOS_FORCEINLINE_FUNCTION void ExtrapolatePrimVar(const int i, + const int j, + const int k, + real vL[], real vR[]) const { + // 1-- Store the primitive variables on the left, right, and averaged states + constexpr int ioffset = (dir==IDIR ? 1 : 0); + constexpr int joffset = (dir==JDIR ? 1 : 0); + constexpr int koffset = (dir==KDIR ? 1 : 0); + + for(int nv = 0 ; nv < Phys::nvar ; nv++) { + if constexpr(order == 1) { + vL[nv] = Vc(nv,k-koffset,j-joffset,i-ioffset); + vR[nv] = Vc(nv,k,j,i); + } else if constexpr(order == 2) { + if(isRegularGrid) { + ///////////////////////////////////// + // Regular Grid, PLM reconstruction + ///////////////////////////////////// + real dvm = Vc(nv,k-koffset,j-joffset,i-ioffset) + -Vc(nv,k-2*koffset,j-2*joffset,i-2*ioffset); + real dvp = Vc(nv,k,j,i)-Vc(nv,k-koffset,j-joffset,i-ioffset); + + real dv; + if(shockFlattening) { + if(flags(k-koffset,j-joffset,i-ioffset) == FlagShock::Shock) { + // Force slope limiter to minmod + dv = SL::MinModLim(dvp,dvm); + } else { + dv = SL::PLMLim(dvp,dvm); + } + } else { // No shock flattening + dv = SL::PLMLim(dvp,dvm); + } + + vL[nv] = Vc(nv,k-koffset,j-joffset,i-ioffset) + HALF_F*dv; + + dvm = dvp; + dvp = Vc(nv,k+koffset,j+joffset,i+ioffset) - Vc(nv,k,j,i); + + if(shockFlattening) { + if(flags(k,j,i) == FlagShock::Shock) { + dv = SL::MinModLim(dvp,dvm); + } else { + dv = SL::PLMLim(dvp,dvm); + } + } else { // No shock flattening + dv = SL::PLMLim(dvp,dvm); + } + + vR[nv] = Vc(nv,k,j,i) - HALF_F*dv; + } else { + ///////////////////////////////////// + // Irregular Grid, PLM reconstruction + ///////////////////////////////////// + const int index = ioffset*i + joffset*j + koffset*k; + + real dvm = Vc(nv,k-koffset,j-joffset,i-ioffset) + -Vc(nv,k-2*koffset,j-2*joffset,i-2*ioffset); + real dvp = Vc(nv,k,j,i)-Vc(nv,k-koffset,j-joffset,i-ioffset); + + dvm *= wmArray(index-1); + dvp *= wpArray(index-1); + real cp = cpArray(index-1); + real cm = cmArray(index-1); + + real dv; + if(shockFlattening) { + if(flags(k-koffset,j-joffset,i-ioffset) == FlagShock::Shock) { + // Force slope limiter to minmod + dv = SL::MinModLim(dvp,dvm); + } else { + dv = SL::PLMLim(dvp,dvm,cp,cm); + } + } else { // No shock flattening + dv = SL::PLMLim(dvp,dvm,cp,cm); + } + + vL[nv] = Vc(nv,k-koffset,j-joffset,i-ioffset) + dpArray(index-1)*dv; + + dvm = Vc(nv,k,j,i)-Vc(nv,k-koffset,j-joffset,i-ioffset); + dvp = Vc(nv,k+koffset,j+joffset,i+ioffset) - Vc(nv,k,j,i); + dvm *= wmArray(index); + dvp *= wpArray(index); + cp = cpArray(index); + cm = cmArray(index); + + if(shockFlattening) { + if(flags(k,j,i) == FlagShock::Shock) { + dv = SL::MinModLim(dvp,dvm); + } else { + dv = SL::PLMLim(dvp,dvm,cp,cm); + } + } else { // No shock flattening + dv = SL::PLMLim(dvp,dvm,cp,cm); + } + vR[nv] = Vc(nv,k,j,i) - dmArray(index)*dv; + } // Regular grid + + } else if constexpr(order == 3) { + // 1D index along the chosen direction + const int index = ioffset*i + joffset*j + koffset*k; + real dvm = Vc(nv,k-koffset,j-joffset,i-ioffset) + -Vc(nv,k-2*koffset,j-2*joffset,i-2*ioffset); + real dvp = Vc(nv,k,j,i)-Vc(nv,k-koffset,j-joffset,i-ioffset); + + // Limo3 limiter + real dv; + if(shockFlattening) { + if(flags(k-koffset,j-joffset,i-ioffset) == FlagShock::Shock) { + // Force slope limiter to minmod + dv = SL::MinModLim(dvp,dvm); + } else { + dv = dvp * SL::LimO3Lim(dvp, dvm, dx(index-1)); + } + } else { // No shock flattening + dv = dvp * SL::LimO3Lim(dvp, dvm, dx(index-1)); + } + + vL[nv] = Vc(nv,k-koffset,j-joffset,i-ioffset) + HALF_F*dv; + + // Check positivity + if(nv==RHO) { + // If face element is negative, revert to minmod + if(vL[nv] <= 0.0) { + dv = SL::MinModLim(dvp,dvm); + vL[nv] = Vc(nv,k-koffset,j-joffset,i-ioffset) + HALF_F*dv; + } + } + if constexpr(Phys::pressure) { + if(nv==PRS) { + // If face element is negative, revert to minmod + if(vL[nv] <= 0.0) { + dv = SL::MinModLim(dvp,dvm); + vL[nv] = Vc(nv,k-koffset,j-joffset,i-ioffset) + HALF_F*dv; + } + } + } + + dvm = dvp; + dvp = Vc(nv,k+koffset,j+joffset,i+ioffset) - Vc(nv,k,j,i); + + // Limo3 limiter + if(shockFlattening) { + if(flags(k,j,i) == FlagShock::Shock) { + // Force slope limiter to minmod + dv = SL::MinModLim(dvp,dvm); + } else { + dv = dvm * SL::LimO3Lim(dvm, dvp, dx(index)); + } + } else { // No shock flattening + dv = dvm * SL::LimO3Lim(dvm, dvp, dx(index)); + } + + vR[nv] = Vc(nv,k,j,i) - HALF_F*dv; + + // Check positivity + if(nv==RHO) { + // If face element is negative, revert to vanleer + if(vR[nv] <= 0.0) { + dv = SL::MinModLim(dvp,dvm); + vR[nv] = Vc(nv,k,j,i) - HALF_F*dv; + } + } + if constexpr(Phys::pressure) { + if(nv==PRS) { + // If face element is negative, revert to vanleer + if(vR[nv] <= 0.0) { + dv = SL::MinModLim(dvp,dvm); + vR[nv] = Vc(nv,k,j,i) - HALF_F*dv; + } + } + } + } else if constexpr(order == 4) { + // Reconstruction in cell i-1 + real vm2 = Vc(nv,k-3*koffset,j-3*joffset,i-3*ioffset);; + real vm1 = Vc(nv,k-2*koffset,j-2*joffset,i-2*ioffset); + real v0 = Vc(nv,k-koffset,j-joffset,i-ioffset); + real vp1 = Vc(nv,k,j,i); + real vp2 = Vc(nv,k+koffset,j+joffset,i+ioffset); + + real vr,vl; + SL::getPPMStates(vm2, vm1, v0, vp1, vp2, vl, vr); + // vL= left side of current interface (i-1/2)= right side of cell i-1 + + // Check positivity + if(nv==RHO) { + // If face element is negative, revert to vanleer + if(vr <= 0.0) { + real dv = SL::PLMLim(vp1-v0,v0-vm1); + vr = v0+HALF_F*dv; + } + } + if constexpr(Phys::pressure) { + if(nv==PRS) { + // If face element is negative, revert to vanleer + if(vr <= 0.0) { + real dv = SL::PLMLim(vp1-v0,v0-vm1); + vr = v0+HALF_F*dv; + } + } + } + + vL[nv] = vr; + // Reconstruction in cell i + + vm2 = vm1; + vm1 = v0; + v0 = vp1; + vp1 = vp2; + vp2 = Vc(nv,k+2*koffset,j+2*joffset,i+2*ioffset); + + SL::getPPMStates(vm2, vm1, v0, vp1, vp2, vl, vr); + + // Check positivity + if(nv==RHO) { + // If face element is negative, revert to vanleer + if(vl <= 0.0) { + real dv = SL::PLMLim(vp1-v0,v0-vm1); + vl = v0-HALF_F*dv; + } + } + if constexpr(Phys::pressure) { + if(nv==PRS) { + // If face element is negative, revert to vanleer + if(vl <= 0.0) { + real dv = SL::PLMLim(vp1-v0,v0-vm1); + vl = v0-HALF_F*dv; + } + } + } + + vR[nv] = vl; + } + } + } + + IdefixArray4D Vc; + IdefixArray1D dx; + IdefixArray3D flags; + + IdefixArray1D cpArray; + IdefixArray1D cmArray; + IdefixArray1D dpArray; + IdefixArray1D dmArray; + IdefixArray1D wpArray; + IdefixArray1D wmArray; + + bool isRegularGrid{true}; + bool shockFlattening{false}; +}; + + +#endif // FLUID_RIEMANNSOLVER_EXTRAPOLATETOFACES_HPP_ diff --git a/src/fluid/RiemannSolver/flux.hpp b/src/fluid/RiemannSolver/flux.hpp new file mode 100644 index 00000000..9c5131d8 --- /dev/null +++ b/src/fluid/RiemannSolver/flux.hpp @@ -0,0 +1,97 @@ +// *********************************************************************************** +// Idefix MHD astrophysical code +// Copyright(C) Geoffroy R. J. Lesur +// and other code contributors +// Licensed under CeCILL 2.1 License, see COPYING for more information +// *********************************************************************************** + +#ifndef FLUID_RIEMANNSOLVER_FLUX_HPP_ +#define FLUID_RIEMANNSOLVER_FLUX_HPP_ +#include "idefix.hpp" +#include "fluid.hpp" + + +// Local Kokkos Inlined functions + +/******************************************************************************************** + * @fn void K_Flux(real F[], real V[], real U[], real Cs2Iso, + * const int Xn, const int Xt, const int Xb, + * const int BXn, const int BXt, const int BXb) + * @param F[] Array of flux variables (output) + * @param V[] Array of primitive variabless (input) + * @param U[] Array of conservative variables (input) + * @param cs2Iso Isothermal sound speed (only used when ISOTHERMAL is defined) + * @param Xn Index of the normal velocity component + * + * This routine computes the MHD out of V and U variables and stores it in F + ********************************************************************************************/ +template +KOKKOS_INLINE_FUNCTION void K_Flux(real *KOKKOS_RESTRICT F, const real *KOKKOS_RESTRICT V, + const real *KOKKOS_RESTRICT U, real Cs2Iso) { + constexpr int Xn = DIR+MX1; + [[maybe_unused]] constexpr int BXn = DIR+BX1; + + // Mass flux (common to all physics) + F[RHO] = U[Xn]; + + // Momentum flux + if constexpr(Phys::mhd) { + EXPAND( F[MX1] = U[MX1]*V[Xn] - V[BXn]*V[BX1]; , + F[MX2] = U[MX2]*V[Xn] - V[BXn]*V[BX2]; , + F[MX3] = U[MX3]*V[Xn] - V[BXn]*V[BX3];) + } else { + EXPAND( F[MX1] = U[MX1]*V[Xn]; , + F[MX2] = U[MX2]*V[Xn]; , + F[MX3] = U[MX3]*V[Xn]; ) + } + + // Magnetic flux + if constexpr(Phys::mhd) { + EXPAND( , + constexpr int Xt = (DIR == IDIR ? MX2 : MX1); , + constexpr int Xb = (DIR == KDIR ? MX2 : MX3); ) + + EXPAND( , + constexpr int BXt = (DIR == IDIR ? BX2 : BX1); , + constexpr int BXb = (DIR == KDIR ? BX2 : BX3); ) + + EXPAND(F[BXn] = ZERO_F; , + F[BXt] = V[Xn]*V[BXt] - V[BXn]*V[Xt]; , + F[BXb] = V[Xn]*V[BXb] - V[BXn]*V[Xb]; ) + } + + if constexpr(Phys::pressure || Phys::isothermal) { + // Pressure-related term + real ptot; + if constexpr(Phys::mhd) { + //////////////// + // MHD VERSION + /////////////// + real Bmag2 = EXPAND(V[BX1]*V[BX1] , + V[BX2]*V[BX2], + V[BX3]*V[BX3]); + if constexpr(Phys::pressure) { + ptot = V[PRS] + HALF_F*Bmag2; + // Energy flux + F[ENG] = (U[ENG] + ptot)*V[Xn] - V[BXn] * (EXPAND( V[VX1]*V[BX1] , + + V[VX2]*V[BX2] , + + V[VX3]*V[BX3] )); + + } else if constexpr(Phys::isothermal) { + ptot = Cs2Iso * V[RHO] + HALF_F*Bmag2; + } + } else { + //////////////// + // Hydro VERSION + /////////////// + if constexpr(Phys::pressure) { + ptot = V[PRS]; + F[ENG] = (U[ENG] + ptot)*V[Xn]; + } else if constexpr(Phys::isothermal) { + ptot = Cs2Iso * V[RHO]; + } + } + // Add back pressure in the flux (not included in original PLUTO implementation) + F[Xn] += ptot; + } +} + +#endif //FLUID_RIEMANNSOLVER_FLUX_HPP_ diff --git a/src/fluid/RiemannSolver/riemannSolver.hpp b/src/fluid/RiemannSolver/riemannSolver.hpp new file mode 100644 index 00000000..4b306f38 --- /dev/null +++ b/src/fluid/RiemannSolver/riemannSolver.hpp @@ -0,0 +1,226 @@ +// *********************************************************************************** +// Idefix MHD astrophysical code +// Copyright(C) Geoffroy R. J. Lesur +// and other code contributors +// Licensed under CeCILL 2.1 License, see COPYING for more information +// *********************************************************************************** + +#ifndef FLUID_RIEMANNSOLVER_RIEMANNSOLVER_HPP_ +#define FLUID_RIEMANNSOLVER_RIEMANNSOLVER_HPP_ + +#include +#include + +#include "fluid.hpp" +#include "input.hpp" + +// Forward declaration +template +class ShockFlattening; + +#include "extrapolateToFaces.hpp" + +template +class RiemannSolver { + public: + // Riemann Solver type + + enum Solver {TVDLF_MHD, HLL_MHD, HLLD_MHD, ROE_MHD, TVDLF, HLL, HLLC, ROE, HLL_DUST}; + + RiemannSolver(Input &input, Fluid* hydro); + + template void CalcFlux(IdefixArray4D &); + + Solver GetSolver() { + return(mySolver); + } + + void ShowConfig(); + + // Riemann Solvers + template + void HlldMHD(IdefixArray4D &); + template + void HllMHD(IdefixArray4D &); + template + void RoeMHD(IdefixArray4D &); + template + void TvdlfMHD(IdefixArray4D &); + + template + void HllcHD(IdefixArray4D &); + template + void HllHD(IdefixArray4D &); + template + void RoeHD(IdefixArray4D &); + template + void TvdlfHD(IdefixArray4D &); + + template + void HllDust(IdefixArray4D &); + // Get the right slope limiter + template + ExtrapolateToFaces* GetExtrapolator(); + + private: + template + friend class ExtrapolateToFaces; + + IdefixArray4D Vc; + IdefixArray4D Vs; + IdefixArray4D Flux; + IdefixArray3D cMax; + Fluid* hydro; + DataBlock *data; + + Solver mySolver; + + std::unique_ptr> shockFlattening; + + // Because each direction is a different template, we can't use + std::unique_ptr> slopeLimIDIR; + std::unique_ptr> slopeLimJDIR; + std::unique_ptr> slopeLimKDIR; + + bool haveShockFlattening; +}; + +#include "shockFlattening.hpp" + +template +RiemannSolver::RiemannSolver(Input &input, Fluid* hydro) : Vc{hydro->Vc}, + Vs{hydro->Vs}, + Flux{hydro->FluxRiemann}, + cMax{hydro->cMax}, + hydro{hydro}, + data{hydro->data} + { + // read Solver from input file + if(!Phys::dust) { + std::string solverString = input.Get(std::string(Phys::prefix),"solver",0); + if (solverString.compare("tvdlf") == 0) { + if constexpr(Phys::mhd) { + mySolver = TVDLF_MHD; + } else { + mySolver = TVDLF; + } + } else if (solverString.compare("hll") == 0) { + if constexpr(Phys::mhd) { + mySolver = HLL_MHD; + } else { + mySolver = HLL; + } + } else if (solverString.compare("hlld") == 0) { + if constexpr(Phys::mhd) { + mySolver = HLLD_MHD; + } else { + IDEFIX_ERROR("hlld Riemann solver requires a MHD fluid"); + } + } else if (solverString.compare("hllc") == 0) { + if constexpr(Phys::mhd) { + IDEFIX_ERROR("hllc Riemann solver requires a HD fluid"); + } else { + mySolver = HLLC; + } + } else if (solverString.compare("roe") == 0) { + if constexpr(Phys::mhd) { + mySolver = ROE_MHD; + } else { + mySolver = ROE; + } + } else { + std::stringstream msg; + if constexpr(Phys::mhd) { + msg << "Unknown MHD solver type " << solverString; + } else { + msg << "Unknown HD solver type " << solverString; + } + IDEFIX_ERROR(msg); + } + // Check if Hall is enabled + if(input.CheckEntry(std::string(Phys::prefix),"hall")>=0) { + // Check consistency + if(mySolver != HLL_MHD ) + IDEFIX_ERROR("Hall effect is only compatible with HLL Riemann solver."); + } + } else { + // We're dealing with dust grains + mySolver = HLL_DUST; + } + + + + // Shock flattening + this->haveShockFlattening = input.CheckEntry(std::string(Phys::prefix),"shockFlattening")>=0; + // Init shock flattening + if(haveShockFlattening) { + this->shockFlattening = std::make_unique>( + hydro,input.Get(std::string(Phys::prefix),"shockFlattening",0)); + } + + // init slope limiters + slopeLimIDIR = std::make_unique>(this); + #if DIMENSIONS >= 2 + slopeLimJDIR = std::make_unique>(this); + #endif + #if DIMENSIONS == 3 + slopeLimKDIR = std::make_unique>(this); + #endif +} + +template +void RiemannSolver::ShowConfig() { + idfx::cout << "RiemannSolver: "; + switch(mySolver) { + case TVDLF: + idfx::cout << "tvdlf (HD)." << std::endl; + break; + case TVDLF_MHD: + idfx::cout << "tvdlf (MHD)." << std::endl; + break; + case HLL: + idfx::cout << "hll (HD)." << std::endl; + break; + case HLL_MHD: + idfx::cout << "hll (MHD)." << std::endl; + break; + case HLLD_MHD: + idfx::cout << "hlld (MHD)." << std::endl; + break; + case HLLC: + idfx::cout << "hllc (HD)." << std::endl; + break; + case ROE: + idfx::cout << "roe (HD)." << std::endl; + break; + case ROE_MHD: + idfx::cout << "roe (MHD)." << std::endl; + break; + case HLL_DUST: + idfx::cout << "HLL (Dust)." << std::endl; + break; + default: + IDEFIX_ERROR("Unknown Riemann solver"); + } + + if(haveShockFlattening) { + idfx::cout << Phys::prefix << ": Shock Flattening ENABLED." << std::endl; + } +} + +template +template +ExtrapolateToFaces* RiemannSolver::GetExtrapolator() { + if constexpr(dir==IDIR) { + return(this->slopeLimIDIR.get()); + } else if constexpr(dir==JDIR) { + return(this->slopeLimJDIR.get()); + } else if constexpr(dir==KDIR) { + return(this->slopeLimKDIR.get()); + } +} + + +#include "calcFlux.hpp" + +#endif //FLUID_RIEMANNSOLVER_RIEMANNSOLVER_HPP_ diff --git a/src/fluid/RiemannSolver/shockFlattening.hpp b/src/fluid/RiemannSolver/shockFlattening.hpp new file mode 100644 index 00000000..df68de76 --- /dev/null +++ b/src/fluid/RiemannSolver/shockFlattening.hpp @@ -0,0 +1,193 @@ +// *********************************************************************************** +// Idefix MHD astrophysical code +// Copyright(C) Geoffroy R. J. Lesur +// and other code contributors +// Licensed under CeCILL 2.1 License, see COPYING for more information +// *********************************************************************************** + +#ifndef FLUID_RIEMANNSOLVER_SHOCKFLATTENING_HPP_ +#define FLUID_RIEMANNSOLVER_SHOCKFLATTENING_HPP_ + +#include "fluid.hpp" +#include "../physics.hpp" +#include "eos.hpp" + +enum class FlagShock{None, Shock}; + +using UserShockFunc = void (*) (DataBlock &, const real t, IdefixArray3D&); + +template +class ShockFlattening { + public: + ShockFlattening(Fluid*, real); + ShockFlattening() {} + + void FindShock(); + + Fluid *hydro; + EquationOfState *eos; + IdefixArray3D flagArray; + bool isActive{false}; + real smoothing{0}; + + void EnrollUserShockFlag(UserShockFunc); + bool haveUserShockFlag{false}; + UserShockFunc userShockFunc{NULL}; +}; + +template +struct ShockFlattening_FindShockFunctor { + //***************************************************************** + // Functor constructor + //***************************************************************** + explicit ShockFlattening_FindShockFunctor(ShockFlattening *sf) { + flags = sf->flagArray; + Vc = sf->hydro->Vc; + smoothing = sf->smoothing; + + #if GEOMETRY == CARTESIAN + dx1 = sf->hydro->data->dx[IDIR]; + dx2 = sf->hydro->data->dx[JDIR]; + dx3 = sf->hydro->data->dx[KDIR]; + #else + Ax1 = sf->hydro->data->A[IDIR]; + Ax2 = sf->hydro->data->A[JDIR]; + Ax3 = sf->hydro->data->A[KDIR]; + dV = sf->hydro->data->dV; + #endif + if constexpr(Phys::isothermal) { + eos = *(sf->hydro->eos.get()); + } + } + //***************************************************************** + // Functor Variables + //***************************************************************** + real smoothing; + IdefixArray3D flags; + IdefixArray4D Vc; + #if GEOMETRY == CARTESIAN + IdefixArray1D dx1, dx2, dx3; + #else + IdefixArray3D Ax1,Ax2,Ax3, dV; + #endif + + EquationOfState eos; + + //***************************************************************** + // Functor Operator + //***************************************************************** + KOKKOS_INLINE_FUNCTION void operator() (const int k, const int j, const int i) const { + flags(k,j,i) = FlagShock::None; + #if GEOMETRY == CARTESIAN + real divV = D_EXPAND( (Vc(VX1,k,j,i+1) - Vc(VX1,k,j,i-1))/(2*dx1(i)) , + + (Vc(VX2,k,j+1,i) - Vc(VX2,k,j-1,i))/(2*dx2(j)) , + + (Vc(VX3,k+1,j,i) - Vc(VX3,k-1,j,i))/(2*dx3(k)) ); + #else + real divV = D_EXPAND( (Vc(VX1,k,j,i+1) + Vc(VX1,k,j,i))*Ax1(k,j,i+1) - + (Vc(VX1,k,j,i-1) + Vc(VX1,k,j,i))*Ax1(k,j,i) , + + (Vc(VX2,k,j+1,i) + Vc(VX2,k,j,i))*Ax2(k,j+1,i) - + (Vc(VX2,k,j-1,i) + Vc(VX2,k,j,i))*Ax2(k,j,i) , + + (Vc(VX3,k+1,j,i) + Vc(VX3,k,j,i))*Ax3(k+1,j,i) - + (Vc(VX3,k-1,j,i) + Vc(VX3,k,j,i))*Ax3(k,j,i) ); + divV = 0.5*divV/dV(k,j,i); + #endif + + if(divV= 2 + cs = eos.GetWaveSpeed(k,j+1,i); + pR = Vc(RHO,k,j+1,i)*cs*cs; + cs = eos.GetWaveSpeed(k,j-1,i); + pL = Vc(RHO,k,j-1,i)*cs*cs; + + pmin = FMIN(pmin,pL); + pmin = FMIN(pmin,pR); + + gradP += FABS(pR - pL); + #endif + #if DIMENSIONS == 3 + cs = eos.GetWaveSpeed(k+1,j,i); + pR = Vc(RHO,k+1,j,i)*cs*cs; + cs = eos.GetWaveSpeed(k-1,j,i); + pL = Vc(RHO,k-1,j,i)*cs*cs; + + pmin = FMIN(pmin,pL); + pmin = FMIN(pmin,pR); + + gradP += FABS(pR - pL); + #endif + } else if constexpr(Phys::pressure) { + pmin = Vc(PRS,k,j,i); + pmin = FMIN(pmin,Vc(PRS,k,j,i+1)); + pmin = FMIN(pmin,Vc(PRS,k,j,i-1)); + gradP = FABS(Vc(PRS,k,j,i+1) - Vc(PRS,k,j,i-1)); + #if DIMENSIONS >= 2 + pmin = FMIN(pmin,Vc(PRS,k,j+1,i)); + pmin = FMIN(pmin,Vc(PRS,k,j-1,i)); + gradP += FABS(Vc(PRS,k,j+1,i) - Vc(PRS,k,j-1,i)); + #endif + #if DIMENSIONS == 3 + pmin = FMIN(pmin,Vc(PRS,k+1,j,i)); + pmin = FMIN(pmin,Vc(PRS,k-1,j,i)); + gradP += FABS(Vc(PRS,k+1,j,i) - Vc(PRS,k-1,j,i)); + #endif + } + if constexpr(Phys::pressure || Phys::isothermal) { + if(gradP > smoothing*pmin) { + flags(k,j,i) = FlagShock::Shock; + } + } + } + } +}; + + +template +ShockFlattening::ShockFlattening(Fluid *h, real smoothing) { + hydro = h; + flagArray = IdefixArray3D("flagArray",h->data->np_tot[KDIR], + h->data->np_tot[JDIR], + h->data->np_tot[IDIR]); + this->isActive = true; + this->smoothing = smoothing; +} + +template +void ShockFlattening::FindShock() { + idfx::pushRegion("ShockFlattening::FindShock"); + + auto func = ShockFlattening_FindShockFunctor(this); + + auto beg = hydro->data->beg; + auto end = hydro->data->end; + idefix_for("findshocks", + beg[KDIR]-KOFFSET, end[KDIR]+KOFFSET, + beg[JDIR]-JOFFSET, end[JDIR]+JOFFSET, + beg[IDIR]-IOFFSET, end[IDIR]+IOFFSET, + func); + + if (haveUserShockFlag) { + userShockFunc(*this->hydro->data, this->hydro->data->t, this->flagArray); + } + idfx::popRegion(); +} + +template +void ShockFlattening::EnrollUserShockFlag(UserShockFunc myFunc) { + this->haveUserShockFlag = true; + this->userShockFunc = myFunc; +} + +#endif // FLUID_RIEMANNSOLVER_SHOCKFLATTENING_HPP_ diff --git a/src/fluid/RiemannSolver/slopeLimiter.hpp b/src/fluid/RiemannSolver/slopeLimiter.hpp new file mode 100644 index 00000000..082ed089 --- /dev/null +++ b/src/fluid/RiemannSolver/slopeLimiter.hpp @@ -0,0 +1,194 @@ +// *********************************************************************************** +// Idefix MHD astrophysical code +// Copyright(C) Geoffroy R. J. Lesur +// and other code contributors +// Licensed under CeCILL 2.1 License, see COPYING for more information +// *********************************************************************************** +#ifndef FLUID_RIEMANNSOLVER_SLOPELIMITER_HPP_ +#define FLUID_RIEMANNSOLVER_SLOPELIMITER_HPP_ + +// The default PLM Limiter template type +enum class PLMLimiter {VanLeer, MinMod, McLim}; + +template +class SlopeLimiter { + public: + KOKKOS_FORCEINLINE_FUNCTION static real MinModLim(const real dvp, const real dvm) { + real dq= 0.0; + // MinMod + if(dvp*dvm >0.0) { + real dq = ( fabs(dvp) < fabs(dvm) ? dvp : dvm); + } + return(dq); + } + + KOKKOS_FORCEINLINE_FUNCTION real static LimO3Lim(const real dvp, const real dvm, const real dx) { + real r = 0.1; + real a,b,c,q, th, lim; + real eta, psi, eps = 1.e-12; + + th = dvm/(dvp + 1.e-16); + + q = (2.0 + th)/3.0; + + a = FMIN(1.5,2.0*th); + a = FMIN(q,a); + b = FMAX(-0.5*th,a); + c = FMIN(q,b); + psi = FMAX(0.0,c); + + eta = r*dx; + eta = (dvm*dvm + dvp*dvp)/(eta*eta); + if ( eta <= 1.0 - eps) { + lim = q; + } else if (eta >= 1.0 + eps) { + lim = psi; + } else { + psi = (1.0 - (eta - 1.0)/eps)*q + + (1.0 + (eta - 1.0)/eps)*psi; + lim = 0.5*psi; + } + return (lim); + } + + KOKKOS_FORCEINLINE_FUNCTION static real VanLeerLim(const real dvp, const real dvm) { + real dq = (dvp*dvm > ZERO_F ? TWO_F*dvp*dvm/(dvp + dvm) : ZERO_F); + return(dq); + } + + // Generalize vanleer for non-homogeneous grids + KOKKOS_FORCEINLINE_FUNCTION static real VanLeerLim(const real dvp, const real dvm, + const real cp , const real cm) { + real dq = (dvp*dvm > 0.0 ? dvp*dvm*(cp*dvm + cm*dvp) + /(dvp*dvp + dvm*dvm + (cp + cm - 2.0)*dvp*dvm) : 0.0); + return dq; + } + + KOKKOS_FORCEINLINE_FUNCTION static real McLim(const real dvp, const real dvm) { + real dq = 0; + if(dvp*dvm >0.0) { + real dqc = 0.5*(dvp+dvm); + real d2q = 2.0*( fabs(dvp) < fabs(dvm) ? dvp : dvm); + dq= fabs(d2q) < fabs(dqc) ? d2q : dqc; + } + return(dq); + } + + // Generalized McLimiter for non-homogeneous grid + KOKKOS_FORCEINLINE_FUNCTION static real McLim(const real dvp, const real dvm, + const real cp , const real cm) { + real dq = 0; + if(dvp*dvm >0.0) { + real dqc = 0.5*(dvp+dvm); + real d2q = fabs(dvp*cp) < fabs(dvm*cm) ? dvp*cp : dvm*cm; + dq= fabs(d2q) < fabs(dqc) ? d2q : dqc; + } + return(dq); + } + + + KOKKOS_FORCEINLINE_FUNCTION static real PLMLim(const real dvp, const real dvm) { + if constexpr(limiter == PLMLimiter::VanLeer) return(VanLeerLim(dvp,dvm)); + if constexpr(limiter == PLMLimiter::McLim) return(McLim(dvp,dvm)); + if constexpr(limiter == PLMLimiter::MinMod) return(MinModLim(dvp,dvm)); + } + + // Overlad of PLM limiter for irregular grids + KOKKOS_FORCEINLINE_FUNCTION static real PLMLim(const real dvp, const real dvm, + const real cp, const real cm) { + if constexpr(limiter == PLMLimiter::VanLeer) return(VanLeerLim(dvp,dvm,cp,cm)); + if constexpr(limiter == PLMLimiter::McLim) return(McLim(dvp,dvm,cp,cm)); + if constexpr(limiter == PLMLimiter::MinMod) return(MinModLim(dvp,dvm)); + } + + + template + KOKKOS_FORCEINLINE_FUNCTION static int sign(T val) { + return (T(0) < val) - (val < T(0)); + } + + // PPM limiter, inspired from + // PH13: Peterson, J. L. & Hammett, G. W. Positivity Preservation and Advection Algorithms + // with Applications to Edge Plasma Turbulence. SIAM J. Sci. Comput. 35, B576–B605 (2013). + // CD11: Colella, P., Dorr, M. R., Hittinger, J. A. F. & Martin, D. F. High-order, + // finite-volume methods in mapped coordinates. Journal of Computational Physics 230, + // 2952–2976 (2011). + // CS08: Colella, P. & Sekora, M. D. A limiter for PPM that preserves accuracy at smooth extrema. + // Journal of Computational Physics 227, 7069–7076 (2008). + // FS18: Felker, K. G. & Stone, J. M. A fourth-order accurate finite volume method for ideal MHD + // via upwind constrained transport. Journal of Computational Physics 375, 1365–1400 (2018). + + KOKKOS_FORCEINLINE_FUNCTION static void limitPPMFaceValues(const real vm1, const real v0, + const real vp1, const real vp2, real &vph) { + // if local extremum, then use limited curvature estimate + if( (vp1-vph)*(vph-v0) < 0.0) { + // CD11, eqns. 85 + const real deltaL = (vm1-2*v0+vp1); + const real deltaC = 3*(v0-2*vph+vp1); + const real deltaR = (v0-2*vp1+vp2); + // Compute limited curvature estimate + real delta = 0.0; + + // CS08 eq. 18 with corrections from FS18 section. 2.2.2 + if(sign(deltaL) == sign(deltaC) && sign(deltaR) == sign(deltaC)) { + const real C = 1.25; + delta = C * FMIN(FABS(deltaL), FABS(deltaR)); + delta = sign(deltaC) * FMIN(delta, FABS(deltaC)); + } + vph = 0.5*(v0+vp1) - delta / 6.0; // CD11 eq. 88 (correction of CS08, eq. 19) + } + } + + KOKKOS_FORCEINLINE_FUNCTION static void getPPMStates(const real vm2, const real vm1, + const real v0, const real vp1, const real vp2, + real &vl, real &vr) { + const int n = 2; + + // 1: unlimited left and right interpolant (PH13 3.26-3.27) + vr = 7.0/12.0*(v0+vp1) - 1.0/12.0*(vm1+vp2); + vl = 7.0/12.0*(vm1+v0) - 1.0/12.0*(vm2+vp1); + + // 2: limit interpolated face values (CD11 4.3.1) + limitPPMFaceValues(vm2,vm1,v0,vp1,vl); + limitPPMFaceValues(vm1,v0,vp1,vp2,vr); + + real d2qf = 6.0*(vl + vr - 2.0*v0); + real d2qc0 = vm1 + vp1 - 2.0*v0; + real d2qcp1 = v0 + vp2 - 2.0*vp1; + real d2qcm1 = vm2 + v0 - 2.0*vm1; + + real d2q = 0.0; + if(sign(d2qf) == sign(d2qc0) && sign(d2qf) == sign(d2qcp1) && sign(d2qf) == sign(d2qcm1)) { + // smooth extrememum + const real C = 1.25; + d2q = FMIN(FABS(d2qc0),FABS(d2qcp1)); + d2q = C * FMIN(FABS(d2qcm1), d2q); + d2q = sign(d2qf) * FMIN(FABS(d2qf), d2q); + } + + real qmax = FMAX(FMAX(FABS(vm1),FABS(v0)),FABS(vp1)); + real rho = 0.0; + // todo(GL): replace 1e-12 by mixed precision value + if(FABS(d2qf) > 1e-12*qmax) { + rho = d2q / d2qf; + } + + // PH13 3.31 + if( ((vr - v0)*(v0 - vl) <= 0) || (vm1 - v0)*(v0 - vp1) <= 0 ) { + if(rho <= (1.0 - 1e-12)) { + vl = v0 - rho * (v0 - vl); + vr = v0 + rho * (vr - v0); + } + } else { + // PH13 3.32 + if(FABS(vr-v0) >= n*FABS(v0-vl)) { + vr = v0 + n*(v0-vl); + } + if(FABS(vl-v0) >= n*FABS(v0-vr)) { + vl = v0 + n*(v0-vr); + } + } + } +}; + +#endif // FLUID_RIEMANNSOLVER_SLOPELIMITER_HPP_ diff --git a/src/fluid/addNonIdealMHDFlux.hpp b/src/fluid/addNonIdealMHDFlux.hpp new file mode 100644 index 00000000..4ea98c76 --- /dev/null +++ b/src/fluid/addNonIdealMHDFlux.hpp @@ -0,0 +1,295 @@ +// *********************************************************************************** +// Idefix MHD astrophysical code +// Copyright(C) Geoffroy R. J. Lesur +// and other code contributors +// Licensed under CeCILL 2.1 License, see COPYING for more information +// *********************************************************************************** + +#ifndef FLUID_ADDNONIDEALMHDFLUX_HPP_ +#define FLUID_ADDNONIDEALMHDFLUX_HPP_ + +#include "fluid.hpp" +#include "dataBlock.hpp" + +// Compute parabolic fluxes +template +template +void Fluid::AddNonIdealMHDFlux(const real t) { + idfx::pushRegion("Fluid::addNonIdealMHDFlux"); + + if constexpr(Phys::mhd) { + int ioffset,joffset,koffset; + + IdefixArray4D Flux = this->FluxRiemann; + IdefixArray4D Vc = this->Vc; + IdefixArray4D Vs = this->Vs; + IdefixArray3D dMax = this->dMax; + IdefixArray4D J = this->J; + IdefixArray3D etaArr = this->etaOhmic; + IdefixArray3D xAmbiArr = this->xAmbipolar; + + // these two are required to ensure that the type is captured by KOKKOS_LAMBDA + HydroModuleStatus resistivity = resistivityStatus.status; + HydroModuleStatus ambipolar = ambipolarStatus.status; + + bool haveResistivity{false}; + bool haveAmbipolar{false}; + + if(data->rklCycle) { + haveResistivity = resistivityStatus.isRKL; + haveAmbipolar = ambipolarStatus.isRKL; + } else { + haveResistivity = resistivityStatus.isExplicit; + haveAmbipolar = ambipolarStatus.isExplicit; + } + + real etaConstant = this->etaO; + real xAConstant = this->xA; + + ioffset=joffset=koffset=0; + + switch(dir) { + case(IDIR): + ioffset = 1; + break; + case(JDIR): + joffset = 1; + break; + case(KDIR): + koffset=1; + break; + default: + IDEFIX_ERROR("Wrong direction"); + } + + // Load the diffusivity array when required + if(resistivity == UserDefFunction && dir == IDIR) { + if(ohmicDiffusivityFunc) + ohmicDiffusivityFunc(*data, t, etaArr); + else + IDEFIX_ERROR("No user-defined Ohmic diffusivity function has been enrolled"); + } + + if(ambipolar == UserDefFunction && dir == IDIR) { + if(ambipolarDiffusivityFunc) + ambipolarDiffusivityFunc(*data, t, xAmbiArr); + else + IDEFIX_ERROR("No user-defined ambipolar diffusivity function has been enrolled"); + } + + // Note the flux follows the same sign convention as the hyperbolic flux + // HEnce signs are reversed compared to the parabolic fluxes found in Pluto 4.3 + idefix_for("CalcParabolicFlux", + data->beg[KDIR],data->end[KDIR]+koffset, + data->beg[JDIR],data->end[JDIR]+joffset, + data->beg[IDIR],data->end[IDIR]+ioffset, + KOKKOS_LAMBDA (int k, int j, int i) { + real Jx1, Jx2, Jx3; + real Bx1, Bx2, Bx3; + real eta,xA; + real locdmax = 0.0; + + int ip1; // Offset indieces + [[maybe_unused]] int jp1, kp1; + ip1=i+1; + #if DIMENSIONS >=2 + jp1 = j+1; + #else + jp1=j; + #endif + #if DIMENSIONS == 3 + kp1 = k+1; + #else + kp1 = k; + #endif + + + Jx1=Jx2=Jx3=ZERO_F; + + if(resistivity == Constant) + eta = etaConstant; + + if(ambipolar == Constant) + xA = xAConstant; + + if(dir==IDIR) { + EXPAND( , + Jx3 = AVERAGE_4D_Y(J, KDIR, k, jp1, i); , + Jx2 = AVERAGE_4D_Z(J, JDIR, kp1, j, i); + Jx1 = AVERAGE_4D_XYZ(J, IDIR, kp1, jp1, i); ) + + EXPAND( Bx1 = Vs(BX1s,k,j,i); , + Bx2 = HALF_F*( Vc(BX2,k,j,i-1) + Vc(BX2,k,j,i)); , + Bx3 = HALF_F*( Vc(BX3,k,j,i-1) + Vc(BX3,k,j,i)); ) + if(haveResistivity) { + if(resistivity == UserDefFunction) + eta = AVERAGE_3D_X(etaArr,k,j,i); + + // Do not update BX2 if BX2s is defined + #if (DIMENSIONS < 2 && COMPONENTS >= 2) + Flux(BX2,k,j,i) += - eta * Jx3; + #endif + #if (DIMENSIONS < 3 && COMPONENTS == 3) + Flux(BX3,k,j,i) += eta * Jx2; + #endif + + #if HAVE_ENERGY + Flux(ENG,k,j,i) += EXPAND( ZERO_F , + - Bx2 * eta * Jx3 , + + Bx3 * eta * Jx2 ); + #endif + + locdmax += eta; + } + + if(haveAmbipolar) { + if(ambipolar == UserDefFunction) + xA = AVERAGE_3D_X(xAmbiArr,k,j,i); + + [[maybe_unused]] real BdotB = EXPAND( Bx1*Bx1, +Bx2*Bx2, +Bx3*Bx3); + + [[maybe_unused]] real Fx2 = -xA * BdotB * Jx3; + [[maybe_unused]] real Fx3 = xA * BdotB * Jx2; + #if COMPONENTS == 3 + real JdotB = Jx1 * Bx1 + Jx2 * Bx2 + Jx3 * Bx3; + Fx2 += xA * JdotB * Bx3; + Fx3 += -xA * JdotB * Bx2; + #endif + #if (DIMENSIONS < 2 && COMPONENTS >= 2) + Flux(BX2,k,j,i) += Fx2; + #endif + #if (DIMENSIONS < 3 && COMPONENTS == 3) + Flux(BX3,k,j,i) += Fx3; + #endif + + #if HAVE_ENERGY + Flux(ENG,k,j,i) += EXPAND( ZERO_F , + + Bx2 * Fx2 , + + Bx3 * Fx3 ); + #endif + + locdmax += xA*BdotB; + } + } + + if(dir==JDIR) { + EXPAND( Jx3 = AVERAGE_4D_X(J, KDIR, k, j, ip1); , + , + Jx1 = AVERAGE_4D_Z(J, IDIR, kp1, j, i); + Jx2 = AVERAGE_4D_XYZ(J, JDIR, kp1, j, ip1); ) + + EXPAND( Bx1 = HALF_F*( Vc(BX1,k,j-1,i) + Vc(BX1,k,j,i)); , + Bx2 = Vs(BX2s,k,j,i); , + Bx3 = HALF_F*( Vc(BX3,k,j-1,i) + Vc(BX3,k,j,i)); ) + + if(haveResistivity) { + if(resistivity == UserDefFunction) + eta = AVERAGE_3D_Y(etaArr,k,j,i); + + // This term is always overwritten by CT, since this sweep is performed whenver + // DIMENSIONS>=2 + //#if DIMENSIONS == 1 + // Flux(BX1,k,j,i) += eta * Jx3; + //#endif + #if (DIMENSIONS < 3 && COMPONENTS == 3) + Flux(BX3,k,j,i) += - eta * Jx1; + #endif + + #if HAVE_ENERGY + Flux(ENG,k,j,i) += EXPAND( Bx1 * eta * Jx3 , + , + - Bx3 * eta * Jx1 ); + #endif + locdmax += eta; + } + + + if(haveAmbipolar) { + if(ambipolar == UserDefFunction) + xA = AVERAGE_3D_Y(xAmbiArr,k,j,i); + + [[maybe_unused]] real BdotB = EXPAND( Bx1*Bx1, +Bx2*Bx2, +Bx3*Bx3); + + [[maybe_unused]] real Fx1 = xA * BdotB * Jx3; + [[maybe_unused]] real Fx3 = -xA * BdotB * Jx1; + #if COMPONENTS == 3 + real JdotB = Jx1 * Bx1 + Jx2 * Bx2 + Jx3 * Bx3; + Fx1 += -xA * JdotB * Bx3; + Fx3 += xA * JdotB * Bx1; + #endif + + // This term is always overwritten by CT, since this sweep is performed whenver + // DIMENSIONS>=2 + //#if DIMENSIONS == 1 + // Flux(BX1,k,j,i) += Fx1; + //#endif + #if (DIMENSIONS < 3 && COMPONENTS == 3) + Flux(BX3,k,j,i) += Fx3; + #endif + + #if HAVE_ENERGY + Flux(ENG,k,j,i) += EXPAND( + Bx1 * Fx1 , + + ZERO_F , + + Bx3 * Fx3 ); + #endif + + locdmax += xA*BdotB; + } + } + + if(dir==KDIR) { + Jx1 = AVERAGE_4D_Y(J, IDIR, k, jp1, i); + Jx2 = AVERAGE_4D_X(J, JDIR, k, j, ip1); + Jx3 = AVERAGE_4D_XYZ(J, KDIR, k, jp1, ip1); + + Bx1 = HALF_F*( Vc(BX1,k-1,j,i) + Vc(BX1,k,j,i)); + Bx2 = HALF_F*( Vc(BX2,k-1,j,i) + Vc(BX2,k,j,i)); + Bx3 = Vs(BX3s,k,j,i); + + + if(haveResistivity) { + if(resistivity == UserDefFunction) + eta = AVERAGE_3D_Z(etaArr,k,j,i); + + // This ie never needed since this is overwritten by CT + //Flux(BX1,k,j,i) += -eta * Jx2; + //Flux(BX2,k,j,i) += eta * Jx1; + + #if HAVE_ENERGY + Flux(ENG,k,j,i) += - Bx1 * eta * Jx2 + Bx2 * eta * Jx1; + #endif + dMax(k,j,i) += eta; + } + + if(haveAmbipolar) { + if(ambipolar == UserDefFunction) + xA = AVERAGE_3D_Z(xAmbiArr,k,j,i); + + [[maybe_unused]] real BdotB = Bx1*Bx1 + Bx2*Bx2 + Bx3*Bx3; + + [[maybe_unused]] real Fx1 = -xA * BdotB * Jx2; + [[maybe_unused]] real Fx2 = xA * BdotB * Jx1; + #if COMPONENTS == 3 + real JdotB = Jx1 * Bx1 + Jx2 * Bx2 + Jx3 * Bx3; + Fx1 += xA * JdotB * Bx2; + Fx2 += -xA * JdotB * Bx1; + #endif + // This is never needed since this is overwritten by CT + //Flux(BX1,k,j,i) += Fx1; + //Flux(BX2,k,j,i) += Fx2; + + #if HAVE_ENERGY + Flux(ENG,k,j,i) += Bx1 * Fx1 + Bx2 * Fx2; + #endif + + locdmax += xA*BdotB; + } + } + dMax(k,j,i) = FMAX(dMax(k,j,i),locdmax); + } + ); + } + idfx::popRegion(); +} + +#endif // FLUID_ADDNONIDEALMHDFLUX_HPP_ diff --git a/src/fluid/addSourceTerms.hpp b/src/fluid/addSourceTerms.hpp new file mode 100644 index 00000000..a0cb5a71 --- /dev/null +++ b/src/fluid/addSourceTerms.hpp @@ -0,0 +1,225 @@ +// *********************************************************************************** +// Idefix MHD astrophysical code +// Copyright(C) Geoffroy R. J. Lesur +// and other code contributors +// Licensed under CeCILL 2.1 License, see COPYING for more information +// *********************************************************************************** + +#ifndef FLUID_ADDSOURCETERMS_HPP_ +#define FLUID_ADDSOURCETERMS_HPP_ + +#include "fluid.hpp" +#include "dataBlock.hpp" +#include "fargo.hpp" + +template +struct Fluid_AddSourceTermsFunctor { + /// @brief Functor for Add source terms + /// @param hydro + //***************************************************************** + // Functor constructor + //***************************************************************** + explicit Fluid_AddSourceTermsFunctor(Fluid *hydro, real dt) { + Uc = hydro->Uc; + Vc = hydro->Vc; + x1 = hydro->data->x[IDIR]; + x2 = hydro->data->x[JDIR]; + + this->dt = dt; + #if GEOMETRY == SPHERICAL + sinx2 = hydro->data->sinx2; + tanx2 = hydro->data->tanx2; + rt = hydro->data->rt; + #endif + if constexpr(Phys::isothermal) { + eos = *(hydro->eos.get()); + } + haveRotation = hydro->haveRotation; + OmegaZ = hydro->OmegaZ; + haveFargo = hydro->data->haveFargo; + if(haveFargo) { + fargoVelocity = hydro->data->fargo->meanVelocity; + } + // shearing box (only with fargo&cartesian) + sbS = hydro->sbS; + } + + //***************************************************************** + // Functor Variables + //***************************************************************** + IdefixArray4D Uc; + IdefixArray4D Vc; + IdefixArray1D x1; + IdefixArray1D x2; + IdefixArray3D csIsoArr; + + real dt; +#if GEOMETRY == SPHERICAL + IdefixArray1D sinx2; + IdefixArray1D tanx2; + IdefixArray1D rt; +#endif + + EquationOfState eos; + + bool haveRotation; + real OmegaZ; + + // Fargo + bool haveFargo; + IdefixArray2D fargoVelocity; + + // shearing box (only with fargo&cartesian) + real sbS; + + //***************************************************************** + // Functor Operator + //***************************************************************** + KOKKOS_INLINE_FUNCTION void operator() (const int k, const int j, const int i) const { + #if GEOMETRY == CARTESIAN + // Manually add Coriolis force in cartesian geometry. Otherwise + // Coriolis is treated as a modification to the fluxes + if(haveRotation) { + Uc(MX1,k,j,i) += TWO_F * dt * Vc(RHO,k,j,i) * OmegaZ * Vc(VX2,k,j,i); + Uc(MX2,k,j,i) += - TWO_F * dt * Vc(RHO,k,j,i) * OmegaZ * Vc(VX1,k,j,i); + } + if(haveFargo) { + Uc(MX1,k,j,i) += TWO_F * dt * Vc(RHO,k,j,i) * OmegaZ * sbS * x1(i); + } + #endif + // fetch fargo velocity when required + [[maybe_unused]] real fargoV = ZERO_F; + if(haveFargo) { + // No source term when CARTESIAN+Fargo + #if GEOMETRY == POLAR + fargoV = fargoVelocity(k,i); + #elif GEOMETRY == SPHERICAL + fargoV = fargoVelocity(j,i); + #endif + } +#if GEOMETRY == CYLINDRICAL + #if COMPONENTS == 3 + real vphi,Sm; + vphi = Vc(iVPHI,k,j,i); + if(haveRotation) vphi += OmegaZ*x1(i); + Sm = Vc(RHO,k,j,i) * vphi*vphi; // Centrifugal + // Presure (because pressure is included in the flux, additional source terms arise) + if constexpr(Phys::isothermal) { + real c2Iso = eos.GetWaveSpeed(k,j,i); + c2Iso *= c2Iso; + + Sm += Vc(RHO,k,j,i)*c2Iso; + } else if constexpr(Phys::pressure) { + Sm += Vc(PRS,k,j,i); + } // Pressure + if constexpr(Phys::mhd) { + Sm -= Vc(iBPHI,k,j,i)*Vc(iBPHI,k,j,i); // Hoop stress + // Magnetic pressure + Sm += HALF_F*(EXPAND( Vc(BX1,k,j,i)*Vc(BX1,k,j,i) , + +Vc(BX2,k,j,i)*Vc(BX2,k,j,i) , + +Vc(BX3,k,j,i)*Vc(BX3,k,j,i) )); + } // MHD + Uc(MX1,k,j,i) += dt * Sm / x1(i); + #endif // COMPONENTS + +#elif GEOMETRY == POLAR + real vphi,Sm; + vphi = Vc(iVPHI,k,j,i) + fargoV; + if(haveRotation) vphi += OmegaZ*x1(i); + Sm = Vc(RHO,k,j,i) * vphi*vphi; // Centrifugal + // Pressure (because we're including pressure in the flux, + // we need that to get the radial pressure gradient) + if constexpr(Phys::isothermal) { + real c2Iso = eos.GetWaveSpeed(k,j,i); + c2Iso *= c2Iso; + Sm += Vc(RHO,k,j,i)*c2Iso; + } else if constexpr(Phys::pressure) { + Sm += Vc(PRS,k,j,i); + } // Pressure + if constexpr(Phys::mhd) { + Sm -= Vc(iBPHI,k,j,i)*Vc(iBPHI,k,j,i); // Hoop stress + // Magnetic pressus + Sm += HALF_F*(EXPAND( Vc(BX1,k,j,i)*Vc(BX1,k,j,i) , + +Vc(BX2,k,j,i)*Vc(BX2,k,j,i) , + +Vc(BX3,k,j,i)*Vc(BX3,k,j,i) )); + } // MHD + Uc(MX1,k,j,i) += dt * Sm / x1(i); + +#elif GEOMETRY == SPHERICAL + real vphi,Sm; + vphi = SELECT(ZERO_F, ZERO_F, Vc(iVPHI,k,j,i))+fargoV; + if(haveRotation) vphi += OmegaZ*x1(i)*FABS(sinx2(j)); + // Centrifugal + Sm = Vc(RHO,k,j,i) * (EXPAND( ZERO_F, + Vc(VX2,k,j,i)*Vc(VX2,k,j,i), + vphi*vphi)); + // Pressure curvature + [[maybe_unused]] real c2Iso{0}; + if constexpr(Phys::isothermal) { + c2Iso = eos.GetWaveSpeed(k,j,i); + c2Iso *= c2Iso; + Sm += 2.0*Vc(RHO,k,j,i)*c2Iso; + } else if constexpr(Phys::pressure) { + Sm += 2.0*Vc(PRS,k,j,i); + } // Pressure + if constexpr(Phys::mhd) { + // Hoop stress + Sm -= EXPAND( ZERO_F , + + Vc(iBTH,k,j,i)*Vc(iBTH,k,j,i) , + + Vc(iBPHI,k,j,i)*Vc(iBPHI,k,j,i) ); + // 2* mag pressure curvature + Sm += EXPAND( Vc(BX1,k,j,i)*Vc(BX1,k,j,i) , + +Vc(BX2,k,j,i)*Vc(BX2,k,j,i) , + +Vc(BX3,k,j,i)*Vc(BX3,k,j,i) ); + } //MHD + Uc(MX1,k,j,i) += dt*Sm/rt(i); + #if COMPONENTS >= 2 + real ct = 1.0/tanx2(j); + // Centrifugal + Sm = Vc(RHO,k,j,i) * (EXPAND( ZERO_F, - Vc(iVTH,k,j,i)*Vc(iVR,k,j,i), + ct*vphi*vphi)); + // Pressure curvature + if constexpr(Phys::isothermal) { + Sm += ct * c2Iso * Vc(RHO,k,j,i); + } else if constexpr(Phys::pressure) { + Sm += ct * Vc(PRS,k,j,i); + } + if constexpr(Phys::mhd) { + // Hoop stress + Sm += EXPAND( ZERO_F , + + Vc(iBTH,k,j,i)*Vc(iBR,k,j,i) , + - ct*Vc(iBPHI,k,j,i)*Vc(iBPHI,k,j,i) ); + // Magnetic pressure + Sm += HALF_F*ct*(EXPAND( Vc(BX1,k,j,i)*Vc(BX1,k,j,i) , + +Vc(BX2,k,j,i)*Vc(BX2,k,j,i) , + +Vc(BX3,k,j,i)*Vc(BX3,k,j,i)) ); + } // MHD + Uc(MX2,k,j,i) += dt*Sm / rt(i); + #endif // COMPONENTS +#endif + } +}; + + +// Add source terms +template +void Fluid::AddSourceTerms(real t, real dt) { + idfx::pushRegion("Fluid::AddSourceTerms"); + + if(haveUserSourceTerm) { + if(userSourceTerm != NULL) { + userSourceTerm(this, t, dt); + } else { + // Deprecated version + userSourceTermOld(*data, t, dt); + } + } + + auto func = Fluid_AddSourceTermsFunctor(this,dt); + + idefix_for("AddSourceTerms", + data->beg[KDIR],data->end[KDIR], + data->beg[JDIR],data->end[JDIR], + data->beg[IDIR],data->end[IDIR], + func); + + idfx::popRegion(); +} +#endif //FLUID_ADDSOURCETERMS_HPP_ diff --git a/src/fluid/boundary/CMakeLists.txt b/src/fluid/boundary/CMakeLists.txt new file mode 100644 index 00000000..35f30dff --- /dev/null +++ b/src/fluid/boundary/CMakeLists.txt @@ -0,0 +1,5 @@ +target_sources(idefix + PUBLIC ${CMAKE_CURRENT_LIST_DIR}/axis.cpp + PUBLIC ${CMAKE_CURRENT_LIST_DIR}/axis.hpp + PUBLIC ${CMAKE_CURRENT_LIST_DIR}/boundary.hpp + ) diff --git a/src/hydro/axis.cpp b/src/fluid/boundary/axis.cpp similarity index 83% rename from src/hydro/axis.cpp rename to src/fluid/boundary/axis.cpp index e699df55..4679af9a 100644 --- a/src/hydro/axis.cpp +++ b/src/fluid/boundary/axis.cpp @@ -5,88 +5,8 @@ // Licensed under CeCILL 2.1 License, see COPYING for more information // *********************************************************************************** -#include #include "axis.hpp" -#include "hydro.hpp" -#include "dataBlock.hpp" - - -void Axis::Init(Grid &grid, Hydro *h) { - this->hydro = h; - this->data = this->hydro->data; - this->emf = & this->hydro->emf; - - #if GEOMETRY != SPHERICAL - IDEFIX_ERROR("Axis boundary conditions are only designed to handle spherical geometry"); - #endif - - - if(fabs((grid.xend[KDIR] - grid.xbeg[KDIR] -2.0*M_PI)) < 1e-10) { - this->isTwoPi = true; - #ifdef WITH_MPI - // Check that there is a domain decomposition in phi - if(data->mygrid->nproc[KDIR]>1) { - if(data->mygrid->nproc[KDIR]%2==1) { - IDEFIX_ERROR("The numbre of processes in the phi direction should" - " be even for axis decomposition"); - } - needMPIExchange = true; - } - #endif - } else { - this->isTwoPi = false; - } - - // Check where the axis is lying. - if(hydro->data->lbound[JDIR] == axis) axisLeft = true; - if(hydro->data->rbound[JDIR] == axis) axisRight = true; - - // Init the symmetry array (used to flip the signs of arrays accross the axis) - symmetryVc = IdefixArray1D("Axis:SymmetryVc",NVAR); - IdefixArray1D::HostMirror symmetryVcHost = Kokkos::create_mirror_view(symmetryVc); - // Init the array - for (int nv = 0; nv < NVAR; nv++) { - symmetryVcHost(nv) = 1; - if (nv == VX2) - symmetryVcHost(nv) = -1; - if (nv == VX3) - symmetryVcHost(nv) = -1; - if (nv == BX2) - symmetryVcHost(nv) = -1; - if (nv == BX3) - symmetryVcHost(nv) = -1; - } - Kokkos::deep_copy(symmetryVc, symmetryVcHost); - -#if MHD == YES - symmetryVs = IdefixArray1D("Axis:SymmetryVs",DIMENSIONS); - IdefixArray1D::HostMirror symmetryVsHost = Kokkos::create_mirror_view(symmetryVs); - // Init the array - for(int nv = 0; nv < DIMENSIONS; nv++) { - symmetryVsHost(nv) = 1; - if (nv == BX2s) - symmetryVsHost(nv) = -1; - if (nv == BX3s) - symmetryVsHost(nv) = -1; - } - Kokkos::deep_copy(symmetryVs, symmetryVsHost); -#endif - - #if MHD == YES - this->Ex1Avg = IdefixArray1D("Axis:Ex1Avg",hydro->data->np_tot[IDIR]); - this->BAvg = IdefixArray2D("Axis:BxAvg",hydro->data->np_tot[IDIR],2); - if(hydro->haveCurrent) { - this->JAvg = IdefixArray2D("Axis:JAvg",hydro->data->np_tot[IDIR],3); - } - #endif - - #ifdef WITH_MPI - if(needMPIExchange) { - // Make MPI exchange datatypes - InitMPI(); - } - #endif -} +#include "boundary.hpp" void Axis::ShowConfig() { idfx::cout << "Axis: Axis regularisation ENABLED." << std::endl; @@ -100,9 +20,11 @@ void Axis::ShowConfig() { } } + void Axis::SymmetrizeEx1Side(int jref) { #if DIMENSIONS == 3 - IdefixArray3D Ex1 = emf->ex; + + IdefixArray3D Ex1 = this->ex; IdefixArray1D Ex1Avg = this->Ex1Avg; if(isTwoPi) { @@ -147,8 +69,9 @@ void Axis::SymmetrizeEx1Side(int jref) { // in Ve(AX3e...), leading potentially numerical instabilities in that region. // Hence, we enforce a regularisation of Ex3 for consistancy. + void Axis::RegularizeEx3side(int jref) { - IdefixArray3D Ex3 = emf->ez; + IdefixArray3D Ex3 = this->ez; idefix_for("Ex3_Regularise",0,data->np_tot[KDIR],0,data->np_tot[IDIR], KOKKOS_LAMBDA(int k,int i) { @@ -156,11 +79,12 @@ void Axis::RegularizeEx3side(int jref) { }); } + void Axis::RegularizeCurrentSide(int side) { // Compute the values of Jx, Jy and Jz that are consistent for all cells touching the axis #if DIMENSIONS == 3 - IdefixArray4D J = hydro->J; - IdefixArray4D Vs = hydro->Vs; + IdefixArray4D J = this->J; + IdefixArray4D Vs = this->Vs; int js = 0; int jc = 0; int sign = 0; @@ -216,16 +140,17 @@ void Axis::RegularizeCurrentSide(int side) { } // Average the Emf component along the axis + void Axis::RegularizeEMFs() { idfx::pushRegion("Axis::RegularizeEMFs"); if(this->axisLeft) { - int jref = hydro->data->beg[JDIR]; + int jref = data->beg[JDIR]; SymmetrizeEx1Side(jref); RegularizeEx3side(jref); } if(this->axisRight) { - int jref = hydro->data->end[JDIR]; + int jref = data->end[JDIR]; SymmetrizeEx1Side(jref); RegularizeEx3side(jref); } @@ -234,6 +159,7 @@ void Axis::RegularizeEMFs() { } // Average the Emf component along the axis + void Axis::RegularizeCurrent() { idfx::pushRegion("Axis::RegularizeCurrent"); @@ -247,10 +173,11 @@ void Axis::RegularizeCurrent() { idfx::popRegion(); } + void Axis::FixBx2sAxis(int side) { // Compute the values of Bx and By that are consistent with BX2 along the axis #if DIMENSIONS == 3 - IdefixArray4D Vs = hydro->Vs; + IdefixArray4D Vs = this->Vs; IdefixArray2D BAvg = this->BAvg; IdefixArray1D phi = data->x[KDIR]; @@ -306,6 +233,7 @@ void Axis::FixBx2sAxis(int side) { #endif // DIMENSIONS } + void Axis::FixBx2sAxisGhostAverage(int side) { // This uses the same method as Athena (Stone+????) to enforce the BX2 value // on the axis : @@ -314,7 +242,7 @@ void Axis::FixBx2sAxisGhostAverage(int side) { // right-side boundary) #if DIMENSIONS == 3 - IdefixArray4D Vs = hydro->Vs; + IdefixArray4D Vs = this->Vs; int jaxis = 0; @@ -334,10 +262,12 @@ void Axis::FixBx2sAxisGhostAverage(int side) { + // enforce the boundary conditions on the ghost zone accross the axis + void Axis::EnforceAxisBoundary(int side) { idfx::pushRegion("Axis::EnforceAxisBoundary"); - IdefixArray4D Vc = hydro->Vc; + IdefixArray4D Vc = this->Vc; IdefixArray1D sVc = this->symmetryVc; int ibeg = 0; @@ -366,7 +296,7 @@ void Axis::EnforceAxisBoundary(int side) { if(needMPIExchange) { ExchangeMPI(side); } else { // no MPI exchange - idefix_for("BoundaryAxis",0,NVAR,kbeg,kend,jbeg,jend,ibeg,iend, + idefix_for("BoundaryAxis",0,this->nVar,kbeg,kend,jbeg,jend,ibeg,iend, KOKKOS_LAMBDA (int n, int k, int j, int i) { int kcomp = nghost_k + (( k - nghost_k + np_int_k/2) % np_int_k); @@ -374,7 +304,7 @@ void Axis::EnforceAxisBoundary(int side) { }); }// MPI Exchange } else { // not 2pi - idefix_for("BoundaryAxis",0,NVAR,kbeg,kend,jbeg,jend,ibeg,iend, + idefix_for("BoundaryAxis",0,this->nVar,kbeg,kend,jbeg,jend,ibeg,iend, KOKKOS_LAMBDA (int n, int k, int j, int i) { // kcomp = k by construction since we're doing a fraction of twopi @@ -382,8 +312,8 @@ void Axis::EnforceAxisBoundary(int side) { }); } - #if MHD == YES - IdefixArray4D Vs = hydro->Vs; + if(haveMHD) { + IdefixArray4D Vs = this->Vs; IdefixArray1D sVs = this->symmetryVs; for(int component=0; component= 2 && MHD == YES - IdefixArray4D Vs = hydro->Vs; + IdefixArray4D Vs = this->Vs; IdefixArray3D Ax1=data->A[IDIR]; IdefixArray3D Ax2=data->A[JDIR]; IdefixArray3D Ax3=data->A[KDIR]; @@ -496,6 +427,7 @@ void Axis::ReconstructBx2s() { + void Axis::ExchangeMPI(int side) { idfx::pushRegion("Axis::ExchangeMPI"); #ifdef WITH_MPI @@ -504,8 +436,8 @@ void Axis::ExchangeMPI(int side) { int nx,ny,nz; auto bufferSend = this->bufferSend; IdefixArray1D map = this->mapVars; - IdefixArray4D Vc = hydro->Vc; - IdefixArray4D Vs = hydro->Vs; + IdefixArray4D Vc = this->Vc; + IdefixArray4D Vs = this->Vs; // If MPI Persistent, start receiving even before the buffers are filled @@ -534,7 +466,7 @@ void Axis::ExchangeMPI(int side) { Vc(map(n),k,j+ny,i); } ); - #if MHD == YES + if (haveMHD) { int VsIndex = mapNVars*nx*ny*nz; idefix_for("LoadBufferX2IDIR",kbeg,kend,jbeg,jend,ibeg,iend+1, KOKKOS_LAMBDA (int k, int j, int i) { @@ -549,7 +481,7 @@ void Axis::ExchangeMPI(int side) { Vs(KDIR,k,j+ny,i); } ); - #endif + } // MHD } else if(side==right) { idefix_for("LoadBufferX2Vc",0,mapNVars,kbeg,kend,jbeg,jend,ibeg,iend, KOKKOS_LAMBDA (int n, int k, int j, int i) { @@ -559,7 +491,7 @@ void Axis::ExchangeMPI(int side) { ); // Load face-centered field in the buffer - #if MHD == YES + if (haveMHD) { int VsIndex = mapNVars*nx*ny*nz; idefix_for("LoadBufferX2IDIR",kbeg,kend,jbeg,jend,ibeg,iend+1, KOKKOS_LAMBDA (int k, int j, int i) { @@ -575,7 +507,7 @@ void Axis::ExchangeMPI(int side) { Vs(KDIR,k,j+offset-ny,i); } ); - #endif // MHD + } // MHD } // side==right Kokkos::fence(); @@ -598,7 +530,7 @@ void Axis::ExchangeMPI(int side) { ); // Load face-centered field in the buffer - #if MHD == YES + if (haveMHD) { int VsIndex = mapNVars*nx*ny*nz; auto sVs = this->symmetryVs; idefix_for("StoreBufferX2IDIR",kbeg,kend,jbeg,jend,ibeg,iend+1, @@ -616,7 +548,7 @@ void Axis::ExchangeMPI(int side) { sVs(KDIR)*bufferRecv(i + (j-jbeg)*nx + (k-kbeg)*nx*ny + VsIndex ); } ); - #endif //MHD + } } else if(side==right) { idefix_for("StoreBufferAxis",0,mapNVars,kbeg,kend,jbeg,jend,ibeg,iend, KOKKOS_LAMBDA (int n, int k, int j, int i) { @@ -626,24 +558,24 @@ void Axis::ExchangeMPI(int side) { ); // Load face-centered field in the buffer - #if MHD == YES - int VsIndex = mapNVars*nx*ny*nz; - auto sVs = this->symmetryVs; - idefix_for("StoreBufferX2IDIR",kbeg,kend,jbeg,jend,ibeg,iend+1, - KOKKOS_LAMBDA (int k, int j, int i) { - Vs(IDIR,k,jend-(j-jbeg)-1+offset,i) = - sVs(IDIR)*bufferRecv(i + (j-jbeg)*(nx+1) + (k-kbeg)*(nx+1)*ny + VsIndex ); - } - ); - VsIndex = mapNVars*nx*ny*nz + (nx+1)*ny*nz; + if (haveMHD) { + int VsIndex = mapNVars*nx*ny*nz; + auto sVs = this->symmetryVs; + idefix_for("StoreBufferX2IDIR",kbeg,kend,jbeg,jend,ibeg,iend+1, + KOKKOS_LAMBDA (int k, int j, int i) { + Vs(IDIR,k,jend-(j-jbeg)-1+offset,i) = + sVs(IDIR)*bufferRecv(i + (j-jbeg)*(nx+1) + (k-kbeg)*(nx+1)*ny + VsIndex ); + } + ); + VsIndex = mapNVars*nx*ny*nz + (nx+1)*ny*nz; - idefix_for("StoreBufferX2KDIR",kbeg,kend+1,jbeg,jend,ibeg,iend, - KOKKOS_LAMBDA ( int k, int j, int i) { - Vs(KDIR,k,jend-(j-jbeg)-1+offset,i) = - sVs(KDIR)*bufferRecv(i + (j-jbeg)*nx + (k-kbeg)*nx*ny + VsIndex ); - } - ); - #endif + idefix_for("StoreBufferX2KDIR",kbeg,kend+1,jbeg,jend,ibeg,iend, + KOKKOS_LAMBDA ( int k, int j, int i) { + Vs(KDIR,k,jend-(j-jbeg)-1+offset,i) = + sVs(KDIR)*bufferRecv(i + (j-jbeg)*nx + (k-kbeg)*nx*ny + VsIndex ); + } + ); + } // MHD } MPI_Wait(&sendRequest, &sendStatus); @@ -655,6 +587,7 @@ void Axis::ExchangeMPI(int side) { idfx::popRegion(); } + void Axis::InitMPI() { idfx::pushRegion("Axis::InitMPI"); #ifdef WITH_MPI @@ -664,11 +597,11 @@ void Axis::InitMPI() { // The variable mapper list all of the variable which are exchanged in MPI boundary calls // This is required since we skip some of the variables in Vc to limit the amount of data // being exchanged - #if MHD == YES - this->mapNVars = NVAR - DIMENSIONS; // We will not send magnetic field components which are in Vs - #else - this->mapNVars = NVAR; - #endif + if (haveMHD) { + this->mapNVars = this->nVar - DIMENSIONS; // We don't send B components (already in Vs) + } else { + this->mapNVars = this->nVar; + } std::vector mapVars; // Init the list of variables we will exchange in MPI routines @@ -676,7 +609,7 @@ void Axis::InitMPI() { for(int n = 0 ; n < mapNVars ; n++) { mapVars.push_back(ntarget); ntarget++; - #if MHD == YES + if (haveMHD) { // Skip centered field components if they are also defined in Vs #if DIMENSIONS >= 1 if(ntarget==BX1) ntarget++; @@ -687,19 +620,19 @@ void Axis::InitMPI() { #if DIMENSIONS == 3 if(ntarget==BX3) ntarget++; #endif - #endif + } // MHD } this->mapVars = idfx::ConvertVectorToIdefixArray(mapVars); this->bufferSize = data->np_tot[IDIR] * data->nghost[JDIR] * data->np_int[KDIR] * mapNVars; - #if MHD == YES + if (haveMHD) { // IDIR bufferSize += (data->np_tot[IDIR]+1) * data->nghost[JDIR] * data->np_int[KDIR]; #if DIMENSIONS==3 bufferSize += data->np_tot[IDIR] * data->nghost[JDIR] * (data->np_int[KDIR]+1); #endif // DIMENSIONS - #endif + } this->bufferRecv = IdefixArray1D("bufferRecvAxis", bufferSize); this->bufferSend = IdefixArray1D("bufferSendAxis", bufferSize); diff --git a/src/fluid/boundary/axis.hpp b/src/fluid/boundary/axis.hpp new file mode 100644 index 00000000..a1c3054b --- /dev/null +++ b/src/fluid/boundary/axis.hpp @@ -0,0 +1,180 @@ +// *********************************************************************************** +// Idefix MHD astrophysical code +// Copyright(C) Geoffroy R. J. Lesur +// and other code contributors +// Licensed under CeCILL 2.1 License, see COPYING for more information +// *********************************************************************************** + +#ifndef FLUID_BOUNDARY_AXIS_HPP_ +#define FLUID_BOUNDARY_AXIS_HPP_ + +#include +#include "idefix.hpp" +#include "grid.hpp" + +// Forward class hydro declaration +#include "physics.hpp" +template class Fluid; +template class ConstrainedTransport; +template class Boundary; +class DataBlock; + +// Whether we use athena++ procedure to regularise BX2s +#define AXIS_BX2S_USE_ATHENA_REGULARISATION + +class Axis { + public: + template + explicit Axis(Boundary *); // Initialisation + void RegularizeEMFs(); // Regularize the EMF sitting on the axis + void RegularizeCurrent(); // Regularize the currents along the axis + void EnforceAxisBoundary(int side); // Enforce the boundary conditions (along X2) + void ReconstructBx2s(); // Reconstruct BX2s in the ghost zone using divB=0 + void ShowConfig(); + + + void SymmetrizeEx1Side(int); // Symmetrize on a specific side (internal method) + void RegularizeEx3side(int); // Regularize Ex3 along the axis (internal method) + void RegularizeCurrentSide(int); // Regularize J along the axis (internal method) + void FixBx2sAxis(int side); // Fix BX2s on the axis using the field around it (internal) + void FixBx2sAxisGhostAverage(int side); //Fix BX2s on the axis using the average of neighbouring + // cell in theta direction (like Athena) + void ExchangeMPI(int side); // Function has to be public for GPU, but its technically + // a private function + + + private: + bool isTwoPi = false; + bool axisRight = false; + bool axisLeft = false; + bool needMPIExchange = false; + bool haveMHD = false; + int nVar; + + enum {faceTop, faceBot}; +#ifdef WITH_MPI + MPI_Request sendRequest; + MPI_Request recvRequest; + + IdefixArray1D bufferSend; + IdefixArray1D bufferRecv; + + int bufferSize; + + IdefixArray1D mapVars; + int mapNVars{0}; + +#endif + void InitMPI(); + + IdefixArray1D Ex1Avg; + IdefixArray2D BAvg; + bool haveCurrent; + IdefixArray2D JAvg; + IdefixArray1D symmetryVc; + IdefixArray1D symmetryVs; + + IdefixArray3D ex; + IdefixArray3D ey; + IdefixArray3D ez; + IdefixArray4D J; + + IdefixArray4D Vc; + IdefixArray4D Vs; + + DataBlock *data; +}; + +#include "constrainedTransport.hpp" + +template +Axis::Axis(Boundary *boundary) { + Vc = boundary->Vc; + Vs = boundary->Vs; + J = boundary->fluid->J; + if constexpr(Phys::mhd) { + ex = boundary->fluid->emf->ex; + ey = boundary->fluid->emf->ey; + ez = boundary->fluid->emf->ez; + } + + data = boundary->data; + haveMHD = Phys::mhd; + nVar = boundary->nVar; + haveCurrent = boundary->fluid->haveCurrent; + + + #if GEOMETRY != SPHERICAL + IDEFIX_ERROR("Axis boundary conditions are only designed to handle spherical geometry"); + #endif + + + if(fabs((data->mygrid->xend[KDIR] - data->mygrid->xbeg[KDIR] -2.0*M_PI)) < 1e-10) { + this->isTwoPi = true; + #ifdef WITH_MPI + // Check that there is a domain decomposition in phi + if(data->mygrid->nproc[KDIR]>1) { + if(data->mygrid->nproc[KDIR]%2==1) { + IDEFIX_ERROR("The numbre of processes in the phi direction should" + " be even for axis decomposition"); + } + needMPIExchange = true; + } + #endif + } else { + this->isTwoPi = false; + } + + // Check where the axis is lying. + if(data->lbound[JDIR] == axis) axisLeft = true; + if(data->rbound[JDIR] == axis) axisRight = true; + + // Init the symmetry array (used to flip the signs of arrays accross the axis) + symmetryVc = IdefixArray1D("Axis:SymmetryVc",nVar); + IdefixArray1D::HostMirror symmetryVcHost = Kokkos::create_mirror_view(symmetryVc); + // Init the array + for (int nv = 0; nv < nVar; nv++) { + symmetryVcHost(nv) = 1; + if (nv == VX2) + symmetryVcHost(nv) = -1; + if (nv == VX3) + symmetryVcHost(nv) = -1; + if (nv == BX2) + symmetryVcHost(nv) = -1; + if (nv == BX3) + symmetryVcHost(nv) = -1; + } + Kokkos::deep_copy(symmetryVc, symmetryVcHost); + + if constexpr(Phys::mhd) { + idfx::cout << "Phys MHD" << std::endl; + symmetryVs = IdefixArray1D("Axis:SymmetryVs",DIMENSIONS); + IdefixArray1D::HostMirror symmetryVsHost = Kokkos::create_mirror_view(symmetryVs); + // Init the array + for(int nv = 0; nv < DIMENSIONS; nv++) { + symmetryVsHost(nv) = 1; + if (nv == BX2s) + symmetryVsHost(nv) = -1; + if (nv == BX3s) + symmetryVsHost(nv) = -1; + } + Kokkos::deep_copy(symmetryVs, symmetryVsHost); + + this->Ex1Avg = IdefixArray1D("Axis:Ex1Avg",data->np_tot[IDIR]); + this->BAvg = IdefixArray2D("Axis:BxAvg",data->np_tot[IDIR],2); + if(haveCurrent) { + this->JAvg = IdefixArray2D("Axis:JAvg",data->np_tot[IDIR],3); + } + } + + #ifdef WITH_MPI + if(needMPIExchange) { + // Make MPI exchange datatypes + InitMPI(); + } + #endif +} + + + +#endif // FLUID_BOUNDARY_AXIS_HPP_ diff --git a/src/fluid/boundary/boundary.hpp b/src/fluid/boundary/boundary.hpp new file mode 100644 index 00000000..e0d8df70 --- /dev/null +++ b/src/fluid/boundary/boundary.hpp @@ -0,0 +1,1073 @@ +// *********************************************************************************** +// Idefix MHD astrophysical code +// Copyright(C) Geoffroy R. J. Lesur +// and other code contributors +// Licensed under CeCILL 2.1 License, see COPYING for more information +// *********************************************************************************** + +#ifndef FLUID_BOUNDARY_BOUNDARY_HPP_ +#define FLUID_BOUNDARY_BOUNDARY_HPP_ +#include +#include +#include +#include "idefix.hpp" +#include "fluid_defs.hpp" +#include "grid.hpp" + +#ifdef WITH_MPI +#include "mpi.hpp" +#endif + +// forward class declaration +class DataBlock; +#include "physics.hpp" +template class Fluid; +class Axis; + +// User-defined boundary conditions signature +template +using UserDefBoundaryFunc = void (*) (Fluid *, int dir, BoundarySide side, const real t); +using UserDefBoundaryFuncOld = void (*) (DataBlock &, int dir, BoundarySide side, const real t); + // This last one is deprecated + +template +using InternalBoundaryFunc = void (*) (Fluid *, const real t); +using InternalBoundaryFuncOld = void (*) (DataBlock &, const real t); // DEPRECATED + +template +class Boundary { + public: + explicit Boundary(Fluid*); + void SetBoundaries(real); ///< Set the ghost zones in all directions + void EnforceBoundaryDir(real, int); ///< write in the ghost zone in specific direction + void ReconstructVcField(IdefixArray4D &); ///< reconstruct cell-centered magnetic field + void ReconstructNormalField(int dir); ///< reconstruct normal field using divB=0 + + void EnforceFluxBoundaries(int dir); ///< Apply boundary condition conditions to the fluxes + + void EnrollUserDefBoundary(UserDefBoundaryFuncOld); ///< Deprecated + void EnrollUserDefBoundary(UserDefBoundaryFunc); ///< User-defined boundary condition + void EnrollInternalBoundary(InternalBoundaryFuncOld); ///< Deprecated + void EnrollInternalBoundary(InternalBoundaryFunc); ///< User-defined internal boundary + void EnrollFluxBoundary(UserDefBoundaryFuncOld); + + void EnforcePeriodic(int, BoundarySide ); ///< Enforce periodic BC in direction and side + void EnforceReflective(int, BoundarySide ); ///< Enforce reflective BC in direction and side + void EnforceOutflow(int, BoundarySide ); ///< Enforce outflow BC in direction and side + void EnforceShearingBox(real, int, BoundarySide ); ///< Enforce Shearing box BCs + + #ifdef WITH_MPI + Mpi mpi; ///< Mpi object when WITH_MPI is set + #endif + + // User defined Boundary conditions + UserDefBoundaryFuncOld userDefBoundaryFuncOld{NULL}; + UserDefBoundaryFunc userDefBoundaryFunc{NULL}; + bool haveUserDefBoundary{false}; + + // Internal boundary function + bool haveInternalBoundary{false}; + InternalBoundaryFuncOld internalBoundaryFuncOld{NULL}; + InternalBoundaryFunc internalBoundaryFunc{NULL}; + + // Flux boundary function + bool haveFluxBoundary{false}; + UserDefBoundaryFuncOld fluxBoundaryFunc{NULL}; + + // specific for loops on ghost cells + template + void BoundaryFor(const std::string &, + const int &, + const BoundarySide &, + Function ); + + template + void BoundaryForAll(const std::string &, + const int &, + const BoundarySide &, + Function ); + + template + void BoundaryForX1s(const std::string &, + const int &, + const BoundarySide &, + Function ); + + template + void BoundaryForX2s(const std::string &, + const int &, + const BoundarySide &, + Function ); + + template + void BoundaryForX3s(const std::string &, + const int &, + const BoundarySide &, + Function ); + IdefixArray4D sBArray; ///< Array use by shearingbox boundary conditions + + IdefixArray4D Vc; ///< reference to cell-centered array that we should sync + IdefixArray4D Vs; ///< reference to face-centered array that we should sync + std::unique_ptr axis; ///< Axis object, initialised if needed. + bool haveAxis{false}; + + private: + friend class Axis; + Fluid *fluid; // pointer to parent hydro object + DataBlock *data; // pointer to parent datablock + int nVar; // # of variables involved in the boundary conditions +}; + +#include "fluid.hpp" +#include "axis.hpp" + +template +Boundary::Boundary(Fluid* fluid) { + idfx::pushRegion("Boundary::Boundary"); + this->fluid = fluid; + this->data = fluid->data; + this->Vc = fluid->Vc; + this->Vs = fluid->Vs; + this->nVar = Phys::nvar; + if(fluid->haveTracer) this->nVar += fluid->nTracer; + + // Do we have to take care of the axis? + if(data->haveAxis) { + this->axis = std::make_unique(this); + this->haveAxis = true; + } + + + if(data->lbound[IDIR] == shearingbox || data->rbound[IDIR] == shearingbox) { + // using np_tot[...]+1 points to allow this buffer to represent + // fields that are defined on faces + sBArray = IdefixArray4D("ShearingBoxArray", + nVar, + data->np_tot[KDIR]+1, + data->np_tot[JDIR]+1, + data->nghost[IDIR]); + } + + // Init MPI stack when needed +#ifdef WITH_MPI + //////////////////////////////////////////////////////////////////////////// + // Init variable mappers + // The variable mapper list all of the variable which are exchanged in MPI boundary calls + // This is required since we skip some of the variables in Vc to limit the amount of data + // being exchanged + int mapNVars = nVar; + if(Phys::mhd) mapNVars -= DIMENSIONS; + + std::vector mapVars; + + // Init the list of variables we will exchange in MPI routines + int ntarget = 0; + for(int n = 0 ; n < mapNVars ; n++) { + mapVars.push_back(ntarget); + ntarget++; + if(Phys::mhd) { + // Skip centered field components if they are also defined in Vs + #if DIMENSIONS >= 1 + if(ntarget==BX1) ntarget++; + #endif + #if DIMENSIONS >= 2 + if(ntarget==BX2) ntarget++; + #endif + #if DIMENSIONS == 3 + if(ntarget==BX3) ntarget++; + #endif + } + } + + mpi.Init(data->mygrid, mapVars, data->nghost.data(), data->np_int.data(), Phys::mhd); + +#endif // MPI + idfx::popRegion(); +} + +template +void Boundary::EnrollFluxBoundary(UserDefBoundaryFuncOld myFunc) { + this->haveFluxBoundary = true; + this->fluxBoundaryFunc = myFunc; +} + +template +void Boundary::EnforceFluxBoundaries(int dir) { + idfx::pushRegion("Boundary::EnforceFluxBoundaries"); + if(haveFluxBoundary) { + if(data->lbound[dir] != internal) { + fluxBoundaryFunc(*data, dir, left, data->t); + } + if(data->rbound[dir] != internal) { + fluxBoundaryFunc(*data, dir, right, data->t); + } + } else { + IDEFIX_ERROR("Cannot enforce flux boundary conditions without enrolling a specific function"); + } + idfx::popRegion(); +} + +template +void Boundary::SetBoundaries(real t) { + idfx::pushRegion("Boundary::SetBoundaries"); + // set internal boundary conditions + if(haveInternalBoundary) { + idfx::pushRegion("Boundary::UserDefInternalBoundary"); + if(internalBoundaryFunc != NULL) { + internalBoundaryFunc(fluid, t); + } else { + internalBoundaryFuncOld(*data, t); + } + idfx::popRegion(); + } + for(int dir=0 ; dir < DIMENSIONS ; dir++ ) { + // MPI Exchange data when needed + #ifdef WITH_MPI + if(data->mygrid->nproc[dir]>1) { + switch(dir) { + case 0: + mpi.ExchangeX1(this->Vc, this->Vs); + break; + case 1: + mpi.ExchangeX2(this->Vc, this->Vs); + break; + case 2: + mpi.ExchangeX3(this->Vc, this->Vs); + break; + } + } + #endif + EnforceBoundaryDir(t, dir); + if constexpr(Phys::mhd) { + // Reconstruct the normal field component when using CT + ReconstructNormalField(dir); + } + } // Loop on dimension ends + + if constexpr(Phys::mhd) { + // Remake the cell-centered field. + ReconstructVcField(this->Vc); + } + + idfx::popRegion(); +} + + +// Enforce boundary conditions by writing into ghost zones +template +void Boundary::EnforceBoundaryDir(real t, int dir) { + idfx::pushRegion("Boundary::EnforceBoundaryDir"); + + // left boundary + + switch(data->lbound[dir]) { + case BoundaryType::internal: + // internal is used for MPI-enforced boundary conditions. Nothing to be done here. + break; + + case BoundaryType::periodic: + if(data->mygrid->nproc[dir] > 1) break; // Periodicity already enforced by MPI calls + EnforcePeriodic(dir,left); + break; + + case BoundaryType::reflective: + EnforceReflective(dir,left); + break; + + case BoundaryType::outflow: + EnforceOutflow(dir,left); + break; + + case BoundaryType::shearingbox: + EnforceShearingBox(t,dir,left); + break; + case BoundaryType::axis: + axis->EnforceAxisBoundary(left); + break; + case BoundaryType::userdef: + if(this->haveUserDefBoundary) { + idfx::pushRegion("Boundary::UserDefBoundary"); + if(this->userDefBoundaryFunc != NULL) { + this->userDefBoundaryFunc(fluid, dir, left, t); + } else { + // Revert to deprecated Boundary condition + this->userDefBoundaryFuncOld(*data, dir, left, t); + } + idfx::popRegion(); + } else { + std::stringstream msg; + msg << "No function has been enrolled to define your own boundary conditions" << std::endl + << "for the fluid " << fluid->prefix << "." << std::endl; + IDEFIX_ERROR(msg); + } + break; + + default: + std::stringstream msg ("Boundary condition type is not yet implemented"); + IDEFIX_ERROR(msg); + } + + // right boundary + + switch(data->rbound[dir]) { + case BoundaryType::internal: + // internal is used for MPI-enforced boundary conditions. Nothing to be done here. + break; + + case BoundaryType::periodic: + if(data->mygrid->nproc[dir] > 1) break; // Periodicity already enforced by MPI calls + EnforcePeriodic(dir,right); + break; + case BoundaryType::reflective: + EnforceReflective(dir,right); + break; + case BoundaryType::outflow: + EnforceOutflow(dir,right); + break; + case BoundaryType::shearingbox: + EnforceShearingBox(t,dir,right); + break; + case BoundaryType::axis: + axis->EnforceAxisBoundary(right); + break; + case BoundaryType::userdef: + if(this->haveUserDefBoundary) { + idfx::pushRegion("Boundary::UserDefBoundary"); + if(this->userDefBoundaryFunc != NULL) { + this->userDefBoundaryFunc(fluid, dir, right, t); + } else { + // Revert to deprecated Boundary condition + this->userDefBoundaryFuncOld(*data, dir, right, t); + } + idfx::popRegion(); + } else { + std::stringstream msg; + msg << "No function has been enrolled to define your own boundary conditions" << std::endl + << "for the fluid " << fluid->prefix << "." << std::endl; + IDEFIX_ERROR(msg); + } + break; + default: + std::stringstream msg("Boundary condition type is not yet implemented"); + IDEFIX_ERROR(msg); + } + + idfx::popRegion(); +} + + +template +void Boundary::ReconstructVcField(IdefixArray4D &Vc) { + idfx::pushRegion("Boundary::ReconstructVcField"); + + IdefixArray4D Vs=this->Vs; + + // Reconstruct cell average field when using CT + idefix_for("ReconstructVcMagField",0,data->np_tot[KDIR],0,data->np_tot[JDIR],0,data->np_tot[IDIR], + KOKKOS_LAMBDA (int k, int j, int i) { + D_EXPAND( Vc(BX1,k,j,i) = HALF_F * (Vs(BX1s,k,j,i) + Vs(BX1s,k,j,i+1)) ; , + Vc(BX2,k,j,i) = HALF_F * (Vs(BX2s,k,j,i) + Vs(BX2s,k,j+1,i)) ; , + Vc(BX3,k,j,i) = HALF_F * (Vs(BX3s,k,j,i) + Vs(BX3s,k+1,j,i)) ; ) + } + ); + + idfx::popRegion(); +} + + +template +void Boundary::ReconstructNormalField(int dir) { + idfx::pushRegion("Boundary::ReconstructNormalField"); + + const bool reconstructLeft = !(data->lbound[dir]==periodic || data->lbound[dir]==internal); + const bool reconstructRight = !(data->rbound[dir]==periodic || data->rbound[dir]==internal); + + // nothing to reconstruct in that direction! + if((!reconstructLeft) && (!reconstructRight)) { + idfx::popRegion(); + return; + } + + // Reconstruct the field + IdefixArray4D Vs = this->Vs; + // Coordinates + IdefixArray1D x1=data->x[IDIR]; + IdefixArray1D x2=data->x[JDIR]; + IdefixArray1D x3=data->x[KDIR]; + IdefixArray1D dx1=data->dx[IDIR]; + IdefixArray1D dx2=data->dx[JDIR]; + IdefixArray1D dx3=data->dx[KDIR]; + + IdefixArray3D Ax1=data->A[IDIR]; + IdefixArray3D Ax2=data->A[JDIR]; + IdefixArray3D Ax3=data->A[KDIR]; + + + // reconstruct BX1s + int nstart = data->beg[IDIR]-1; + int nend = data->end[IDIR]; + + const int nx1=data->np_tot[IDIR]; + const int nx2=data->np_tot[JDIR]; + const int nx3=data->np_tot[KDIR]; + + if(dir==IDIR) { + idefix_for("ReconstructBX1s",0,nx3,0,nx2, + KOKKOS_LAMBDA (int k, int j) { + if(reconstructLeft) { + for(int i = nstart ; i>=0 ; i-- ) { + Vs(BX1s,k,j,i) = 1.0 / Ax1(k,j,i) * ( Ax1(k,j,i+1)*Vs(BX1s,k,j,i+1) + +(D_EXPAND( ZERO_F , + + Ax2(k,j+1,i) * Vs(BX2s,k,j+1,i) - Ax2(k,j,i) * Vs(BX2s,k,j,i) , + + Ax3(k+1,j,i) * Vs(BX3s,k+1,j,i) - Ax3(k,j,i) * Vs(BX3s,k,j,i) ))); + } + } + + if(reconstructRight) { + for(int i = nend ; i=2 + if(dir==JDIR) { + nstart = data->beg[JDIR]-1; + nend = data->end[JDIR]; + if(!this->haveAxis) { + idefix_for("ReconstructBX2s",0,data->np_tot[KDIR],0,data->np_tot[IDIR], + KOKKOS_LAMBDA (int k, int i) { + if(reconstructLeft) { + for(int j = nstart ; j>=0 ; j-- ) { + Vs(BX2s,k,j,i) = 1.0 / Ax2(k,j,i) * ( Ax2(k,j+1,i)*Vs(BX2s,k,j+1,i) + +(D_EXPAND( Ax1(k,j,i+1) * Vs(BX1s,k,j,i+1) - Ax1(k,j,i) * Vs(BX1s,k,j,i) , + , + + Ax3(k+1,j,i) * Vs(BX3s,k+1,j,i) - Ax3(k,j,i) * Vs(BX3s,k,j,i) ))); + } + } + if(reconstructRight) { + for(int j = nend ; jReconstructBx2s(); + } + } +#endif + +#if DIMENSIONS == 3 + if(dir==KDIR) { + nstart = data->beg[KDIR]-1; + nend = data->end[KDIR]; + + idefix_for("ReconstructBX3s",0,data->np_tot[JDIR],0,data->np_tot[IDIR], + KOKKOS_LAMBDA (int j, int i) { + if(reconstructLeft) { + for(int k = nstart ; k>=0 ; k-- ) { + Vs(BX3s,k,j,i) = 1.0 / Ax3(k,j,i) * ( Ax3(k+1,j,i)*Vs(BX3s,k+1,j,i) + + ( Ax1(k,j,i+1) * Vs(BX1s,k,j,i+1) - Ax1(k,j,i) * Vs(BX1s,k,j,i) + + Ax2(k,j+1,i) * Vs(BX2s,k,j+1,i) - Ax2(k,j,i) * Vs(BX2s,k,j,i) )); + } + } + if(reconstructRight) { + for(int k = nend ; k +void Boundary::EnrollUserDefBoundary(UserDefBoundaryFuncOld myFunc) { + std::stringstream msg; + msg << "The old signature for user-defined boundary condition " << std::endl + << "(DataBlock &, int dir, BoundarySide side, const real t)" << std::endl + << "is deprecated. You should now use "<< std::endl + << "(Fluid *, int dir, BoundarySide side, const real t)" << std::endl + << "With the Phys of your choice (DefaultPhysics, DustPhysics...)" << std::endl; + + IDEFIX_WARNING(msg); + this->userDefBoundaryFuncOld = myFunc; + this->haveUserDefBoundary = true; +} + +template +void Boundary::EnrollUserDefBoundary(UserDefBoundaryFunc myFunc) { + std::stringstream msg; + + this->userDefBoundaryFunc = myFunc; + this->haveUserDefBoundary = true; +} + + +template +void Boundary::EnrollInternalBoundary(InternalBoundaryFuncOld myFunc) { + std::stringstream msg; + msg << "The old signature for internal boundary condition " << std::endl + << "(DataBlock &, const real t)" << std::endl + << "is deprecated. You should now use "<< std::endl + << "(Fluid *, const real t)" << std::endl + << "With the Phys of your choice (DefaultPhysics, DustPhysics...)" << std::endl; + + IDEFIX_WARNING(msg); + this->internalBoundaryFuncOld = myFunc; + this->haveInternalBoundary = true; +} + +template +void Boundary::EnrollInternalBoundary(InternalBoundaryFunc myFunc) { + this->internalBoundaryFunc = myFunc; + this->haveInternalBoundary = true; +} + +template +void Boundary::EnforcePeriodic(int dir, BoundarySide side ) { + idfx::pushRegion("Boundary::EnforcePeriodic"); + IdefixArray4D Vc = this->Vc; + int nxi = data->np_int[IDIR]; + int nxj = data->np_int[JDIR]; + int nxk = data->np_int[KDIR]; + + const int ighost = data->nghost[IDIR]; + const int jghost = data->nghost[JDIR]; + const int kghost = data->nghost[KDIR]; + + BoundaryForAll("BoundaryPeriodic", dir, side, + KOKKOS_LAMBDA (int n, int k, int j, int i) { + int iref, jref, kref; + // This hack takes care of cases where we have more ghost zones than active zones + if(dir==IDIR) + iref = ighost + (i+ighost*(nxi-1))%nxi; + else + iref = i; + if(dir==JDIR) + jref = jghost + (j+jghost*(nxj-1))%nxj; + else + jref = j; + if(dir==KDIR) + kref = kghost + (k+kghost*(nxk-1))%nxk; + else + kref = k; + + Vc(n,k,j,i) = Vc(n,kref,jref,iref); + }); + + if constexpr(Phys::mhd) { + IdefixArray4D Vs = this->Vs; + BoundaryForX1s("BoundaryPeriodicX1s",dir,side, + KOKKOS_LAMBDA (int k, int j, int i) { + int iref, jref, kref; + // This hack takes care of cases where we have more ghost zones than active zones + if(dir==IDIR) + iref = ighost + (i+ighost*(nxi-1))%nxi; + else + iref = i; + if(dir==JDIR) + jref = jghost + (j+jghost*(nxj-1))%nxj; + else + jref = j; + if(dir==KDIR) + kref = kghost + (k+kghost*(nxk-1))%nxk; + else + kref = k; + + Vs(BX1s,k,j,i) = Vs(BX1s,kref,jref,iref); + }); + #if DIMENSIONS >=2 + BoundaryForX2s("BoundaryPeriodicX2s",dir,side, + KOKKOS_LAMBDA (int k, int j, int i) { + int iref, jref, kref; + // This hack takes care of cases where we have more ghost zones than active zones + if(dir==IDIR) + iref = ighost + (i+ighost*(nxi-1))%nxi; + else + iref = i; + if(dir==JDIR) + jref = jghost + (j+jghost*(nxj-1))%nxj; + else + jref = j; + if(dir==KDIR) + kref = kghost + (k+kghost*(nxk-1))%nxk; + else + kref = k; + + Vs(BX2s,k,j,i) = Vs(BX2s,kref,jref,iref); + }); + #endif + #if DIMENSIONS == 3 + BoundaryForX3s("BoundaryPeriodicX3s",dir,side, + KOKKOS_LAMBDA (int k, int j, int i) { + int iref, jref, kref; + // This hack takes care of cases where we have more ghost zones than active zones + if(dir==IDIR) + iref = ighost + (i+ighost*(nxi-1))%nxi; + else + iref = i; + if(dir==JDIR) + jref = jghost + (j+jghost*(nxj-1))%nxj; + else + jref = j; + if(dir==KDIR) + kref = kghost + (k+kghost*(nxk-1))%nxk; + else + kref = k; + + Vs(BX3s,k,j,i) = Vs(BX3s,kref,jref,iref); + }); + #endif + }// MHD + idfx::popRegion(); +} + + +template +void Boundary::EnforceReflective(int dir, BoundarySide side ) { + idfx::pushRegion("Boundary::EnforceReflective"); + IdefixArray4D Vc = this->Vc; + const int nxi = data->np_int[IDIR]; + const int nxj = data->np_int[JDIR]; + const int nxk = data->np_int[KDIR]; + + const int ighost = data->nghost[IDIR]; + const int jghost = data->nghost[JDIR]; + const int kghost = data->nghost[KDIR]; + + BoundaryForAll("BoundaryReflective", dir, side, + KOKKOS_LAMBDA (int n, int k, int j, int i) { + // ref= 2*ibound -i -1 + // with ibound = nghost on the left and ibount = nghost + nx -1 on the right + const int iref = (dir==IDIR) ? 2*(ighost + side*(nxi-1)) - i - 1 : i; + const int jref = (dir==JDIR) ? 2*(jghost + side*(nxj-1)) - j - 1 : j; + const int kref = (dir==KDIR) ? 2*(kghost + side*(nxk-1)) - k - 1 : k; + + const int sign = (n == VX1+dir) ? -1.0 : 1.0; + + Vc(n,k,j,i) = sign * Vc(n,kref,jref,iref); + }); + + if constexpr(Phys::mhd) { + IdefixArray4D Vs = this->Vs; + if(dir==JDIR || dir==KDIR) { + BoundaryForX1s("BoundaryReflectiveX1s",dir,side, + KOKKOS_LAMBDA (int k, int j, int i) { + // ref= 2*ibound -i -1 + // with ibound = nghost on the left and ibount = nghost + nx -1 on the right + //const int iref = (dir==IDIR) ? 2*(ighost + side*(nxi-1)) - i - 1 : i; + const int jref = (dir==JDIR) ? 2*(jghost + side*(nxj-1)) - j - 1 : j; + const int kref = (dir==KDIR) ? 2*(kghost + side*(nxk-1)) - k - 1 : k; + + Vs(BX1s,k,j,i) = -Vs(BX1s,kref,jref,i); + }); + } + #if DIMENSIONS >=2 + if(dir==IDIR || dir==KDIR) { + BoundaryForX2s("BoundaryReflectiveX2s",dir,side, + KOKKOS_LAMBDA (int k, int j, int i) { + const int iref = (dir==IDIR) ? 2*(ighost + side*(nxi-1)) - i - 1 : i; + //const int jref = (dir==JDIR) ? 2*(jghost + side*(nxj-1)) - j - 1 : j; + const int kref = (dir==KDIR) ? 2*(kghost + side*(nxk-1)) - k - 1 : k; + + Vs(BX2s,k,j,i) = -Vs(BX2s,kref,j,iref); + }); + } + #endif + #if DIMENSIONS == 3 + if(dir==IDIR || dir==JDIR) { + BoundaryForX3s("BoundaryReflectiveX3s",dir,side, + KOKKOS_LAMBDA (int k, int j, int i) { + const int iref = (dir==IDIR) ? 2*(ighost + side*(nxi-1)) - i - 1 : i; + const int jref = (dir==JDIR) ? 2*(jghost + side*(nxj-1)) - j - 1 : j; + //const int kref = (dir==KDIR) ? 2*(kghost + side*(nxk-1)) - k - 1 : k; + + Vs(BX3s,k,j,i) = -Vs(BX3s,k,jref,iref); + }); + } + #endif + }// MHD + idfx::popRegion(); +} + +template +void Boundary::EnforceOutflow(int dir, BoundarySide side ) { + idfx::pushRegion("Boundary::EnforceOutflow"); + IdefixArray4D Vc = this->Vc; + const int nxi = data->np_int[IDIR]; + const int nxj = data->np_int[JDIR]; + const int nxk = data->np_int[KDIR]; + + const int ighost = data->nghost[IDIR]; + const int jghost = data->nghost[JDIR]; + const int kghost = data->nghost[KDIR]; + + BoundaryForAll("BoundaryOutflow", dir, side, + KOKKOS_LAMBDA (int n, int k, int j, int i) { + // ref= ibound + // with ibound = nghost on the left and ibound = nghost + nx -1 on the right + const int iref = (dir==IDIR) ? ighost + side*(nxi-1) : i; + const int jref = (dir==JDIR) ? jghost + side*(nxj-1) : j; + const int kref = (dir==KDIR) ? kghost + side*(nxk-1) : k; + + // should it go inwards or outwards? + // side = 1 on the left and =-1 on the right + const int sign = 1-2*side; + + if( (n== VX1+dir) && (sign*Vc(n,kref,jref,iref) >= ZERO_F) ) { + Vc(n,k,j,i) = ZERO_F; + } else { + Vc(n,k,j,i) = Vc(n,kref,jref,iref); + } + }); + + if constexpr(Phys::mhd) { + IdefixArray4D Vs = this->Vs; + if(dir==JDIR || dir==KDIR) { + BoundaryForX1s("BoundaryOutflowX1s",dir,side, + KOKKOS_LAMBDA (int k, int j, int i) { + // with ibound = nghost on the left and ibount = nghost + nx -1 on the right + //const int iref = (dir==IDIR) ? ighost + side*(nxi-1) : i; + const int jref = (dir==JDIR) ? jghost + side*(nxj-1) : j; + const int kref = (dir==KDIR) ? kghost + side*(nxk-1) : k; + + Vs(BX1s,k,j,i) = Vs(BX1s,kref,jref,i); + }); + } + #if DIMENSIONS >=2 + if(dir==IDIR || dir==KDIR) { + BoundaryForX2s("BoundaryOutflowX2s",dir,side, + KOKKOS_LAMBDA (int k, int j, int i) { + const int iref = (dir==IDIR) ? ighost + side*(nxi-1) : i; + //const int jref = (dir==JDIR) ? jghost + side*(nxj-1) : j; + const int kref = (dir==KDIR) ? kghost + side*(nxk-1) : k; + + Vs(BX2s,k,j,i) = Vs(BX2s,kref,j,iref); + }); + } + #endif + #if DIMENSIONS == 3 + if(dir==IDIR || dir==JDIR) { + BoundaryForX3s("BoundaryOutflowX3s",dir,side, + KOKKOS_LAMBDA (int k, int j, int i) { + const int iref = (dir==IDIR) ? ighost + side*(nxi-1) : i; + const int jref = (dir==JDIR) ? jghost + side*(nxj-1) : j; + //const int kref = (dir==KDIR) ? kghost + side*(nxk-1) : k; + + Vs(BX3s,k,j,i) = Vs(BX3s,k,jref,iref); + }); + } + #endif + }// MHD + idfx::popRegion(); +} + +template +void Boundary::EnforceShearingBox(real t, int dir, BoundarySide side) { + idfx::pushRegion("Boundary::EnforceShearingBox"); + if(dir != IDIR) + IDEFIX_ERROR("Shearing box boundaries can only be applied along the X1 direction"); + if(data->mygrid->nproc[JDIR]>1) + IDEFIX_ERROR("Shearing box is not yet compatible with domain decomposition in X2"); + + // First thing is to enforce periodicity (already performed by MPI) + if(data->mygrid->nproc[dir] == 1) EnforcePeriodic(dir, side); + + IdefixArray4D scrh = sBArray; + IdefixArray4D Vc = this->Vc; + + const int nxi = data->np_int[IDIR]; + const int nxj = data->np_int[JDIR]; + + const int ighost = data->nghost[IDIR]; + const int jghost = data->nghost[JDIR]; + + // Where does the boundary starts along x1? + const int istart = side*(ighost+nxi); + + // Shear rate + const real S = fluid->sbS; + + // Box size + const real Lx = data->mygrid->xend[IDIR] - data->mygrid->xbeg[IDIR]; + const real Ly = data->mygrid->xend[JDIR] - data->mygrid->xbeg[JDIR]; + + // total number of cells in y (active domain) + const int ny = data->mygrid->np_int[JDIR]; + const real dy = Ly/ny; + + // Compute offset in y modulo the box size + const int sign=2*side-1; + const real sbVelocity = sign*S*Lx; + real dL = std::fmod(sbVelocity*t,Ly); + + // translate this into # of cells + const int m = static_cast (std::floor(dL/dy+HALF_F)); + + // remainding shift + const real eps = dL / dy - m; + + + // Now we need to perform the shift + BoundaryForAll("BoundaryShearingBox", dir, side, + KOKKOS_LAMBDA ( int n, int k, int j, int i) { + // jorigin + const int jo = jghost + ((j-m-jghost)%nxj+nxj)%nxj; + const int jop2 = jghost + ((jo+2-jghost)%nxj+nxj)%nxj; + const int jop1 = jghost + ((jo+1-jghost)%nxj+nxj)%nxj; + const int jom1 = jghost + ((jo-1-jghost)%nxj+nxj)%nxj; + const int jom2 = jghost + ((jo-2-jghost)%nxj+nxj)%nxj; + + // Define Left and right fluxes + // Fluxes are defined from slope-limited interpolation + // Using Van-leer slope limiter (consistently with the main advection scheme) + real Fl,Fr; + real dqm, dqp, dq; + + if(eps>=ZERO_F) { + // Compute Fl + dqm = Vc(n,k,jom1,i) - Vc(n,k,jom2,i); + dqp = Vc(n,k,jo,i) - Vc(n,k,jom1,i); + dq = (dqp*dqm > ZERO_F ? TWO_F*dqp*dqm/(dqp + dqm) : ZERO_F); + + Fl = Vc(n,k,jom1,i) + 0.5*dq*(1.0-eps); + //Compute Fr + dqm=dqp; + dqp = Vc(n,k,jop1,i) - Vc(n,k,jo,i); + dq = (dqp*dqm > ZERO_F ? TWO_F*dqp*dqm/(dqp + dqm) : ZERO_F); + + Fr = Vc(n,k,jo,i) + 0.5*dq*(1.0-eps); + } else { + //Compute Fl + dqm = Vc(n,k,jo,i) - Vc(n,k,jom1,i); + dqp = Vc(n,k,jop1,i) - Vc(n,k,jo,i); + dq = (dqp*dqm > ZERO_F ? TWO_F*dqp*dqm/(dqp + dqm) : ZERO_F); + + Fl = Vc(n,k,jo,i) - 0.5*dq*(1.0+eps); + // Compute Fr + dqm=dqp; + dqp = Vc(n,k,jop2,i) - Vc(n,k,jop1,i); + dq = (dqp*dqm > ZERO_F ? TWO_F*dqp*dqm/(dqp + dqm) : ZERO_F); + + Fr = Vc(n,k,jop1,i) - 0.5*dq*(1.0+eps); + } + scrh(n,k,j,i-istart) = Vc(n,k,jo,i) - eps*(Fr - Fl); + }); + // Copy scrach back to our boundary + BoundaryForAll("BoundaryShearingBoxCopy", dir, side, + KOKKOS_LAMBDA ( int n, int k, int j, int i) { + Vc(n,k,j,i) = scrh(n,k,j,i-istart); + if(n==VX2) Vc(n,k,j,i) += sbVelocity; + }); + + // Magnetised version of the same thing + if constexpr(Phys::mhd) { + IdefixArray4D Vs = this->Vs; + #if DIMENSIONS >= 2 + for(int component = BX2s ; component < DIMENSIONS ; component++) { + BoundaryFor("BoundaryShearingBoxBXs", dir, side, + KOKKOS_LAMBDA (int k, int j, int i) { + // jorigin + const int jo = jghost + ((j-m-jghost)%nxj+nxj)%nxj; + const int jop2 = jghost + ((jo+2-jghost)%nxj+nxj)%nxj; + const int jop1 = jghost + ((jo+1-jghost)%nxj+nxj)%nxj; + const int jom1 = jghost + ((jo-1-jghost)%nxj+nxj)%nxj; + const int jom2 = jghost + ((jo-2-jghost)%nxj+nxj)%nxj; + + // Define Left and right fluxes + // Fluxes are defined from slope-limited interpolation + // Using Van-leer slope limiter (consistently with the main advection scheme) + real Fl,Fr; + real dqm, dqp, dq; + + if(eps>=ZERO_F) { + // Compute Fl + dqm = Vs(component,k,jom1,i) - Vs(component,k,jom2,i); + dqp = Vs(component,k,jo,i) - Vs(component,k,jom1,i); + dq = (dqp*dqm > ZERO_F ? TWO_F*dqp*dqm/(dqp + dqm) : ZERO_F); + + Fl = Vs(component,k,jom1,i) + 0.5*dq*(1.0-eps); + //Compute Fr + dqm=dqp; + dqp = Vs(component,k,jop1,i) - Vs(component,k,jo,i); + dq = (dqp*dqm > ZERO_F ? TWO_F*dqp*dqm/(dqp + dqm) : ZERO_F); + + Fr = Vs(component,k,jo,i) + 0.5*dq*(1.0-eps); + } else { + //Compute Fl + dqm = Vs(component,k,jo,i) - Vs(component,k,jom1,i); + dqp = Vs(component,k,jop1,i) - Vs(component,k,jo,i); + dq = (dqp*dqm > ZERO_F ? TWO_F*dqp*dqm/(dqp + dqm) : ZERO_F); + + Fl = Vs(component,k,jo,i) - 0.5*dq*(1.0+eps); + // Compute Fr + dqm=dqp; + dqp = Vs(component,k,jop2,i) - Vs(component,k,jop1,i); + dq = (dqp*dqm > ZERO_F ? TWO_F*dqp*dqm/(dqp + dqm) : ZERO_F); + + Fr = Vs(component,k,jop1,i) - 0.5*dq*(1.0+eps); + } + scrh(0,k,j,i-istart) = Vs(component,k,jo,i) - eps*(Fr - Fl); + }); + // Copy scratch back to our boundary + BoundaryFor("BoundaryShearingBoxCopyBXs", dir, side, + KOKKOS_LAMBDA ( int k, int j, int i) { + Vs(component,k,j,i) = scrh(0,k,j,i-istart); + }); + }// loop on components + #endif// DIMENSIONS + } // MHD + idfx::popRegion(); +} + +template +template +inline void Boundary::BoundaryForAll( + const std::string & name, + const int &dir, + const BoundarySide &side, + Function function) { + const int nxi = data->np_int[IDIR]; + const int nxj = data->np_int[JDIR]; + const int nxk = data->np_int[KDIR]; + + const int ighost = data->nghost[IDIR]; + const int jghost = data->nghost[JDIR]; + const int kghost = data->nghost[KDIR]; + + // Boundaries of the loop + const int ibeg = (dir == IDIR) ? side*(ighost+nxi) : 0; + const int iend = (dir == IDIR) ? ighost + side*(ighost+nxi) : data->np_tot[IDIR]; + const int jbeg = (dir == JDIR) ? side*(jghost+nxj) : 0; + const int jend = (dir == JDIR) ? jghost + side*(jghost+nxj) : data->np_tot[JDIR]; + const int kbeg = (dir == KDIR) ? side*(kghost+nxk) : 0; + const int kend = (dir == KDIR) ? kghost + side*(kghost+nxk) : data->np_tot[KDIR]; + + idefix_for(name, 0, this->nVar, kbeg, kend, jbeg, jend, ibeg, iend, function); +} + +template +template +inline void Boundary::BoundaryFor( + const std::string & name, + const int &dir, + const BoundarySide &side, + Function function) { + const int nxi = data->np_int[IDIR]; + const int nxj = data->np_int[JDIR]; + const int nxk = data->np_int[KDIR]; + + const int ighost = data->nghost[IDIR]; + const int jghost = data->nghost[JDIR]; + const int kghost = data->nghost[KDIR]; + + // Boundaries of the loop + const int ibeg = (dir == IDIR) ? side*(ighost+nxi) : 0; + const int iend = (dir == IDIR) ? ighost + side*(ighost+nxi) : data->np_tot[IDIR]; + const int jbeg = (dir == JDIR) ? side*(jghost+nxj) : 0; + const int jend = (dir == JDIR) ? jghost + side*(jghost+nxj) : data->np_tot[JDIR]; + const int kbeg = (dir == KDIR) ? side*(kghost+nxk) : 0; + const int kend = (dir == KDIR) ? kghost + side*(kghost+nxk) : data->np_tot[KDIR]; + + + + idefix_for(name, kbeg, kend, jbeg, jend, ibeg, iend, function); +} + +template +template +inline void Boundary::BoundaryForX1s( + const std::string & name, + const int &dir, + const BoundarySide &side, + Function function) { + const int nxi = data->np_int[IDIR]+1; + const int nxj = data->np_int[JDIR]; + const int nxk = data->np_int[KDIR]; + + const int ighost = data->nghost[IDIR]; + const int jghost = data->nghost[JDIR]; + const int kghost = data->nghost[KDIR]; + + // Boundaries of the loop + const int ibeg = (dir == IDIR) ? side*(ighost+nxi) : 0; + const int iend = (dir == IDIR) ? ighost + side*(ighost+nxi) : data->np_tot[IDIR]+1; + const int jbeg = (dir == JDIR) ? side*(jghost+nxj) : 0; + const int jend = (dir == JDIR) ? jghost + side*(jghost+nxj) : data->np_tot[JDIR]; + const int kbeg = (dir == KDIR) ? side*(kghost+nxk) : 0; + const int kend = (dir == KDIR) ? kghost + side*(kghost+nxk) : data->np_tot[KDIR]; + + idefix_for(name, kbeg, kend, jbeg, jend, ibeg, iend, function); +} + +template +template +inline void Boundary::BoundaryForX2s( + const std::string & name, + const int &dir, + const BoundarySide &side, + Function function) { + const int nxi = data->np_int[IDIR]; + const int nxj = data->np_int[JDIR]+1; + const int nxk = data->np_int[KDIR]; + + const int ighost = data->nghost[IDIR]; + const int jghost = data->nghost[JDIR]; + const int kghost = data->nghost[KDIR]; + + // Boundaries of the loop + const int ibeg = (dir == IDIR) ? side*(ighost+nxi) : 0; + const int iend = (dir == IDIR) ? ighost + side*(ighost+nxi) : data->np_tot[IDIR]; + const int jbeg = (dir == JDIR) ? side*(jghost+nxj) : 0; + const int jend = (dir == JDIR) ? jghost + side*(jghost+nxj) : data->np_tot[JDIR]+1; + const int kbeg = (dir == KDIR) ? side*(kghost+nxk) : 0; + const int kend = (dir == KDIR) ? kghost + side*(kghost+nxk) : data->np_tot[KDIR]; + + idefix_for(name, kbeg, kend, jbeg, jend, ibeg, iend, function); +} + +template +template +inline void Boundary::BoundaryForX3s( + const std::string & name, + const int &dir, + const BoundarySide &side, + Function function) { + const int nxi = data->np_int[IDIR]; + const int nxj = data->np_int[JDIR]; + const int nxk = data->np_int[KDIR]+1; + + const int ighost = data->nghost[IDIR]; + const int jghost = data->nghost[JDIR]; + const int kghost = data->nghost[KDIR]; + + // Boundaries of the loop + const int ibeg = (dir == IDIR) ? side*(ighost+nxi) : 0; + const int iend = (dir == IDIR) ? ighost + side*(ighost+nxi) : data->np_tot[IDIR]; + const int jbeg = (dir == JDIR) ? side*(jghost+nxj) : 0; + const int jend = (dir == JDIR) ? jghost + side*(jghost+nxj) : data->np_tot[JDIR]; + const int kbeg = (dir == KDIR) ? side*(kghost+nxk) : 0; + const int kend = (dir == KDIR) ? kghost + side*(kghost+nxk) : data->np_tot[KDIR]+1; + + idefix_for(name, kbeg, kend, jbeg, jend, ibeg, iend, function); +} + + +#endif // FLUID_BOUNDARY_BOUNDARY_HPP_ diff --git a/src/fluid/braginskii/CMakeLists.txt b/src/fluid/braginskii/CMakeLists.txt new file mode 100644 index 00000000..8e2198c2 --- /dev/null +++ b/src/fluid/braginskii/CMakeLists.txt @@ -0,0 +1,6 @@ +target_sources(idefix + PUBLIC ${CMAKE_CURRENT_LIST_DIR}/bragThermalDiffusion.hpp + PUBLIC ${CMAKE_CURRENT_LIST_DIR}/bragThermalDiffusion.cpp + PUBLIC ${CMAKE_CURRENT_LIST_DIR}/bragViscosity.hpp + PUBLIC ${CMAKE_CURRENT_LIST_DIR}/bragViscosity.cpp + ) diff --git a/src/fluid/braginskii/bragThermalDiffusion.cpp b/src/fluid/braginskii/bragThermalDiffusion.cpp new file mode 100644 index 00000000..82d5dac9 --- /dev/null +++ b/src/fluid/braginskii/bragThermalDiffusion.cpp @@ -0,0 +1,75 @@ +// *********************************************************************************** +// Idefix MHD astrophysical code +// Copyright(C) Geoffroy R. J. Lesur +// and other code contributors +// Licensed under CeCILL 2.1 License, see COPYING for more information +// *********************************************************************************** + +// This source code is largely inspired from the th_flux of Pluto4.2 +// ((c) P. Tzeferacos & A. Mignone) + +// Implementation of monotonicity-preserving heat flux following Sharma & Hammett 2007, +// Jour. of Comp. Physics + +#include + +#include "bragThermalDiffusion.hpp" +#include "dataBlock.hpp" +#include "fluid.hpp" +#include "eos.hpp" + + + +void BragThermalDiffusion::ShowConfig() { + if(status.status==Constant) { + idfx::cout << "Braginskii Thermal Diffusion: ENABLED with constant diffusivity kpar=" + << this->kpar << " and knor=" << this->knor << " ."<< std::endl; + } else if (status.status==UserDefFunction) { + idfx::cout << "Braginskii Thermal Diffusion: ENABLED with user-defined diffusivity function." + << std::endl; + if(!diffusivityFunc) { + IDEFIX_ERROR("No braginskii thermal diffusion function has been enrolled"); + } + } else { + IDEFIX_ERROR("Unknown braginskii thermal diffusion mode"); + } + if(status.isExplicit) { + idfx::cout << "Braginskii Thermal Diffusion: uses an explicit time integration." << std::endl; + } else if(status.isRKL) { + idfx::cout << "Braginskii Thermal Diffusion: uses a Runge-Kutta-Legendre time integration." + << std::endl; + } else { + IDEFIX_ERROR("Unknown time integrator for braginskii thermal diffusion."); + } + if(haveSlopeLimiter) { + idfx::cout << "Braginskii Thermal Diffusion: uses a slope limiter." << std::endl; + } +} + +void BragThermalDiffusion::EnrollBragThermalDiffusivity(BragDiffusivityFunc myFunc) { + if(this->status.status != UserDefFunction) { + IDEFIX_WARNING("Braginskii thermal diffusivity enrollment requires Hydro/BragThermalDiffusion " + "to be set to userdef in .ini file"); + } + this->diffusivityFunc = myFunc; +} + +void BragThermalDiffusion::AddBragDiffusiveFlux(int dir, const real t, + const IdefixArray4D &Flux) { + idfx::pushRegion("BragThermalDiffusion::AddBragDiffusiveFlux"); + switch(limiter) { + case PLMLimiter::VanLeer: + this->AddBragDiffusiveFluxLim(dir,t,Flux); + break; + case PLMLimiter::McLim: + this->AddBragDiffusiveFluxLim(dir,t,Flux); + break; + case PLMLimiter::MinMod: + this->AddBragDiffusiveFluxLim(dir,t,Flux); + break; + default: + IDEFIX_ERROR("The slope limiter for the Braginskii heat flux is not defined."); + break; + } + idfx::popRegion(); +} diff --git a/src/fluid/braginskii/bragThermalDiffusion.hpp b/src/fluid/braginskii/bragThermalDiffusion.hpp new file mode 100644 index 00000000..68a08278 --- /dev/null +++ b/src/fluid/braginskii/bragThermalDiffusion.hpp @@ -0,0 +1,669 @@ +// *********************************************************************************** +// Idefix MHD astrophysical code +// Copyright(C) Geoffroy R. J. Lesur +// and other code contributors +// Licensed under CeCILL 2.1 License, see COPYING for more information +// *********************************************************************************** + +#ifndef FLUID_BRAGINSKII_BRAGTHERMALDIFFUSION_HPP_ +#define FLUID_BRAGINSKII_BRAGTHERMALDIFFUSION_HPP_ + +#include + +#include "idefix.hpp" +#include "input.hpp" +#include "grid.hpp" +#include "fluid_defs.hpp" +#include "eos.hpp" +#include "slopeLimiter.hpp" + +// Forward class hydro declaration +template class Fluid; + +class DataBlock; + +class BragThermalDiffusion { + public: + template + BragThermalDiffusion(Input &, Grid &, Fluid *); // Initialisation + + void ShowConfig(); // display configuration + + void AddBragDiffusiveFlux(int, const real, const IdefixArray4D &); + + template + void AddBragDiffusiveFluxLim(int, const real, const IdefixArray4D &); + + // Enroll user-defined thermal conductivity + void EnrollBragThermalDiffusivity(BragDiffusivityFunc); + + IdefixArray3D heatSrc; // Source terms of the thermal operator + IdefixArray3D knorArr; + IdefixArray3D kparArr; + + // pre-computed geometrical factors in non-cartesian geometry + IdefixArray1D one_dmu; + + private: + DataBlock *data; + + // status of the module + ParabolicModuleStatus &status; + + BragDiffusivityFunc diffusivityFunc; + + bool haveSlopeLimiter{false}; + + // helper array + IdefixArray4D &Vc; + IdefixArray4D &Vs; + IdefixArray3D &dMax; + + // constant diffusion coefficient (when needed) + real knor, kpar; + + // equation of state (required to get the heat capacity) + EquationOfState *eos; + + PLMLimiter limiter{PLMLimiter::VanLeer}; +}; + +#include "fluid.hpp" +#include "dataBlock.hpp" + +template +BragThermalDiffusion::BragThermalDiffusion(Input &input, Grid &grid, Fluid *hydroin): + Vc{hydroin->Vc}, + Vs{hydroin->Vs}, + dMax{hydroin->dMax}, + eos{hydroin->eos.get()}, + data{hydroin->data}, + status{hydroin->bragThermalDiffusionStatus} { + idfx::pushRegion("BragThermalDiffusion::BragThermalDiffusion"); + + if(input.CheckEntry("Hydro","bragTDiffusion")>=0) { + if(input.Get("Hydro","bragTDiffusion",1).compare("mc") == 0) { + this->haveSlopeLimiter = true; + limiter = PLMLimiter::McLim; + } else if(input.Get("Hydro","bragTDiffusion",1).compare("vanleer") == 0) { + this->haveSlopeLimiter = true; + limiter = PLMLimiter::VanLeer; + } else if(input.Get("Hydro","bragTDiffusion",1).compare("minmod") == 0) { + IDEFIX_ERROR("The minmod slope limiter is not available because it has been " + "found to be too diffusive."); + } else if(input.Get("Hydro","bragTDiffusion",1).compare("nolimiter") == 0) { + this->haveSlopeLimiter = false; +// limiter = PLMLimiter::VanLeer; + } else { + IDEFIX_ERROR("Unknown braginskii thermal diffusion limiter in idefix.ini. " + "Can only be vanleer, mc or nolimiter."); + } + if(input.Get("Hydro","bragTDiffusion",2).compare("constant") == 0) { + this->kpar = input.Get("Hydro","bragTDiffusion",3); + this->knor = input.GetOrSet("Hydro","bragTDiffusion",4,0.); + this->status.status = Constant; + } else if(input.Get("Hydro","bragTDiffusion",2).compare("userdef") == 0) { + this->status.status = UserDefFunction; + this->kparArr = IdefixArray3D("BragThermalDiffusionKparArray",data->np_tot[KDIR], + data->np_tot[JDIR], + data->np_tot[IDIR]); + this->knorArr = IdefixArray3D("BragThermalDiffusionKnorArray",data->np_tot[KDIR], + data->np_tot[JDIR], + data->np_tot[IDIR]); + } else { + IDEFIX_ERROR("Unknown braginskii thermal diffusion definition in idefix.ini. " + "Can only be constant or userdef."); + } + } else { + IDEFIX_ERROR("I cannot create a BragThermalDiffusion object without bragTDiffusion defined" + "in the .ini file"); + } + + #ifndef MHD + IDEFIX_ERROR("Braginskii Thermal diffusion requires MHD"); + #endif + #ifdef ISOTHERMAL + IDEFIX_ERROR("Braginskii Thermal diffusion is not compatible" + "with the ISOTHERMAL approximation"); + #endif + + idfx::popRegion(); +} + +//We now define spatial derivative macros for the temperature field. +// The temperature is not a primitive variable +// and therefore not available as such in the DataBlock. +// It is rather defined as PRS/RHO. +// Special spatial derivative macros are therefore needed and defined here +// directly at the right cell interface according to the direciton of the flux. +#define D_DX_I_T(q) (q(PRS,k,j,i)/q(RHO,k,j,i) - q(PRS,k,j,i - 1)/q(RHO,k,j,i - 1)) +#define D_DY_J_T(q) (q(PRS,k,j,i)/q(RHO,k,j,i) - q(PRS,k,j - 1,i)/q(RHO,k,j - 1,i)) +#define D_DZ_K_T(q) (q(PRS,k,j,i)/q(RHO,k,j,i) - q(PRS,k - 1,j,i)/q(RHO,k - 1,j,i)) + +#define SL_DX_T(q,k,j,i) (q(PRS,k,j,i)/q(RHO,k,j,i) \ + - q(PRS,k,j,i - 1)/q(RHO,k,j,i - 1)) +#define SL_DY_T(q,k,j,i) (q(PRS,k,j,i)/q(RHO,k,j,i) \ + - q(PRS,k,j - 1,i)/q(RHO,k,j - 1,i)) +#define SL_DZ_T(q,k,j,i) (q(PRS,k,j,i)/q(RHO,k,j,i) \ + - q(PRS,k - 1,j,i)/q(RHO,k - 1,j,i)) + +#define D_DY_I_T(q) ( 0.25*(q(PRS,k,j + 1,i ) / q(RHO,k,j + 1,i) \ + + q(PRS,k,j + 1,i - 1) / q(RHO,k,j + 1,i - 1)) \ + - 0.25*(q(PRS,k,j - 1,i) / q(RHO,k,j - 1,i) \ + + q(PRS,k,j - 1,i - 1) / q(RHO,k,j - 1,i - 1))) + +#define D_DZ_I_T(q) ( 0.25*(q(PRS,k + 1,j,i) / q(RHO,k + 1,j,i) \ + + q(PRS,k + 1,j,i - 1) / q(RHO,k + 1,j,i - 1)) \ + - 0.25*(q(PRS,k - 1,j,i) / q(RHO,k - 1,j,i) \ + + q(PRS,k - 1,j,i - 1) / q(RHO,k - 1,j,i - 1))) + +#define D_DX_J_T(q) ( 0.25*(q(PRS,k,j,i + 1) / q(RHO,k,j,i + 1) \ + + q(PRS,k,j - 1,i + 1) / q(RHO,k,j - 1,i + 1)) \ + - 0.25*(q(PRS,k,j,i - 1) / q(RHO,k,j,i - 1) \ + + q(PRS,k,j - 1,i - 1) / q(RHO,k,j - 1,i - 1))) + +#define D_DZ_J_T(q) ( 0.25*(q(PRS,k + 1,j,i) / q(RHO,k + 1,j,i) \ + + q(PRS,k + 1,j - 1,i) / q(RHO,k + 1,j - 1,i)) \ + - 0.25*(q(PRS,k - 1,j,i) / q(RHO,k - 1,j,i) \ + + q(PRS,k - 1,j - 1,i) / q(RHO,k - 1,j - 1,i))) + +#define D_DX_K_T(q) ( 0.25*(q(PRS,k,j,i + 1) / q(RHO,k,j,i + 1) \ + + q(PRS,k - 1,j,i + 1) / q(RHO,k - 1,j,i + 1)) \ + - 0.25*(q(PRS,k,j,i - 1) / q(RHO,k,j,i - 1) \ + + q(PRS,k - 1,j,i - 1) / q(RHO,k - 1,j,i - 1))) + +#define D_DY_K_T(q) ( 0.25*(q(PRS,k,j + 1,i) / q(RHO,k,j + 1,i) \ + + q(PRS,k - 1,j + 1,i) / q(RHO,k - 1,j + 1,i)) \ + - 0.25*(q(PRS,k,j - 1,i) / q(RHO,k,j - 1,i) \ + + q(PRS,k - 1,j - 1,i) / q(RHO,k - 1,j - 1,i))) \ + +//We now define spatial average macros for the magnetic field. +// The magnetic field appears in the expression of the Braginskii heat flux. +// It is therefore needed at the right cell interface according to the direction of the flux. +#define BX_I Vs(BX1s,k,j,i) +#define BY_J Vs(BX2s,k,j,i) +#define BZ_K Vs(BX3s,k,j,i) +#define BY_I (0.25*(Vs(BX2s,k,j,i) + Vs(BX2s,k,j + 1,i) \ + + Vs(BX2s,k,j,i - 1) + Vs(BX2s,k,j + 1,i - 1))) +#define BZ_I (0.25*(Vs(BX3s,k,j,i) + Vs(BX3s,k + 1,j,i) \ + + Vs(BX3s,k,j,i - 1) + Vs(BX3s,k + 1,j,i - 1))) +#define BX_J (0.25*(Vs(BX1s,k,j,i) + Vs(BX1s,k,j,i + 1) \ + + Vs(BX1s,k,j - 1,i) + Vs(BX1s,k,j - 1,i + 1))) +#define BZ_J (0.25*(Vs(BX3s,k,j,i) + Vs(BX3s,k + 1,j,i) \ + + Vs(BX3s,k,j - 1,i) + Vs(BX3s,k + 1,j - 1,i))) +#define BX_K (0.25*(Vs(BX1s,k,j,i) + Vs(BX1s,k,j,i + 1) \ + + Vs(BX1s,k - 1,j,i) + Vs(BX1s,k - 1,j,i + 1))) +#define BY_K (0.25*(Vs(BX2s,k,j,i) + Vs(BX2s,k,j + 1,i) \ + + Vs(BX2s,k - 1,j,i) + Vs(BX2s,k - 1,j + 1,i))) + + +// (this avoids an extra array) +template +void BragThermalDiffusion::AddBragDiffusiveFluxLim(int dir, const real t, + const IdefixArray4D &Flux) { + idfx::pushRegion("BragThermalDiffusion::AddBragDiffusiveFluxLim"); + + IdefixArray4D Vc = this->Vc; + IdefixArray4D Vs = this->Vs; + IdefixArray3D dMax = this->dMax; + EquationOfState eos = *(this->eos); + + HydroModuleStatus haveThermalDiffusion = this->status.status; + bool haveSlopeLimiter = this->haveSlopeLimiter; + using SL = SlopeLimiter; + + int ibeg, iend, jbeg, jend, kbeg, kend; + ibeg = this->data->beg[IDIR]; + iend = this->data->end[IDIR]; + jbeg = this->data->beg[JDIR]; + jend = this->data->end[JDIR]; + kbeg = this->data->beg[KDIR]; + kend = this->data->end[KDIR]; + + // Determine the offset along which we do the extrapolation + if(dir==IDIR) iend++; + if(dir==JDIR) jend++; + if(dir==KDIR) kend++; + + IdefixArray1D dx = this->data->dx[dir]; + + #if GEOMETRY == SPHERICAL + IdefixArray1D rt = this->data->rt; + IdefixArray1D dmu = this->data->dmu; + #endif + + IdefixArray1D x1 = this->data->x[IDIR]; + IdefixArray1D x2 = this->data->x[JDIR]; + IdefixArray1D x3 = this->data->x[KDIR]; + IdefixArray1D x1l = this->data->xl[IDIR]; + IdefixArray1D x2l = this->data->xl[JDIR]; + IdefixArray1D x3l = this->data->xl[KDIR]; + IdefixArray1D dx1 = this->data->dx[IDIR]; + IdefixArray1D dx2 = this->data->dx[JDIR]; + IdefixArray1D dx3 = this->data->dx[KDIR]; + + #if GEOMETRY == SPHERICAL + IdefixArray1D sinx2 = this->data->sinx2; + IdefixArray1D tanx2 = this->data->tanx2; + IdefixArray1D sinx2m = this->data->sinx2m; + IdefixArray1D tanx2m = this->data->tanx2m; + #endif + real knorConstant = this->knor; + real kparConstant = this->kpar; + IdefixArray3D knorArr = this->knorArr; + IdefixArray3D kparArr = this->kparArr; + + if(haveThermalDiffusion == UserDefFunction && dir == IDIR) { + if(diffusivityFunc) { + idfx::pushRegion("UserDef::BragThermalDiffusivityFunction"); + diffusivityFunc(*this->data, t, kparArr, knorArr); + idfx::popRegion(); + } else { + IDEFIX_ERROR("No user-defined thermal diffusion function has been enrolled"); + } + } + + idefix_for("BragDiffusiveFlux",kbeg, kend, jbeg, jend, ibeg, iend, + KOKKOS_LAMBDA (int k, int j, int i) { + real knor, kpar; + real bgradT, Bmag, bn; + real q; + [[maybe_unused]] real Bi, Bj, Bk, Bn; + Bi = Bj = Bk = Bn = ZERO_F; + + [[maybe_unused]] real dTi, dTj, dTk, dTn; + dTi = dTj = dTk = dTn = ZERO_F; + + real locdmax = 0; + /////////////////////////////////////////// + // IDIR sweep // + /////////////////////////////////////////// + + if(dir == IDIR) { + if(haveThermalDiffusion == UserDefFunction) { + knor = HALF_F*(knorArr(k,j,i-1)+knorArr(k,j,i)); + if(haveSlopeLimiter) { + kpar = 2.*(kparArr(k,j,i-1)*kparArr(k,j,i))/(kparArr(k,j,i-1)+kparArr(k,j,i)); + } else { + kpar = HALF_F*(kparArr(k,j,i-1)+kparArr(k,j,i)); + } + } else { + knor = knorConstant; + kpar = kparConstant; + } + + EXPAND( Bi = BX_I; , + Bj = BY_I; , + Bk = BZ_I; ) + Bn = BX_I; + + #if GEOMETRY == CARTESIAN + dTi = D_DX_I_T(Vc)/dx1(i); + #if DIMENSIONS >= 2 + if (haveSlopeLimiter) { + dTj = SL::PLMLim(SL_DY_T(Vc,k,j,i-1)/dx2(j), + SL_DY_T(Vc,k,j+1,i-1)/dx2(j+1)); + dTj = SL::PLMLim(dTj, + SL::PLMLim(SL_DY_T(Vc,k,j,i)/dx2(j), + SL_DY_T(Vc,k,j+1,i)/dx2(j+1))); + } else { + dTj = D_DY_I_T(Vc)/dx2(j); + } + #if DIMENSIONS == 3 + if (haveSlopeLimiter) { + dTk = SL::PLMLim(SL_DZ_T(Vc,k,j,i-1)/dx3(k), + SL_DZ_T(Vc,k+1,j,i-1)/dx3(k+1)); + dTk = SL::PLMLim(dTk, + SL::PLMLim(SL_DZ_T(Vc,k,j,i)/dx3(k), + SL_DZ_T(Vc,k+1,j,i)/dx3(k+1))); + } else { + dTk = D_DZ_I_T(Vc)/dx3(k); + } + #endif + #endif + #elif GEOMETRY == CYLINDRICAL + dTi = D_DX_I_T(Vc)/dx1(i); + #if DIMENSIONS >= 2 + if (haveSlopeLimiter) { + dTj = SL::PLMLim(SL_DY_T(Vc,k,j,i-1)/dx2(j), + SL_DY_T(Vc,k,j+1,i-1)/dx2(j+1)); + dTj = SL::PLMLim(dTj, + SL::PLMLim(SL_DY_T(Vc,k,j,i)/dx2(j), + SL_DY_T(Vc,k,j+1,i)/dx2(j+1))); + } else { + dTj = D_DY_I_T(Vc)/dx2(j); + } + #endif + // No cylindrical geometry in 3D! + #elif GEOMETRY == POLAR + dTi = D_DX_I_T(Vc)/dx1(i); + #if DIMENSIONS >= 2 + if (haveSlopeLimiter) { + dTj = 1./x1(i-1)*SL::PLMLim(SL_DY_T(Vc,k,j,i-1)/dx2(j), + SL_DY_T(Vc,k,j+1,i-1)/dx2(j+1)); + dTj = SL::PLMLim(dTj, + 1./x1(i)*SL::PLMLim(SL_DY_T(Vc,k,j,i)/dx2(j), + SL_DY_T(Vc,k,j+1,i)/dx2(j+1))); + } else { + dTj = 1./x1l(i)*D_DY_I_T(Vc)/dx2(j); // 1/r dTj/dxj + } + #if DIMENSIONS == 3 + if (haveSlopeLimiter) { + dTk = SL::PLMLim(SL_DZ_T(Vc,k,j,i-1)/dx3(k), + SL_DZ_T(Vc,k+1,j,i-1)/dx3(k+1)); + dTk = SL::PLMLim(dTk, + SL::PLMLim(SL_DZ_T(Vc,k,j,i)/dx3(k), + SL_DZ_T(Vc,k+1,j,i)/dx3(k+1))); + } else { + dTk = D_DZ_I_T(Vc)/dx3(k); + } + #endif + #endif + #elif GEOMETRY == SPHERICAL + real s_1 = sinx2(j); + + // Trick to ensure that the axis does not lead to Nans + if(FABS(s_1) < SMALL_NUMBER) { + s_1 = ZERO_F; + } else { + s_1 = ONE_F/s_1; + } + + dTi = D_DX_I_T(Vc)/dx1(i); + #if DIMENSIONS >= 2 + if (haveSlopeLimiter) { + dTj = 1./x1(i-1)*SL::PLMLim(SL_DY_T(Vc,k,j,i-1)/dx2(j), + SL_DY_T(Vc,k,j+1,i-1)/dx2(j+1)); + dTj = SL::PLMLim(dTj, + 1./x1(i)*SL::PLMLim(SL_DY_T(Vc,k,j,i)/dx2(j), + SL_DY_T(Vc,k,j+1,i)/dx2(j+1))); + } else { + dTj = 1./x1l(i)*D_DY_I_T(Vc)/dx2(j); // 1/r dTj/dxj + } + #if DIMENSIONS == 3 + if (haveSlopeLimiter) { + dTk = s_1/x1(i-1)*SL::PLMLim(SL_DZ_T(Vc,k,j,i-1)/dx3(k), + SL_DZ_T(Vc,k+1,j,i-1)/dx3(k+1)); + dTk = SL::PLMLim(dTk, + s_1/x1(i)*SL::PLMLim(SL_DZ_T(Vc,k,j,i)/dx3(k), + SL_DZ_T(Vc,k+1,j,i)/dx3(k+1))); + } else { + dTk = s_1/x1l(i)*D_DZ_I_T(Vc)/dx3(k); + } + #endif + #endif + #endif // GEOMETRY + + real gamma_m1 = eos.GetGamma(Vc(PRS,k,j,i),Vc(RHO,k,j,i)) - ONE_F; + + locdmax = FMAX(kpar,knor)/(0.5*(Vc(RHO,k,j,i) + Vc(RHO,k,j,i - 1)))*gamma_m1; + + dTn = dTi; + } else if(dir == JDIR) { + ////////////// + // JDIR + ///////////// + + if(haveThermalDiffusion == UserDefFunction) { + knor = HALF_F*(knorArr(k,j-1,i)+knorArr(k,j,i)); + if(haveSlopeLimiter) { + kpar = 2.*(kparArr(k,j-1,i)*kparArr(k,j,i))/(kparArr(k,j-1,i)+kparArr(k,j,i)); + } else { + kpar = HALF_F*(kparArr(k,j-1,i)+kparArr(k,j,i)); + } + } else { + knor = knorConstant; + kpar = kparConstant; + } + + EXPAND( Bi = BX_J; , + Bj = BY_J; , + Bk = BZ_J; ) + Bn = BY_J; + + #if GEOMETRY == CARTESIAN + if (haveSlopeLimiter) { + dTi = SL::PLMLim(SL_DX_T(Vc,k,j-1,i)/dx1(i), + SL_DX_T(Vc,k,j-1,i+1)/dx1(i+1)); + dTi = SL::PLMLim(dTi, + SL::PLMLim(SL_DX_T(Vc,k,j,i)/dx1(i), + SL_DX_T(Vc,k,j,i+1)/dx1(i+1))); + } else { + dTi = D_DX_J_T(Vc)/dx1(i); + } + dTj = D_DY_J_T(Vc)/dx2(j); + #if DIMENSIONS == 3 + if (haveSlopeLimiter) { + dTk = SL::PLMLim(SL_DX_T(Vc,k,j-1,i)/dx3(k), + SL_DX_T(Vc,k+1,j-1,i)/dx3(k+1)); + dTk = SL::PLMLim(dTk, + SL::PLMLim(SL_DX_T(Vc,k,j,i)/dx3(k), + SL_DX_T(Vc,k+1,j,i)/dx3(k+1))); + } else { + dTk = D_DZ_J_T(Vc)/dx3(k); + } + #endif + #elif GEOMETRY == CYLINDRICAL + if (haveSlopeLimiter) { + dTi = SL::PLMLim(SL_DX_T(Vc,k,j-1,i)/dx1(i), + SL_DX_T(Vc,k,j-1,i+1)/dx1(i+1)); + dTi = SL::PLMLim(dTi, + SL::PLMLim(SL_DX_T(Vc,k,j,i)/dx1(i), + SL_DX_T(Vc,k,j,i+1)/dx1(i+1))); + } else { + dTi = D_DX_J_T(Vc)/dx1(i); + } + dTj = D_DY_J_T(Vc)/dx2(j); + // No cylindrical geometry in 3D! + #elif GEOMETRY == POLAR + //gradT = ... + 1/r dT/dtheta + ... + if (haveSlopeLimiter) { + dTi = SL::PLMLim(SL_DX_T(Vc,k,j-1,i)/dx1(i), + SL_DX_T(Vc,k,j-1,i+1)/dx1(i+1)); + dTi = SL::PLMLim(dTi, + SL::PLMLim(SL_DX_T(Vc,k,j,i)/dx1(i), + SL_DX_T(Vc,k,j,i+1)/dx1(i+1))); + } else { + dTi = D_DX_J_T(Vc)/dx1(i); + } + dTj = 1./x1(i)*D_DY_J_T(Vc)/dx2(j); + #if DIMENSIONS == 3 + if (haveSlopeLimiter) { + dTk = SL::PLMLim(SL_DX_T(Vc,k,j-1,i)/dx3(k), + SL_DX_T(Vc,k+1,j-1,i)/dx3(k+1)); + dTk = SL::PLMLim(dTk, + SL::PLMLim(SL_DX_T(Vc,k,j,i)/dx3(k), + SL_DX_T(Vc,k+1,j,i)/dx3(k+1))); + } else { + dTk = D_DZ_J_T(Vc)/dx3(k); + } + #endif + #elif GEOMETRY == SPHERICAL + if (haveSlopeLimiter) { + dTi = SL::PLMLim(SL_DX_T(Vc,k,j-1,i)/dx1(i), + SL_DX_T(Vc,k,j-1,i+1)/dx1(i+1)); + dTi = SL::PLMLim(dTi, + SL::PLMLim(SL_DX_T(Vc,k,j,i)/dx1(i), + SL_DX_T(Vc,k,j,i+1)/dx1(i+1))); + } else { + dTi = D_DX_J_T(Vc)/dx1(i); + } + dTj = 1./x1(i)*D_DY_J_T(Vc)/dx2(j); + #if DIMENSIONS == 3 + if (haveSlopeLimiter) { + real s_1 = sinx2(j-1); + + // Trick to ensure that the axis does not lead to Nans + if(FABS(s_1) < SMALL_NUMBER) { + s_1 = ZERO_F; + } else { + s_1 = ONE_F/s_1; + } + dTk = s_1/x1(i)*SL::PLMLim(SL_DX_T(Vc,k,j-1,i)/dx3(k), + SL_DX_T(Vc,k+1,j-1,i)/dx3(k+1)); + + s_1 = sinx2(j); + + // Trick to ensure that the axis does not lead to Nans + if(FABS(s_1) < SMALL_NUMBER) { + s_1 = ZERO_F; + } else { + s_1 = ONE_F/s_1; + } + dTk = s_1/x1(i)*SL::PLMLim(dTk, + SL::PLMLim(SL_DX_T(Vc,k,j,i)/dx3(k), + SL_DX_T(Vc,k+1,j,i)/dx3(k+1))); + } else { + real s_1 = sinx2m(j); + + // Trick to ensure that the axis does not lead to Nans + if(FABS(s_1) < SMALL_NUMBER) { + s_1 = ZERO_F; + } else { + s_1 = ONE_F/s_1; + } + + dTk = s_1/x1(i)*D_DZ_J_T(Vc)/dx3(k); + } + #endif + #endif // GEOMETRY + + real gamma_m1 = eos.GetGamma(Vc(PRS,k,j,i),Vc(RHO,k,j,i)) - ONE_F; + + locdmax = FMAX(kpar,knor)/(0.5*(Vc(RHO,k,j,i) + Vc(RHO,k,j - 1,i)))*gamma_m1; + + dTn = dTj; + } else if(dir == KDIR) { + ////////////// + // KDIR + ///////////// + if(haveThermalDiffusion == UserDefFunction) { + knor = HALF_F*(knorArr(k-1,j,i)+knorArr(k,j,i)); + if(haveSlopeLimiter) { + kpar = 2.*(kparArr(k-1,j,i)*kparArr(k,j,i))/(kparArr(k-1,j,i)+kparArr(k,j,i)); + } else { + kpar = HALF_F*(kparArr(k-1,j,i)+kparArr(k,j,i)); + } + } else { + knor = knorConstant; + kpar = kparConstant; + } + + Bi = BX_K; + Bj = BY_K; + Bk = BZ_K; + Bn = Bk; + + #if GEOMETRY == CARTESIAN + if (haveSlopeLimiter) { + dTi = SL::PLMLim(SL_DX_T(Vc,k-1,j,i)/dx1(i), + SL_DX_T(Vc,k-1,j,i+1)/dx1(i+1)); + dTi = SL::PLMLim(dTi, + SL::PLMLim(SL_DX_T(Vc,k,j,i)/dx1(i), + SL_DX_T(Vc,k,j,i+1)/dx1(i+1))); + dTj = SL::PLMLim(SL_DY_T(Vc,k-1,j,i)/dx2(j), + SL_DY_T(Vc,k-1,j+1,i)/dx2(j+1)); + dTj = SL::PLMLim(dTj, + SL::PLMLim(SL_DY_T(Vc,k,j,i)/dx2(j), + SL_DY_T(Vc,k,j+1,i)/dx2(j+1))); + } else { + dTi = D_DX_K_T(Vc)/dx1(i); + dTj = D_DY_K_T(Vc)/dx2(j); + } + dTk = D_DZ_K_T(Vc)/dx3(k); + // No cylindrical geometry in 3D! + #elif GEOMETRY == POLAR + if (haveSlopeLimiter) { + dTi = SL::PLMLim(SL_DX_T(Vc,k-1,j,i)/dx1(i), + SL_DX_T(Vc,k-1,j,i+1)/dx1(i+1)); + dTi = SL::PLMLim(dTi, + SL::PLMLim(SL_DX_T(Vc,k,j,i)/dx1(i), + SL_DX_T(Vc,k,j,i+1)/dx1(i+1))); + dTj = SL::PLMLim(SL_DY_T(Vc,k-1,j,i)/dx2(j), + SL_DY_T(Vc,k-1,j+1,i)/dx2(j+1)); + dTj = SL::PLMLim(dTj, + SL::PLMLim(SL_DY_T(Vc,k,j,i)/dx2(j), + SL_DY_T(Vc,k,j+1,i)/dx2(j+1))); + dTj *= 1./x1(i); + } else { + dTi = D_DX_K_T(Vc)/dx1(i); + dTj = 1./x1(i)*D_DY_K_T(Vc)/dx2(j); // 1/r dTj/dxj + } + dTk = D_DZ_K_T(Vc)/dx3(k); + #elif GEOMETRY == SPHERICAL + real s_1 = sinx2(j); + + // Trick to ensure that the axis does not lead to Nans + if(FABS(s_1) < SMALL_NUMBER) { + s_1 = ZERO_F; + } else { + s_1 = ONE_F/s_1; + } + + if (haveSlopeLimiter) { + dTi = SL::PLMLim(SL_DX_T(Vc,k-1,j,i)/dx1(i), + SL_DX_T(Vc,k-1,j,i+1)/dx1(i+1)); + dTi = SL::PLMLim(dTi, + SL::PLMLim(SL_DX_T(Vc,k,j,i)/dx1(i), + SL_DX_T(Vc,k,j,i+1)/dx1(i+1))); + dTj = SL::PLMLim(SL_DY_T(Vc,k-1,j,i)/dx2(j), + SL_DY_T(Vc,k-1,j+1,i)/dx2(j+1)); + dTj = SL::PLMLim(dTj, + SL::PLMLim(SL_DY_T(Vc,k,j,i)/dx2(j), + SL_DY_T(Vc,k,j+1,i)/dx2(j+1))); + dTj *= 1./x1(i); + } else { + dTi = D_DX_K_T(Vc)/dx1(i); + dTj = 1./x1(i)*D_DY_K_T(Vc)/dx2(j); // 1/r dTj/dxj + } + dTk = s_1/x1(i)*D_DZ_K_T(Vc)/dx3(k); + //gradT = ... + ... + 1/(r*sin(theta)) dTphi/dphi + #endif // GEOMETRY + + real gamma_m1 = eos.GetGamma(Vc(PRS,k,j,i),Vc(RHO,k,j,i)) - ONE_F; + + locdmax = FMAX(kpar,knor)/(0.5*(Vc(RHO,k,j,i) + Vc(RHO,k-1,j,i)))*gamma_m1; + + dTn = dTk; + } + // From here, gradients and normal have been computed, so we just need to get the fluxes + + bgradT = EXPAND( Bi*dTi , + Bj*dTj, +Bk*dTk); + Bmag = EXPAND( Bi*Bi , + Bj*Bj, + Bk*Bk); + Bmag = sqrt(Bmag); + Bmag = FMAX(1e-6*SMALL_NUMBER,Bmag); + + bgradT /= Bmag; + + bn = Bn/Bmag; /* -- unit vector component -- */ + q = kpar*bgradT*bn + knor*(dTn - bn*bgradT); + + Flux(ENG, k, j, i) -= q; + + dMax(k,j,i) = FMAX(dMax(k,j,i),locdmax); + }); + idfx::popRegion(); +} + +#undef D_DX_I_T +#undef D_DY_J_T +#undef D_DZ_K_T +#undef SL_DX_T +#undef SL_DY_T +#undef SL_DZ_T +#undef D_DY_I_T +#undef D_DZ_I_T +#undef D_DX_J_T +#undef D_DZ_J_T +#undef D_DX_K_T +#undef D_DY_K_T +#undef BX_I +#undef BX_J +#undef BX_K +#undef BY_I +#undef BY_J +#undef BY_K +#undef BZ_I +#undef BZ_J +#undef BZ_K +#endif // FLUID_BRAGINSKII_BRAGTHERMALDIFFUSION_HPP_ diff --git a/src/fluid/braginskii/bragViscosity.cpp b/src/fluid/braginskii/bragViscosity.cpp new file mode 100644 index 00000000..6d5007b5 --- /dev/null +++ b/src/fluid/braginskii/bragViscosity.cpp @@ -0,0 +1,100 @@ +// *********************************************************************************** +// Idefix MHD astrophysical code +// Copyright(C) Geoffroy R. J. Lesur +// and other code contributors +// Licensed under CeCILL 2.1 License, see COPYING for more information +// *********************************************************************************** + +// This source code is largely inspired from the viscous_flux of Pluto4.2 +// ((c) P. Tzeferacos & A. Mignone) + +// Implementation of monotonicity-preserving viscous flux following ZuHone et al., +// ApJ + +#include + +#include "idefix.hpp" +#include "bragViscosity.hpp" +#include "dataBlock.hpp" +#include "fluid.hpp" + + + +void BragViscosity::InitArrays() { + // Allocate and fill arrays when needed + #if GEOMETRY != CARTESIAN + one_dmu = IdefixArray1D("BragViscosity_1dmu", data->np_tot[JDIR]); + IdefixArray1D dmu = one_dmu; + IdefixArray1D th = data->x[JDIR]; + idefix_for("ViscousInitGeometry",1,data->np_tot[JDIR], + KOKKOS_LAMBDA(int j) { + real scrch = FABS((1.0-cos(th(j)))*(sin(th(j)) >= 0.0 ? 1.0:-1.0) + -(1.0-cos(th(j-1))) * (sin(th(j-1)) > 0.0 ? 1.0:-1.0)); + dmu(j) = 1.0/scrch; + }); + #endif + bragViscSrc = IdefixArray4D("BragViscosity_source", COMPONENTS, data->np_tot[KDIR], + data->np_tot[JDIR], + data->np_tot[IDIR]); +} +void BragViscosity::ShowConfig() { + if(status.status==Constant) { + idfx::cout << "Braginskii Viscosity: ENABLED with constant braginskii viscosity etaBrag=" + << this->etaBrag << " ."<< std::endl; + } else if (status.status==UserDefFunction) { + idfx::cout << "Braginskii Viscosity: ENABLED with user-defined braginskii viscosity function." + << std::endl; + if(!bragViscousDiffusivityFunc) { + IDEFIX_ERROR("No braginskii viscosity function has been enrolled"); + } + } else { + IDEFIX_ERROR("Unknown braginskii viscosity mode"); + } + if(this->status.isExplicit) { + idfx::cout << "Braginskii Viscosity: uses an explicit time integration." << std::endl; + } else if(this->status.isRKL) { + idfx::cout << "Braginskii Viscosity: uses a Runge-Kutta-Legendre time integration." + << std::endl; + } else { + IDEFIX_ERROR("Unknown time integrator for braginskii viscosity."); + } + if(haveSlopeLimiter) { + idfx::cout << "Braginskii Viscosity: uses a slope limiter." << std::endl; + } + + #if GEOMETRY == CYLINDRICAL || GEOMETRY == POLAR + IDEFIX_WARNING("Braginskii viscosity in cylindrical and polar coordinates " + "has been implemented but never tested so far."); + #endif +} + +void BragViscosity::EnrollBragViscousDiffusivity(DiffusivityFunc myFunc) { + if(this->status.status != UserDefFunction) { + IDEFIX_WARNING("Braginskii viscous diffusivity enrollment requires Hydro/BragViscosity " + "to be set to userdef in .ini file"); + } + this->bragViscousDiffusivityFunc = myFunc; +} + +// This function computes the viscous flux and stores it in hydro->fluxRiemann +// (this avoids an extra array) +// Associated source terms, present in non-cartesian geometry are also computed +// and stored in this->viscSrc for later use (in calcRhs). +void BragViscosity::AddBragViscousFlux(int dir, const real t, const IdefixArray4D &Flux) { + idfx::pushRegion("BragViscosity::AddBragViscousFlux"); + switch(limiter) { + case PLMLimiter::VanLeer: + this->AddBragViscousFluxLim(dir,t,Flux); + break; + case PLMLimiter::McLim: + this->AddBragViscousFluxLim(dir,t,Flux); + break; + case PLMLimiter::MinMod: + this->AddBragViscousFluxLim(dir,t,Flux); + break; + default: + IDEFIX_ERROR("The slope limiter for the Braginskii viscosity is not defined."); + break; + } + idfx::popRegion(); +} diff --git a/src/fluid/braginskii/bragViscosity.hpp b/src/fluid/braginskii/bragViscosity.hpp new file mode 100644 index 00000000..c4f24003 --- /dev/null +++ b/src/fluid/braginskii/bragViscosity.hpp @@ -0,0 +1,957 @@ +// *********************************************************************************** +// Idefix MHD astrophysical code +// Copyright(C) Geoffroy R. J. Lesur +// and other code contributors +// Licensed under CeCILL 2.1 License, see COPYING for more information +// *********************************************************************************** + +#ifndef FLUID_BRAGINSKII_BRAGVISCOSITY_HPP_ +#define FLUID_BRAGINSKII_BRAGVISCOSITY_HPP_ + +#include + +#include "idefix.hpp" +#include "input.hpp" +#include "grid.hpp" +#include "fluid_defs.hpp" +#include "slopeLimiter.hpp" + +// Forward class hydro declaration +template class Fluid; +class DataBlock; + +class BragViscosity { + public: + template + BragViscosity(Input &, Grid &, Fluid *); + void ShowConfig(); // print configuration + void AddBragViscousFlux(int, const real, const IdefixArray4D &); + + template + void AddBragViscousFluxLim(int, const real, const IdefixArray4D &); + + // Enroll user-defined viscous diffusivity + void EnrollBragViscousDiffusivity(DiffusivityFunc); + + // Function for internal use (but public to allow for Cuda lambda capture) + void InitArrays(); + + IdefixArray4D bragViscSrc; // Source terms of the viscous operator + IdefixArray3D etaBragArr; + + + // pre-computed geometrical factors in non-cartesian geometry + IdefixArray1D one_dmu; + + private: + DataBlock* data; + + // Viscosity status + ParabolicModuleStatus &status; + + DiffusivityFunc bragViscousDiffusivityFunc; + + bool haveSlopeLimiter{false}; + + IdefixArray4D &Vc; + IdefixArray4D &Vs; + IdefixArray3D &dMax; + + // constant diffusion coefficient (when needed) + real etaBrag; + + PLMLimiter limiter{PLMLimiter::VanLeer}; +}; + +#include "fluid.hpp" + +template +BragViscosity::BragViscosity(Input &input, Grid &grid, Fluid *hydroin): + Vc{hydroin->Vc}, + Vs{hydroin->Vs}, + dMax{hydroin->dMax}, + status{hydroin->bragViscosityStatus} { + idfx::pushRegion("BragViscosity::BragViscosity"); + // Save the parent hydro object + this->data = hydroin->data; + + if(input.CheckEntry("Hydro","bragViscosity")>=0) { + if(input.Get("Hydro","bragViscosity",1).compare("vanleer") == 0) { + this->haveSlopeLimiter = true; + limiter = PLMLimiter::VanLeer; + } else if(input.Get("Hydro","bragViscosity",1).compare("mc") == 0) { + this->haveSlopeLimiter = true; + limiter = PLMLimiter::McLim; + } else if (input.Get("Hydro","bragViscosity",1).compare("nolimiter") == 0) { + this->haveSlopeLimiter = false; +// limiter = PLMLimiter::VanLeer; + } else { + IDEFIX_ERROR("Unknown braginskii viscosity limiter in idefix.ini. " + "Can only be vanleer, mc or nolimiter."); + } + if(input.Get("Hydro","bragViscosity",2).compare("constant") == 0) { + this->etaBrag = input.Get("Hydro","bragViscosity",3); + this->status.status = Constant; + } else if(input.Get("Hydro","bragViscosity",2).compare("userdef") == 0) { + this->status.status = UserDefFunction; + this->etaBragArr = IdefixArray3D("BragViscosityEtaArray",data->np_tot[KDIR], + data->np_tot[JDIR], + data->np_tot[IDIR]); + } else { + IDEFIX_ERROR("Unknown braginskii viscosity definition in idefix.ini. " + "Can only be constant or userdef."); + } + } else { + IDEFIX_ERROR("I cannot create a BragViscosity object without viscosity defined" + "in the .ini file"); + } + + InitArrays(); + + idfx::popRegion(); +} + + +//We now define spatial derivative macros for the velocity field. +// They are directly computed at the right cell interface according to the direciton of the flux. +#define D_DX_I(q,n) (q(n,k,j,i) - q(n,k,j,i - 1)) +#define D_DY_J(q,n) (q(n,k,j,i) - q(n,k,j - 1,i)) +#define D_DZ_K(q,n) (q(n,k,j,i) - q(n,k - 1,j,i)) + +#define SL_DX(q,n,k,j,i) (q(n,k,j,i) - q(n,k,j,i - 1)) +#define SL_DY(q,n,k,j,i) (q(n,k,j,i) - q(n,k,j - 1,i)) +#define SL_DZ(q,n,k,j,i) (q(n,k,j,i) - q(n,k - 1,j,i)) + +#define D_DY_I(q,n) ( 0.25*(q(n,k,j + 1,i) + q(n,k,j + 1,i - 1)) \ + - 0.25*(q(n,k,j - 1,i) + q(n,k,j - 1,i - 1))) + +#define D_DZ_I(q,n) ( 0.25*(q(n,k + 1,j,i) + q(n,k + 1,j,i - 1)) \ + - 0.25*(q(n,k - 1,j,i) + q(n,k - 1,j,i - 1))) + +#define D_DX_J(q,n) ( 0.25*(q(n,k,j,i + 1) + q(n,k,j - 1,i + 1)) \ + - 0.25*(q(n,k,j,i - 1) + q(n,k,j - 1,i - 1))) + +#define D_DZ_J(q,n) ( 0.25*(q(n,k + 1,j,i) + q(n,k + 1,j - 1,i)) \ + - 0.25*(q(n,k - 1,j,i) + q(n,k - 1,j - 1,i))) + +#define D_DX_K(q,n) ( 0.25*(q(n,k,j,i + 1) + q(n,k - 1,j,i + 1)) \ + - 0.25*(q(n,k,j,i - 1) + q(n,k - 1,j,i - 1))) + +#define D_DY_K(q,n) ( 0.25*(q(n,k,j + 1,i) + q(n,k - 1,j + 1,i)) \ + - 0.25*(q(n,k,j - 1,i) + q(n,k - 1,j - 1,i))) + +//We now define spatial average macros for the magnetic field. +// The magnetic field appears in the expression of the Braginskii heat flux. +// It is therefore needed at the right cell interface according to the direction of the flux. +#define BX_I Vs(BX1s,k,j,i) +#define BY_J Vs(BX2s,k,j,i) +#define BZ_K Vs(BX3s,k,j,i) +#define BY_I (0.25*(Vs(BX2s,k,j,i) + Vs(BX2s,k,j + 1,i) \ + + Vs(BX2s,k,j,i - 1) + Vs(BX2s,k,j + 1,i - 1))) +#define BZ_I (0.25*(Vs(BX3s,k,j,i) + Vs(BX3s,k + 1,j,i) \ + + Vs(BX3s,k,j,i - 1) + Vs(BX3s,k + 1,j,i - 1))) +#define BX_J (0.25*(Vs(BX1s,k,j,i) + Vs(BX1s,k,j,i + 1) \ + + Vs(BX1s,k,j - 1,i) + Vs(BX1s,k,j - 1,i + 1))) +#define BZ_J (0.25*(Vs(BX3s,k,j,i) + Vs(BX3s,k + 1,j,i) \ + + Vs(BX3s,k,j - 1,i) + Vs(BX3s,k + 1,j - 1,i))) +#define BX_K (0.25*(Vs(BX1s,k,j,i) + Vs(BX1s,k,j,i + 1) \ + + Vs(BX1s,k - 1,j,i) + Vs(BX1s,k - 1,j,i + 1))) +#define BY_K (0.25*(Vs(BX2s,k,j,i) + Vs(BX2s,k,j + 1,i) \ + + Vs(BX2s,k - 1,j,i) + Vs(BX2s,k - 1,j + 1,i))) + + +// This function computes the Braginskii viscous flux and stores it in hydro->fluxRiemann +// (this avoids an extra array) +// Associated source terms, present in non-cartesian geometry are also computed +// and stored in this->bragViscSrc for later use (in calcRhs). +template +void BragViscosity::AddBragViscousFluxLim(int dir, const real t, const IdefixArray4D &Flux) { + idfx::pushRegion("BragViscosity::AddBragViscousFlux"); + IdefixArray4D Vc = this->Vc; + IdefixArray4D Vs = this->Vs; + IdefixArray4D bragViscSrc = this->bragViscSrc; + IdefixArray3D dMax = this->dMax; + IdefixArray3D etaBragArr = this->etaBragArr; + IdefixArray1D one_dmu = this->one_dmu; + IdefixArray1D x1 = this->data->x[IDIR]; + IdefixArray1D x2 = this->data->x[JDIR]; + IdefixArray1D x3 = this->data->x[KDIR]; + IdefixArray1D x1l = this->data->xl[IDIR]; + IdefixArray1D x2l = this->data->xl[JDIR]; + IdefixArray1D x3l = this->data->xl[KDIR]; + IdefixArray1D dx1 = this->data->dx[IDIR]; + IdefixArray1D dx2 = this->data->dx[JDIR]; + IdefixArray1D dx3 = this->data->dx[KDIR]; + + #if GEOMETRY == SPHERICAL + IdefixArray1D sinx2 = this->data->sinx2; + IdefixArray1D tanx2 = this->data->tanx2; + IdefixArray1D sinx2m = this->data->sinx2m; + IdefixArray1D tanx2m = this->data->tanx2m; + #endif + + HydroModuleStatus haveViscosity = this->status.status; + bool haveSlopeLimiter = this->haveSlopeLimiter; + + using SL = SlopeLimiter; + + // Braginskii Viscosity + // Compute viscosity if needed + if(haveViscosity == UserDefFunction && dir == IDIR) { + if(bragViscousDiffusivityFunc) { + bragViscousDiffusivityFunc(*this->data, t, etaBragArr); + } else { + IDEFIX_ERROR("No user-defined Braginskii viscosity function has been enrolled"); + } + } + + int ibeg, iend, jbeg, jend, kbeg, kend; + ibeg = this->data->beg[IDIR]; + iend = this->data->end[IDIR]; + jbeg = this->data->beg[JDIR]; + jend = this->data->end[JDIR]; + kbeg = this->data->beg[KDIR]; + kend = this->data->end[KDIR]; + + // Determine the offset along which we do the extrapolation + if(dir==IDIR) iend++; + if(dir==JDIR) jend++; + if(dir==KDIR) kend++; + + real etaBragConstant = this->etaBrag; + + idefix_for("ViscousFlux",kbeg, kend, jbeg, jend, ibeg, iend, + KOKKOS_LAMBDA (int k, int j, int i) { + real bi, bj, bk; + real Bmag; + real Pnor_par; + real bbgradV; + + [[maybe_unused]] real tau_xx, tau_xy, tau_xz; + [[maybe_unused]] real tau_yy, tau_yz; + [[maybe_unused]] real tau_zz; + + [[maybe_unused]] real dVxi, dVxj, dVxk; + [[maybe_unused]] real dVyi, dVyj, dVyk; + [[maybe_unused]] real dVzi, dVzj, dVzk; + + real divV; + + bi = bj = bk = ZERO_F; + + tau_xx = tau_xy = tau_xz = ZERO_F; + tau_yy = tau_yz = ZERO_F; + tau_zz = ZERO_F; + + dVxi = dVxj = dVxk = ZERO_F; + dVyi = dVyj = dVyk = ZERO_F; + dVzi = dVzj = dVzk = ZERO_F; + + real etaBrag; + [[maybe_unused]] real etaBragC; + + + //Cell-centered values to compute source terms + #if GEOMETRY != CARTESIAN + real biC, bjC, bkC; + real BmagC; + real Pnor_parC; + real bbgradVC; + + [[maybe_unused]] real tau_xxC, tau_xyC, tau_xzC; + [[maybe_unused]] real tau_yyC, tau_yzC; + [[maybe_unused]] real tau_zzC; + + [[maybe_unused]] real dVxiC, dVxjC, dVxkC; + [[maybe_unused]] real dVyiC, dVyjC, dVykC; + [[maybe_unused]] real dVziC, dVzjC, dVzkC; + + real divVC; + + biC = bjC = bkC = ZERO_F; + tau_xxC = tau_xyC = tau_xzC = ZERO_F; + tau_yyC = tau_yzC = ZERO_F; + tau_zzC = ZERO_F; + + dVxiC = dVxjC = dVxkC = ZERO_F; + dVyiC = dVyjC = dVykC = ZERO_F; + dVziC = dVzjC = dVzkC = ZERO_F; + + //Compute values at the center of the cells, no slope limiter are used at this stage + EXPAND( dVxiC = HALF_F*(Vc(VX1,k,j,i + 1) - Vc(VX1,k,j,i - 1))/dx1(i); , + dVyiC = HALF_F*(Vc(VX2,k,j,i + 1) - Vc(VX2,k,j,i - 1))/dx1(i); , + dVziC = HALF_F*(Vc(VX3,k,j,i + 1) - Vc(VX3,k,j,i - 1))/dx1(i); ) + #if DIMENSIONS >= 2 + EXPAND( dVxjC = HALF_F*(Vc(VX1,k,j + 1,i) - Vc(VX1,k,j - 1,i))/dx2(j); , + dVyjC = HALF_F*(Vc(VX2,k,j + 1,i) - Vc(VX2,k,j - 1,i))/dx2(j); , + dVzjC = HALF_F*(Vc(VX3,k,j + 1,i) - Vc(VX3,k,j - 1,i))/dx2(j); ) + #if DIMENSIONS == 3 + EXPAND ( dVxkC = HALF_F*(Vc(VX1,k + 1,j,i) - Vc(VX1,k - 1,j,i))/dx3(k); , + dVykC = HALF_F*(Vc(VX2,k + 1,j,i) - Vc(VX2,k - 1,j,i))/dx3(k); , + dVzkC = HALF_F*(Vc(VX3,k + 1,j,i) - Vc(VX3,k - 1,j,i))/dx3(k); ) + #endif + #endif + + EXPAND( biC = Vc(BX1,k,j,i); , + bjC = Vc(BX2,k,j,i); , + bkC = Vc(BX3,k,j,i); ) + + BmagC = EXPAND( Vc(BX1,k,j,i)*Vc(BX1,k,j,i) , + + Vc(BX2,k,j,i)*Vc(BX2,k,j,i) , + + Vc(BX3,k,j,i)*Vc(BX3,k,j,i) ); + if(BmagC< 0.001*SMALL_NUMBER) { + BmagC = sqrt(BmagC) + 0.000001*SMALL_NUMBER; + } else { + BmagC = sqrt(BmagC); + } + biC /= BmagC; + bjC /= BmagC; + bkC /= BmagC; + #endif //GEOMETRY != CARTESIAN + + #if GEOMETRY == CYLINDRICAL + bbgradVC = D_EXPAND( + biC*biC*dVxiC + biC*bjC*dVyiC + biC*bkC*(dVziC - Vc(VX3,k,j,i)/x1(i)) , + + bjC*biC*dVxjC + bjC*bjC*dVyjC + bjC*bkC*dVzjC + bkC*bkC*Vc(VX1,k,j,i)/x1(i) , + ); + + divVC = D_EXPAND(dVxiC + Vc(VX1,k,j,i)/x1(i) , + + dVyjC , + ); + // No cylindrical geometry in 3D! + #elif GEOMETRY == POLAR + bbgradVC = EXPAND( + biC*biC*dVxiC + bjC*biC*1./x1(i)*dVxjC + + bkC*biC*dVxkC, + + biC*bjC*(dVyiC - Vc(VX2,k,j,i)/x1(i)) + bjC*bjC*(1./x1(i)*dVyjC + Vc(VX1,k,j,i)/x1(i)) + + bkC*bjC*dVykC, + + biC*bkC*dVziC + bjC*bkC*1./x1(i)*dVzjC + + bkC*bkC*dVzkC + ); + + divVC = D_EXPAND(dVxiC + Vc(VX1,k,j,i)/x1(i) , + + 1./x1(i)*dVyjC , + + dVzkC); + #elif GEOMETRY == SPHERICAL + real tan_1 = tanx2(j); + real s_1 = sinx2(j); + + // Trick to ensure that the axis does not lead to Nans + if(FABS(tan_1) < SMALL_NUMBER) { + tan_1 = ZERO_F; + } else { + tan_1 = ONE_F/tan_1; + } + + if(FABS(s_1) < SMALL_NUMBER) { + s_1 = ZERO_F; + } else { + s_1 = ONE_F/s_1; + } + + bbgradVC = EXPAND( + biC*biC*dVxiC + bjC*biC*1./x1(i)*dVxjC + + bkC*biC*s_1/x1(i)*dVxkC, + + biC*bjC*(dVyiC - Vc(VX2,k,j,i)/x1(i)) + bjC*bjC*(1./x1(i)*dVyjC + Vc(VX1,k,j,i)/x1(i)) + + bkC*bjC*s_1/x1(i)*dVykC, + + biC*bkC*(dVziC - Vc(VX3,k,j,i)/x1(i)) + bjC*bkC*(1./x1(i)*dVzjC - tan_1/x1(i)*Vc(VX3,k,j,i)) + + bkC*bkC*(s_1/x1(i)*dVzkC + Vc(VX1,k,j,i)/x1(i) + tan_1/x1(i)*Vc(VX2,k,j,i))); + + divVC = D_EXPAND(2.*Vc(VX1,k,j,i)/x1(i) + dVxiC, + + dVyjC/x1(i) + tan_1*Vc(VX2,k,j,i)/x1(i), + + dVzkC/x1(i)*s_1 ); + #endif + + /////////////////////////////////////////// + // IDIR sweep // + /////////////////////////////////////////// + if(dir == IDIR) { + if(haveViscosity == UserDefFunction) { + etaBragC = etaBragArr(k,j,i); + if(haveSlopeLimiter) { + etaBrag = 2.*(etaBragArr(k,j,i-1)*etaBragArr(k,j,i)) / + (etaBragArr(k,j,i-1)+etaBragArr(k,j,i)); + } else { + etaBrag = HALF_F*(etaBragArr(k,j,i - 1)+etaBragArr(k,j,i)); + } + } else { + etaBragC = etaBrag = etaBragConstant; + } + + + EXPAND( dVxi = D_DX_I(Vc,VX1)/dx1(i); , + dVyi = D_DX_I(Vc,VX2)/dx1(i); , + dVzi = D_DX_I(Vc,VX3)/dx1(i); ) + + bi = BX_I; + Bmag = BX_I*BX_I; + #if DIMENSIONS >= 2 + bj = BY_I; + Bmag += BY_I*BY_I; + if (haveSlopeLimiter) { + dVxj = SL::PLMLim(SL_DY(Vc,VX1,k,j + 1,i)/dx2(j+1), + SL_DY(Vc,VX1,k,j + 1,i - 1)/dx2(j+1)); + dVxj = SL::PLMLim(dVxj, + SL::PLMLim(SL_DY(Vc,VX1,k,j,i)/dx2(j), + SL_DY(Vc,VX1,k,j,i - 1)/dx2(j))); + dVyj = SL::PLMLim(SL_DY(Vc,VX2,k,j + 1,i)/dx2(j+1), + SL_DY(Vc,VX2,k,j + 1,i - 1)/dx2(j+1)); + dVyj = SL::PLMLim(dVyj, + SL::PLMLim(SL_DY(Vc,VX2,k,j,i)/dx2(j), + SL_DY(Vc,VX2,k,j,i - 1)/dx2(j))); + #if DIMENSIONS == 3 + dVzj = SL::PLMLim(SL_DY(Vc,VX3,k,j + 1,i)/dx2(j+1), + SL_DY(Vc,VX3,k,j + 1,i - 1)/dx2(j+1)); + dVzj = SL::PLMLim(dVzj, + SL::PLMLim(SL_DY(Vc,VX3,k,j,i)/dx2(j), + SL_DY(Vc,VX3,k,j,i - 1)/dx2(j))); + #endif + } else { + EXPAND( dVxj = D_DY_I(Vc,VX1)/dx2(j); , + dVyj = D_DY_I(Vc,VX2)/dx2(j); , + dVzj = D_DY_I(Vc,VX3)/dx2(j); ) + } + #if DIMENSIONS == 3 + bk = BZ_I; + Bmag += BZ_I*BZ_I; + if (haveSlopeLimiter) { + dVxk = SL::PLMLim(SL_DZ(Vc,VX1,k + 1,j,i)/dx3(k+1), + SL_DZ(Vc,VX1,k + 1,j,i - 1)/dx3(k+1)); + dVxk = SL::PLMLim(dVxk, + SL::PLMLim(SL_DZ(Vc,VX1,k,j,i)/dx3(k), + SL_DZ(Vc,VX1,k,j,i - 1)/dx3(k))); + dVyk = SL::PLMLim(SL_DZ(Vc,VX2,k + 1,j,i)/dx3(k+1), + SL_DZ(Vc,VX2,k + 1,j,i - 1)/dx3(k+1)); + dVyk = SL::PLMLim(dVyk, + SL::PLMLim(SL_DZ(Vc,VX2,k,j,i)/dx3(k), + SL_DZ(Vc,VX2,k,j,i - 1)/dx3(k))); + dVzk = SL::PLMLim(SL_DZ(Vc,VX3,k + 1,j,i)/dx3(k+1), + SL_DZ(Vc,VX3,k + 1,j,i - 1)/dx3(k+1)); + dVzk = SL::PLMLim(dVzk, + SL::PLMLim(SL_DZ(Vc,VX3,k,j,i)/dx3(k), + SL_DZ(Vc,VX3,k,j,i - 1)/dx3(k))); + } else { + EXPAND ( dVxk = D_DZ_I(Vc,VX1)/dx3(k); , + dVyk = D_DZ_I(Vc,VX2)/dx3(k); , + dVzk = D_DZ_I(Vc,VX3)/dx3(k); ) + } + #endif + #endif + if(Bmag< 0.001*SMALL_NUMBER) { + Bmag = sqrt(Bmag) + 0.000001*SMALL_NUMBER; + } else { + Bmag = sqrt(Bmag); + } + bi /= Bmag; + bj /= Bmag; + bk /= Bmag; + + #if GEOMETRY == CARTESIAN + bbgradV = EXPAND( bi*bi*dVxi + bj*bi*dVxj + bk*bi*dVxk, + + bi*bj*dVyi + bj*bj*dVyj + bk*bj*dVyk, + + bi*bk*dVzi + bj*bk*dVzj + bk*bk*dVzk); + + divV = D_EXPAND(dVxi, + dVyj, + dVzk); + // No source term in cartesian geometry + #else + [[maybe_unused]] real vx1i, vx2i, vx3i; + vx1i = vx2i = vx3i = ZERO_F; + + EXPAND( vx1i = 0.5*(Vc(VX1,k,j,i-1)+Vc(VX1,k,j,i)); , + vx2i = 0.5*(Vc(VX2,k,j,i-1)+Vc(VX2,k,j,i)); , + vx3i = 0.5*(Vc(VX3,k,j,i-1)+Vc(VX3,k,j,i)); ) + + Pnor_parC = 3.0*etaBragC*(bbgradVC - divVC/3.0); + #endif // GEOMETRY != CARTESIAN + + #if GEOMETRY == CYLINDRICAL + bbgradV = D_EXPAND( + bi*bi*dVxi + bi*bj*dVyi + bi*bk*(dVzi - vx3i/x1l(i)), + + bj*bi*dVxj + bj*bj*dVyj + bj*bk*dVzj + bk*bk*vx1i/x1l(i), + ); + + divV = D_EXPAND(dVxi + vx1i/x1l(i) , + + dVyj , + ); + // No cylindrical geometry in 3D! + + //cell-centered values for source terms + tau_zzC = Pnor_parC*(bkC*bkC - 1./3.); + + EXPAND( bragViscSrc(IDIR,k,j,i) = -tau_zzC/x1(i); , + bragViscSrc(JDIR,k,j,i) = ZERO_F; , + bragViscSrc(KDIR,k,j,i) = ZERO_F; ) + + #elif GEOMETRY == POLAR + bbgradV = EXPAND( + bi*bi*dVxi + bj*bi*1./x1l(i)*dVxj + + bk*bi*dVxk, + + bi*bj*(dVyi - vx2i/x1l(i)) + bj*bj*(1./x1l(i)*dVyj + vx1i/x1l(i)) + + bk*bj*dVyk, + + bi*bk*dVzi + bj*bk*1./x1l(i)*dVzj + + bk*bk*dVzk + ); + + divV = D_EXPAND(dVxi + vx1i/x1l(i) , + + 1./x1l(i)*dVyj , + + dVzk); + + //cell-centered values for source terms + tau_yyC = Pnor_parC*(bjC*bjC - 1./3.); + + EXPAND( bragViscSrc(IDIR,k,j,i) = -tau_yyC/x1(i); , + bragViscSrc(JDIR,k,j,i) = ZERO_F; , + bragViscSrc(KDIR,k,j,i) = ZERO_F; ) + + #elif GEOMETRY == SPHERICAL + // NOLINT + bbgradV = EXPAND( + bi*bi*dVxi + bj*bi*1./x1l(i)*dVxj + bk*bi*s_1/x1l(i)*dVxk, + + bi*bj*(dVyi - vx2i/x1l(i)) + bj*bj*(1./x1l(i)*dVyj + vx1i/x1l(i)) + bk*bj*s_1/x1l(i)*dVyk, + + bi*bk*(dVzi - vx3i/x1l(i)) + bj*bk*(1./x1l(i)*dVzj - tan_1/x1l(i)*vx3i) + + bk*bk*(s_1/x1l(i)*dVzk + vx1i/x1l(i) + tan_1/x1l(i)*vx2i)); + + divV = D_EXPAND(2.0*vx1i/x1l(i) + dVxi, + + dVyj/x1l(i) + tan_1*vx2i/x1l(i), + + dVzk/x1l(i)*s_1 ); + + //cell-centered values for source terms + tau_yyC = Pnor_parC*(bjC*bjC - 1./3.); + tau_zzC = Pnor_parC*(bkC*bkC - 1./3.); + + EXPAND( bragViscSrc(IDIR,k,j,i) = -(tau_yyC + tau_zzC)/x1(i); , + bragViscSrc(JDIR,k,j,i) = ZERO_F; , + bragViscSrc(KDIR,k,j,i) = ZERO_F; ) + #endif + + Pnor_par = 3.0*etaBrag*(bbgradV - divV/3.0); + + tau_xx = Pnor_par*(bi*bi-1./3.); + tau_xy = Pnor_par*bi*bj; + tau_xz = Pnor_par*bi*bk; + + // Update flux with the stress tensor + EXPAND( Flux(MX1, k, j, i) -= tau_xx; , + Flux(MX2, k, j, i) -= tau_xy; , + Flux(MX3, k, j, i) -= tau_xz; ) + + #if HAVE_ENERGY + Flux(ENG, k, j, i) -= EXPAND( 0.5*(Vc(VX1,k,j,i) + Vc(VX1,k,j,i-1))*tau_xx , + + 0.5*(Vc(VX2,k,j,i) + Vc(VX2,k,j,i-1))*tau_xy , + + 0.5*(Vc(VX3,k,j,i) + Vc(VX3,k,j,i-1))*tau_xz); + #endif + + real locdmax = etaBrag/(0.5*(Vc(RHO,k,j,i)+Vc(RHO,k,j,i-1))); + dMax(k,j,i) = FMAX(dMax(k,j,i),locdmax); + } + + + /////////////////////////////////////////// + // JDIR sweep // + /////////////////////////////////////////// + if(dir == JDIR) { + if(haveViscosity == UserDefFunction) { + etaBragC = etaBragArr(k,j,i); + if(haveSlopeLimiter) { + etaBrag = 2.*(etaBragArr(k,j-1,i)*etaBragArr(k,j,i)) / + (etaBragArr(k,j-1,i)+etaBragArr(k,j,i)); + } else { + etaBrag = HALF_F*(etaBragArr(k,j - 1,i)+etaBragArr(k,j,i)); + } + } else { + etaBragC = etaBrag = etaBragConstant; + } + + if (haveSlopeLimiter) { + dVxi = SL::PLMLim(SL_DX(Vc,VX1,k,j,i + 1)/dx1(i+1), + SL_DX(Vc,VX1,k,j - 1,i + 1)/dx1(i+1)); + dVxi = SL::PLMLim(dVxi, + SL::PLMLim(SL_DX(Vc,VX1,k,j,i)/dx1(i), + SL_DX(Vc,VX1,k,j - 1,i)/dx1(i))); + #if DIMENSIONS >= 2 + dVyi = SL::PLMLim(SL_DX(Vc,VX2,k,j,i + 1)/dx1(i+1), + SL_DX(Vc,VX2,k,j - 1,i + 1)/dx1(i+1)); + dVyi = SL::PLMLim(dVyi, + SL::PLMLim(SL_DX(Vc,VX2,k,j,i)/dx1(i), + SL_DX(Vc,VX2,k,j - 1,i)/dx1(i))); + #if DIMENSIONS == 3 + dVzi = SL::PLMLim(SL_DX(Vc,VX3,k,j,i + 1)/dx1(i+1), + SL_DX(Vc,VX3,k,j - 1,i + 1)/dx1(i+1)); + dVzi = SL::PLMLim(dVzi, + SL::PLMLim(SL_DX(Vc,VX3,k,j,i)/dx1(i), + SL_DX(Vc,VX3,k,j - 1,i)/dx1(i))); + #endif + #endif + } else { + EXPAND( dVxi = D_DX_J(Vc,VX1)/dx1(i); , + dVyi = D_DX_J(Vc,VX2)/dx1(i); , + dVzi = D_DX_J(Vc,VX3)/dx1(i); ) + } + + bi = BX_J; + Bmag = BX_J*BX_J; + #if DIMENSIONS >= 2 + bj = BY_J; + Bmag += BY_J*BY_J; + EXPAND( dVxj = D_DY_J(Vc,VX1)/dx2(j); , + dVyj = D_DY_J(Vc,VX2)/dx2(j); , + dVzj = D_DY_J(Vc,VX3)/dx2(j); ) + #if DIMENSIONS == 3 + bk = BZ_J; + Bmag += BZ_J*BZ_J; + if (haveSlopeLimiter) { + dVxk = SL::PLMLim(SL_DZ(Vc,VX1,k + 1,j,i)/dx3(k+1), + SL_DZ(Vc,VX1,k + 1,j - 1,i)/dx3(k+1)); + dVxk = SL::PLMLim(dVxk, + SL::PLMLim(SL_DZ(Vc,VX1,k,j,i)/dx3(k), + SL_DZ(Vc,VX1,k,j - 1,i)/dx3(k))); + dVyk = SL::PLMLim(SL_DZ(Vc,VX2,k + 1,j,i)/dx3(k+1), + SL_DZ(Vc,VX2,k + 1,j - 1,i)/dx3(k+1)); + dVyk = SL::PLMLim(dVyk, + SL::PLMLim(SL_DZ(Vc,VX2,k,j,i)/dx3(k), + SL_DZ(Vc,VX2,k,j - 1,i)/dx3(k))); + dVzk = SL::PLMLim(SL_DZ(Vc,VX3,k + 1,j,i)/dx3(k+1), + SL_DZ(Vc,VX3,k + 1,j - 1,i)/dx3(k+1)); + dVzk = SL::PLMLim(dVzk, + SL::PLMLim(SL_DZ(Vc,VX3,k,j,i)/dx3(k), + SL_DZ(Vc,VX3,k,j - 1,i)/dx3(k))); + } else { + EXPAND ( dVxk = D_DZ_J(Vc,VX1)/dx3(k); , + dVyk = D_DZ_J(Vc,VX2)/dx3(k); , + dVzk = D_DZ_J(Vc,VX3)/dx3(k); ) + } + #endif + #endif + if(Bmag< 0.001*SMALL_NUMBER) { + Bmag = sqrt(Bmag) + 0.000001*SMALL_NUMBER; + } else { + Bmag = sqrt(Bmag); + } + bi /= Bmag; + bj /= Bmag; + bk /= Bmag; + + #if GEOMETRY == CARTESIAN + bbgradV = EXPAND( bi*bi*dVxi + bj*bi*dVxj + bk*bi*dVxk, + + bi*bj*dVyi + bj*bj*dVyj + bk*bj*dVyk, + + bi*bk*dVzi + bj*bk*dVzj + bk*bk*dVzk); + + divV = D_EXPAND(dVxi, + dVyj, + dVzk); + + Pnor_par = 3.0*etaBrag*(bbgradV - divV/3.0); + + tau_xy = Pnor_par*bi*bj; + tau_yy = Pnor_par*(bj*bj - 1./3.); + tau_yz = Pnor_par*bj*bk; + // No source term in cartesian geometry + #else + [[maybe_unused]] real vx1i, vx2i, vx3i; + vx1i = vx2i = vx3i = ZERO_F; + + EXPAND( vx1i = 0.5*(Vc(VX1,k,j-1,i)+Vc(VX1,k,j,i)); , + vx2i = 0.5*(Vc(VX2,k,j-1,i)+Vc(VX2,k,j,i)); , + vx3i = 0.5*(Vc(VX3,k,j-1,i)+Vc(VX3,k,j,i)); ) + + Pnor_parC = 3.0*etaBragC*(bbgradVC - divVC/3.0); + #endif // GEOMETRY != CARTESIAN + + #if GEOMETRY == CYLINDRICAL + bbgradV = D_EXPAND( + bi*bi*dVxi + bi*bj*dVyi + bi*bk*(dVzi - vx3i/x1(i)), + + bj*bi*dVxj + bj*bj*dVyj + bj*bk*dVzj + bk*bk*vx1i/x1(i), + ); + + divV = D_EXPAND(dVxi + vx1i/x1(i) , + + dVyj , + ); + // No cylindrical geometry in 3D! + // No source term along the z-axis in cylindrical/polar coordinates + EXPAND( bragViscSrc(IDIR,k,j,i) = ZERO_F; , + bragViscSrc(JDIR,k,j,i) = ZERO_F; , + bragViscSrc(KDIR,k,j,i) = ZERO_F; ) + #elif GEOMETRY == POLAR + bbgradV = EXPAND( + bi*bi*dVxi + bj*bi*1./x1(i)*dVxj + + bk*bi*dVxk, + + bi*bj*(dVyi - vx2i/x1(i)) + bj*bj*(1./x1(i)*dVyj + vx1i/x1(i)) + + bk*bj*dVyk, + + bi*bk*dVzi + bj*bk*1./x1(i)*dVzj + + bk*bk*dVzk + ); + + divV = D_EXPAND(dVxi + vx1i/x1(i) , + + 1./x1(i)*dVyj , + + dVzk); + + // See Geoffroy's trick for curvature source terms in curvilinear coordinates + EXPAND( bragViscSrc(IDIR,k,j,i) = ZERO_F; , + bragViscSrc(JDIR,k,j,i) = ZERO_F; , + bragViscSrc(KDIR,k,j,i) = ZERO_F; ) + + #elif GEOMETRY == SPHERICAL + tan_1 = tanx2m(j); + s_1 = sinx2m(j); + + // Trick to ensure that the axis does not lead to Nans + if(FABS(tan_1) < SMALL_NUMBER) { + tan_1 = ZERO_F; + } else { + tan_1 = ONE_F/tan_1; + } + + if(FABS(s_1) < SMALL_NUMBER) { + s_1 = ZERO_F; + } else { + s_1 = ONE_F/s_1; + } + + bbgradV = EXPAND( + bi*bi*dVxi + bj*bi*1./x1(i)*dVxj + bk*bi*s_1/x1(i)*dVxk, + + bi*bj*(dVyi - vx2i/x1(i)) + bj*bj*(1./x1(i)*dVyj + vx1i/x1(i)) + bk*bj*s_1/x1(i)*dVyk, + + bi*bk*(dVzi - vx3i/x1(i)) + bj*bk*(1./x1(i)*dVzj - tan_1/x1(i)*vx3i) + + bk*bk*(s_1/x1(i)*dVzk + vx1i/x1(i) + tan_1/x1(i)*vx2i)); + + divV = D_EXPAND( 2.0*vx1i/x1(i) + dVxi, + +(SIN(x2(j))*Vc(VX2,k,j,i) - FABS(SIN(x2(j-1)))*Vc(VX2,k,j-1,i))/x1(i) + *one_dmu(j) , + + dVzk/x1(i)*s_1 ); + + //Cell-centered values for the source terms + tan_1 = tanx2(j); + + // Trick to ensure that the axis does not lead to Nans + if(FABS(tan_1) < SMALL_NUMBER) { + tan_1 = ZERO_F; + } else { + tan_1 = ONE_F/tan_1; + } + + tau_xyC = Pnor_parC*biC*bjC; + tau_zzC = Pnor_parC*(bkC*bkC - 1./3.); + + EXPAND( bragViscSrc(IDIR,k,j,i) = ZERO_F; , + bragViscSrc(JDIR,k,j,i) = (tau_xyC - tau_zzC*tan_1)/x1(i); , + bragViscSrc(KDIR,k,j,i) = ZERO_F; ) + #endif + + Pnor_par = 3.0*etaBrag*(bbgradV - divV/3.0); + + tau_xy = Pnor_par*bi*bj; + tau_yy = Pnor_par*(bj*bj - 1./3.); + tau_yz = Pnor_par*bj*bk; + + // Update flux with the stress tensor + EXPAND( Flux(MX1, k, j, i) -= tau_xy; , + Flux(MX2, k, j, i) -= tau_yy; , + Flux(MX3, k, j, i) -= tau_yz; ) + + #if HAVE_ENERGY + Flux(ENG, k, j, i) -= EXPAND( 0.5*(Vc(VX1,k,j,i) + Vc(VX1,k,j-1,i))*tau_xy , + + 0.5*(Vc(VX2,k,j,i) + Vc(VX2,k,j-1,i))*tau_yy , + + 0.5*(Vc(VX3,k,j,i) + Vc(VX3,k,j-1,i))*tau_yz); + #endif + + real locdmax = etaBrag/(0.5*(Vc(RHO,k,j,i)+Vc(RHO,k,j-1,i))); + dMax(k,j,i) = FMAX(dMax(k,j,i),locdmax); + } + + + /////////////////////////////////////////// + // KDIR sweep // + /////////////////////////////////////////// + if(dir == KDIR) { + if(haveViscosity == UserDefFunction) { + etaBragC = etaBragArr(k,j,i); + if(haveSlopeLimiter) { + etaBrag = 2.*(etaBragArr(k-1,j,i)*etaBragArr(k,j,i)) / + (etaBragArr(k-1,j,i)+etaBragArr(k,j,i)); + } else { + etaBrag = HALF_F*(etaBragArr(k - 1,j,i)+etaBragArr(k,j,i)); + } + } else { + etaBragC = etaBrag = etaBragConstant; + } + + if (haveSlopeLimiter) { + dVxi = SL::PLMLim(SL_DX(Vc,VX1,k,j,i + 1)/dx1(i+1), + SL_DX(Vc,VX1,k - 1,j,i + 1)/dx1(i+1)); + dVxi = SL::PLMLim(dVxi, + SL::PLMLim(SL_DX(Vc,VX1,k,j,i)/dx1(i), + SL_DX(Vc,VX1,k - 1,j,i)/dx1(i))); + dVyi = SL::PLMLim(SL_DX(Vc,VX2,k,j,i + 1)/dx1(i+1), + SL_DX(Vc,VX2,k - 1,j,i + 1)/dx1(i+1)); + dVyi = SL::PLMLim(dVyi, + SL::PLMLim(SL_DX(Vc,VX2,k,j,i)/dx1(i), + SL_DX(Vc,VX2,k - 1,j,i)/dx1(i))); + dVzi = SL::PLMLim(SL_DX(Vc,VX3,k,j,i + 1)/dx1(i+1), + SL_DX(Vc,VX3,k - 1,j,i + 1)/dx1(i+1)); + dVzi = SL::PLMLim(dVzi, + SL::PLMLim(SL_DX(Vc,VX3,k,j,i)/dx1(i), + SL_DX(Vc,VX3,k - 1,j,i)/dx1(i))); + } else { + EXPAND( dVxi = D_DX_K(Vc,VX1)/dx1(i); , + dVyi = D_DX_K(Vc,VX2)/dx1(i); , + dVzi = D_DX_K(Vc,VX3)/dx1(i); ) + } + + bi = BX_K; + Bmag = BX_K*BX_K; + #if DIMENSIONS >= 2 + bj = BY_K; + Bmag += BY_K*BY_K; + if (haveSlopeLimiter) { + dVxj = SL::PLMLim(SL_DY(Vc,VX1,k,j + 1,i)/dx2(j+1), + SL_DY(Vc,VX1,k - 1,j + 1,i)/dx2(j+1)); + dVxj = SL::PLMLim(dVxj, + SL::PLMLim(SL_DY(Vc,VX1,k,j,i)/dx2(j), + SL_DY(Vc,VX1,k - 1,j,i)/dx2(j))); + dVyj = SL::PLMLim(SL_DY(Vc,VX2,k,j + 1,i)/dx2(j+1), + SL_DY(Vc,VX2,k - 1,j + 1,i)/dx2(j+1)); + dVyj = SL::PLMLim(dVyj, + SL::PLMLim(SL_DY(Vc,VX2,k,j,i)/dx2(j), + SL_DY(Vc,VX2,k - 1,j,i)/dx2(j))); + dVzj = SL::PLMLim(SL_DY(Vc,VX3,k,j + 1,i)/dx2(j+1), + SL_DY(Vc,VX3,k - 1,j + 1,i)/dx2(j+1)); + dVzj = SL::PLMLim(dVzj, + SL::PLMLim(SL_DY(Vc,VX3,k,j,i)/dx2(j), + SL_DY(Vc,VX3,k - 1,j,i)/dx2(j))); + } else { + EXPAND( dVxj = D_DY_K(Vc,VX1)/dx2(j); , + dVyj = D_DY_K(Vc,VX2)/dx2(j); , + dVzj = D_DY_K(Vc,VX3)/dx2(j); ) + } + + #if DIMENSIONS == 3 + bk = BZ_K; + Bmag += BZ_K*BZ_K; + EXPAND ( dVxk = D_DZ_K(Vc,VX1)/dx3(k); , + dVyk = D_DZ_K(Vc,VX2)/dx3(k); , + dVzk = D_DZ_K(Vc,VX3)/dx3(k); ) + #endif + #endif + if(Bmag< 0.001*SMALL_NUMBER) { + Bmag = sqrt(Bmag) + 0.000001*SMALL_NUMBER; + } else { + Bmag = sqrt(Bmag); + } + bi /= Bmag; + bj /= Bmag; + bk /= Bmag; + + #if GEOMETRY == CARTESIAN + bbgradV = EXPAND( bi*bi*dVxi + bj*bi*dVxj + bk*bi*dVxk, + + bi*bj*dVyi + bj*bj*dVyj + bk*bj*dVyk, + + bi*bk*dVzi + bj*bk*dVzj + bk*bk*dVzk); + + divV = D_EXPAND(dVxi, + dVyj, + dVzk); + + Pnor_par = 3.0*etaBrag*(bbgradV - divV/3.0); + + tau_xz = Pnor_par*bi*bk; + tau_yz = Pnor_par*bk*bj; + tau_zz = Pnor_par*(bk*bk - 1./3.); + + // No source term in cartesian geometry + #else + [[maybe_unused]] real vx1i, vx2i, vx3i; + vx1i = vx2i = vx3i = ZERO_F; + + EXPAND( vx1i = 0.5*(Vc(VX1,k-1,j,i)+Vc(VX1,k,j,i)); , + vx2i = 0.5*(Vc(VX2,k-1,j,i)+Vc(VX2,k,j,i)); , + vx3i = 0.5*(Vc(VX3,k-1,j,i)+Vc(VX3,k,j,i)); ) + #endif // GEOMETRY != CARTESIAN + + #if GEOMETRY == CYLINDRICAL + bbgradV = D_EXPAND( + bi*bi*dVxi + bi*bj*dVyi + bi*bk*(dVzi - vx3i/x1(i)), + + bj*bi*dVxj + bj*bj*dVyj + bj*bk*dVzj + bk*bk*vx1i/x1(i), + ); + + divV = D_EXPAND(dVxi + vx1i/x1(i) , + + dVyj , + ); + // No cylindrical geometry in 3D! + // See Geoffroy's trick for curvature source terms in cylindrical/polar coordinates + bragViscSrc(IDIR,k,j,i) = ZERO_F; + bragViscSrc(JDIR,k,j,i) = ZERO_F; + bragViscSrc(KDIR,k,j,i) = ZERO_F; + #elif GEOMETRY == POLAR + bbgradV = EXPAND( + bi*bi*dVxi + bj*bi*1./x1(i)*dVxj + + bk*bi*dVxk, + + bi*bj*(dVyi - vx2i/x1(i)) + bj*bj*(1./x1(i)*dVyj + vx1i/x1(i)) + + bk*bj*dVyk, + + bi*bk*dVzi + bj*bk*1./x1(i)*dVzj + + bk*bk*dVzk + ); + + divV = D_EXPAND(dVxi + vx1i/x1(i) , + + 1./x1(i)*dVyj , + + dVzk); + + // No source term along the z-axis in cylindrical/polar coordinates + bragViscSrc(IDIR,k,j,i) = ZERO_F; + bragViscSrc(JDIR,k,j,i) = ZERO_F; + bragViscSrc(KDIR,k,j,i) = ZERO_F; + + #elif GEOMETRY == SPHERICAL + bbgradV = EXPAND( + bi*bi*dVxi + bj*bi*1./x1(i)*dVxj + bk*bi*s_1/x1(i)*dVxk, + + bi*bj*(dVyi - vx2i/x1(i)) + bj*bj*(1./x1(i)*dVyj + vx1i/x1(i)) + bk*bj*s_1/x1(i)*dVyk, + + bi*bk*(dVzi - vx3i/x1(i)) + bj*bk*(1./x1(i)*dVzj - tan_1/x1(i)*vx3i) + + bk*bk*(s_1/x1(i)*dVzk + vx1i/x1(i) + tan_1/x1(i)*vx2i)); + + divV = 2.0*vx1i/x1(i) + dVxi + dVyj/x1(i) + tan_1*vx2i/x1(i) + dVzk/x1(i)*s_1; + + bragViscSrc(IDIR,k,j,i) = ZERO_F; + bragViscSrc(JDIR,k,j,i) = ZERO_F; + bragViscSrc(KDIR,k,j,i) = ZERO_F; + #endif + + Pnor_par = 3.0*etaBrag*(bbgradV - divV/3.0); + + tau_xz = Pnor_par*bi*bk; + tau_yz = Pnor_par*bk*bj; + tau_zz = Pnor_par*(bk*bk - 1./3.); + + // Update flux with the stress tensor + EXPAND( Flux(MX1, k, j, i) -= tau_xz; , + Flux(MX2, k, j, i) -= tau_yz; , + Flux(MX3, k, j, i) -= tau_zz; ) + + #if HAVE_ENERGY + Flux(ENG, k, j, i) -= EXPAND( 0.5*(Vc(VX1,k,j,i) + Vc(VX1,k-1,j,i))*tau_xz , + + 0.5*(Vc(VX2,k,j,i) + Vc(VX2,k-1,j,i))*tau_yz , + + 0.5*(Vc(VX3,k,j,i) + Vc(VX3,k-1,j,i))*tau_zz); + #endif + + real locdmax = etaBrag/(0.5*(Vc(RHO,k,j,i)+Vc(RHO,k-1,j,i))); + dMax(k,j,i) = FMAX(dMax(k,j,i),locdmax); + } + }); + idfx::popRegion(); +} + + +#undef D_DX_I +#undef D_DY_J +#undef D_DZ_K +#undef SL_DX +#undef SL_DY +#undef SL_DZ +#undef D_DY_I +#undef D_DZ_I +#undef D_DX_J +#undef D_DZ_J +#undef D_DX_K +#undef D_DY_K +#undef BX_I +#undef BX_J +#undef BX_K +#undef BY_I +#undef BY_J +#undef BY_K +#undef BZ_I +#undef BZ_J +#undef BZ_K +#endif // FLUID_BRAGINSKII_BRAGVISCOSITY_HPP_ diff --git a/src/hydro/calcCurrent.cpp b/src/fluid/calcCurrent.hpp similarity index 94% rename from src/hydro/calcCurrent.cpp rename to src/fluid/calcCurrent.hpp index 5116d0de..87a23e40 100644 --- a/src/hydro/calcCurrent.cpp +++ b/src/fluid/calcCurrent.hpp @@ -4,13 +4,16 @@ // and other code contributors // Licensed under CeCILL 2.1 License, see COPYING for more information // *********************************************************************************** +#ifndef FLUID_CALCCURRENT_HPP_ +#define FLUID_CALCCURRENT_HPP_ -#include "hydro.hpp" +#include "fluid.hpp" #include "dataBlock.hpp" // Compute the electrical current on faces -void Hydro::CalcCurrent() { - idfx::pushRegion("Hydro::CalcCurrent"); +template +void Fluid::CalcCurrent() { + idfx::pushRegion("Fluid::CalcCurrent"); IdefixArray4D Vc = this->Vc; IdefixArray4D Vs = this->Vs; IdefixArray4D J = this->J; @@ -155,8 +158,9 @@ void Hydro::CalcCurrent() { // Regularize the current along the axis in cases where an axis is present. // This feature is not yet ready, so it is commented out if(this->haveAxis) { - this->myAxis.RegularizeCurrent(); + boundary->axis->RegularizeCurrent(); } idfx::popRegion(); } +#endif //FLUID_CALCCURRENT_HPP_ diff --git a/src/hydro/calcParabolicFlux.hpp b/src/fluid/calcParabolicFlux.hpp similarity index 62% rename from src/hydro/calcParabolicFlux.hpp rename to src/fluid/calcParabolicFlux.hpp index 38489d85..a8c6761c 100644 --- a/src/hydro/calcParabolicFlux.hpp +++ b/src/fluid/calcParabolicFlux.hpp @@ -4,17 +4,19 @@ // and other code contributors // Licensed under CeCILL 2.1 License, see COPYING for more information // *********************************************************************************** -#ifndef HYDRO_CALCPARABOLICFLUX_HPP_ -#define HYDRO_CALCPARABOLICFLUX_HPP_ +#ifndef FLUID_CALCPARABOLICFLUX_HPP_ +#define FLUID_CALCPARABOLICFLUX_HPP_ -#include "hydro.hpp" +#include "fluid.hpp" #include "dataBlock.hpp" #include "addNonIdealMHDFlux.hpp" +#include "fargo.hpp" // Compute parabolic fluxes +template template -void Hydro::CalcParabolicFlux(const real t) { - idfx::pushRegion("Hydro::CalcParabolicFlux"); +void Fluid::CalcParabolicFlux(const real t) { + idfx::pushRegion("Fluid::CalcParabolicFlux"); IdefixArray3D dMax = this->dMax; @@ -38,23 +40,34 @@ void Hydro::CalcParabolicFlux(const real t) { || (viscosityStatus.isRKL && data->rklCycle)) { // Add fargo velocity if using fargo if(data->haveFargo && viscosityStatus.isExplicit) { - data->fargo.AddVelocity(t); + data->fargo->AddVelocityFluid(t,this); } - this->viscosity.AddViscousFlux(dir,t); + this->viscosity->AddViscousFlux(dir,t, this->FluxRiemann); // Remove back Fargo velocity if(data->haveFargo && viscosityStatus.isExplicit) { - data->fargo.SubstractVelocity(t); + data->fargo->SubstractVelocityFluid(t,this); } } // Add thermal diffusion if( (thermalDiffusionStatus.isExplicit && (!data->rklCycle)) || (thermalDiffusionStatus.isRKL && data->rklCycle)) { - this->thermalDiffusion.AddDiffusiveFlux(dir,t); + this->thermalDiffusion->AddDiffusiveFlux(dir,t, this->FluxRiemann); + } + + if( (bragViscosityStatus.isExplicit && (!data->rklCycle)) + || (bragViscosityStatus.isRKL && data->rklCycle)) { + this->bragViscosity->AddBragViscousFlux(dir,t, this->FluxRiemann); + } + + // Add braginskii thermal diffusion + if( (bragThermalDiffusionStatus.isExplicit && (!data->rklCycle)) + || (bragThermalDiffusionStatus.isRKL && data->rklCycle)) { + this->bragThermalDiffusion->AddBragDiffusiveFlux(dir,t, this->FluxRiemann); } idfx::popRegion(); } -#endif //HYDRO_CALCPARABOLICFLUX_HPP_ +#endif //FLUID_CALCPARABOLICFLUX_HPP_ diff --git a/src/fluid/calcRightHandSide.hpp b/src/fluid/calcRightHandSide.hpp new file mode 100644 index 00000000..0bfcca8b --- /dev/null +++ b/src/fluid/calcRightHandSide.hpp @@ -0,0 +1,589 @@ +// *********************************************************************************** +// Idefix MHD astrophysical code +// Copyright(C) Geoffroy R. J. Lesur +// and other code contributors +// Licensed under CeCILL 2.1 License, see COPYING for more information +// *********************************************************************************** + +#ifndef FLUID_CALCRIGHTHANDSIDE_HPP_ +#define FLUID_CALCRIGHTHANDSIDE_HPP_ + +#include "fluid.hpp" +#include "dataBlock.hpp" +#include "gravity.hpp" + +template +struct Fluid_CorrectFluxFunctor { + // Correct the flux to take into account non-cartesian geometries and Fargo + //***************************************************************** + // Functor constructor + //***************************************************************** + explicit Fluid_CorrectFluxFunctor (Fluid *hydro, real dt) { + Uc = hydro->Uc; + Vc = hydro->Vc; + Flux = hydro->FluxRiemann; + A = hydro->data->A[dir]; + dV = hydro->data->dV; + x1m = hydro->data->xl[IDIR]; + x1 = hydro->data->x[IDIR]; + sinx2m = hydro->data->sinx2m; + sinx2 = hydro->data->sinx2; + + this->dt = dt; + // Fargo + haveFargo = hydro->data->haveFargo; + if(haveFargo) { + fargoVelocity = hydro->data->fargo->meanVelocity; + fargoType = hydro->data->fargo->type; + } + + // Rotation + haveRotation = hydro->haveRotation; + // disable rotation in cartesian geometry, as Coriolis is then treated as a source term + #if GEOMETRY == CARTESIAN + haveRotation = false; + #endif + Omega = hydro->OmegaZ; + + // Shearing box shear rate + sbS = hydro->sbS; + } + + //***************************************************************** + // Functor Variables + //***************************************************************** + IdefixArray4D Uc; + IdefixArray4D Vc; + IdefixArray4D Flux; + IdefixArray3D A; + IdefixArray3D dV; + IdefixArray1D x1m; + IdefixArray1D x1; + IdefixArray1D sinx2m; + IdefixArray1D sinx2; + + // Fargo + IdefixArray2D fargoVelocity; + Fargo::FargoType fargoType; + bool haveFargo; + + + //Rotation + bool haveRotation; + real Omega; + + // shearingBox + real sbS; + + // timestep + real dt; + + //***************************************************************** + // Functor Operator + //***************************************************************** + KOKKOS_INLINE_FUNCTION void operator() (const int k, const int j, const int i) const { + // Add Fargo velocity to the fluxes + if(haveFargo || haveRotation) { + // Set mean advection direction + #if (GEOMETRY == CARTESIAN || GEOMETRY == POLAR) && DIMENSIONS >=2 + const int meanDir = JDIR; + #elif GEOMETRY == SPHERICAL && DIMENSIONS == 3 + const int meanDir = KDIR; + #else + const int meanDir = 0; + #endif + + // set mean advection velocity + real meanV = ZERO_F; + if constexpr(dir == IDIR) { + #if (GEOMETRY == CARTESIAN || GEOMETRY == POLAR) && DIMENSIONS >=2 + if(haveFargo) { + if(fargoType==Fargo::userdef) { + meanV = HALF_F*(fargoVelocity(k,i-1)+fargoVelocity(k,i)); + } else if(fargoType==Fargo::shearingbox) { + meanV = sbS * x1m(i); + } + } + #if GEOMETRY != CARTESIAN + if(haveRotation) { + meanV += x1m(i)*Omega; + } + #endif + #elif GEOMETRY == SPHERICAL && DIMENSIONS == 3 + if(haveFargo) { + meanV = HALF_F*(fargoVelocity(j,i-1)+fargoVelocity(j,i)); + } + if(haveRotation) { + meanV += x1m(i)*sinx2(j)*Omega; + } + #endif// GEOMETRY + } + #if GEOMETRY == SPHERICAL && DIMENSIONS == 3 + if constexpr (dir == JDIR) { + if(haveFargo) { + meanV = HALF_F*(fargoVelocity(j-1,i)+fargoVelocity(j,i)); + } + if(haveRotation) { + meanV += x1(i)*sinx2m(j)*Omega; + } + } + #elif (GEOMETRY == CARTESIAN || GEOMETRY == POLAR) && DIMENSIONS >=2 + if constexpr (dir == KDIR) { + if(haveFargo) { + if(fargoType==Fargo::userdef) { + meanV = HALF_F*(fargoVelocity(k-1,i)+fargoVelocity(k,i)); + } else if (fargoType==Fargo::shearingbox) { + meanV = sbS*x1(i); + } + } + } + #endif // GEOMETRY + + // Should do nothing along meanV direction, automatically satisfied + // since in that case meanV=0 + if constexpr(Phys::pressure) { + // Mignone (2012): second and third term of rhs of (25) + Flux(ENG,k,j,i) += meanV * (HALF_F*meanV*Flux(RHO,k,j,i) + Flux(MX1+meanDir,k,j,i)); + } + // Mignone+2012: second term of rhs of (24) + Flux(MX1+meanDir,k,j,i) += meanV * Flux(RHO,k,j,i); + } // Fargo & Rotation corrections + + real Ax = A(k,j,i); + + for(int nv = 0 ; nv < Phys::nvar ; nv++) { + Flux(nv,k,j,i) = Flux(nv,k,j,i) * Ax; + } + + // Curvature terms +#if (GEOMETRY == POLAR && COMPONENTS >= 2) \ + || (GEOMETRY == CYLINDRICAL && COMPONENTS == 3) + if constexpr (dir==IDIR) { + // Conserve angular momentum, hence flux is R*Bphi + Flux(iMPHI,k,j,i) = Flux(iMPHI,k,j,i) * FABS(x1m(i)); + if constexpr(Phys::mhd) { + if(Ax +struct Fluid_CalcRHSFunctor { + //***************************************************************** + // Functor constructor + //***************************************************************** + explicit Fluid_CalcRHSFunctor (Fluid *hydro, real dt) { + Uc = hydro->Uc; + Vc = hydro->Vc; + Flux = hydro->FluxRiemann; + A = hydro->data->A[dir]; + dV = hydro->data->dV; + x1m = hydro->data->xl[IDIR]; + x1 = hydro->data->x[IDIR]; + + rt = hydro->data->rt; + dmu = hydro->data->dmu; + sinx2m = hydro->data->sinx2m; + sinx2 = hydro->data->sinx2; + dx = hydro->data->dx[dir]; + dx2 = hydro->data->dx[JDIR]; + invDt = hydro->InvDt; + cMax = hydro->cMax; + dMax = hydro->dMax; + this->dt = dt; + + // Grid coarsening + if(hydro->data->haveGridCoarsening) { + haveGridCoarsening = hydro->data->coarseningDirection[dir]; + coarseningLevel = hydro->data->coarseningLevel[dir]; + } + + if(hydro->data->haveGravity) { + // Gravitational potential + phiP = hydro->data->gravity->phiP; + needPotential = hydro->data->gravity->havePotential; + + // BodyForce + bodyForce = hydro->data->gravity->bodyForceVector; + needBodyForce = hydro->data->gravity->haveBodyForce; + } + + // parabolic terms + haveParabolicTerms = hydro->haveExplicitParabolicTerms; + + // Viscosity (source term only when non-cartesian geometry) + haveViscosity = hydro->viscosityStatus.isExplicit; + if(haveViscosity) viscSrc = hydro->viscosity->viscSrc; + + // BragViscosity (source term only when non-cartesian geometry) + haveBragViscosity = hydro->bragViscosityStatus.isExplicit; + if(haveBragViscosity) bragViscSrc = hydro->bragViscosity->bragViscSrc; + + // Fargo + haveFargo = hydro->data->haveFargo; + if(haveFargo) { + fargoVelocity = hydro->data->fargo->meanVelocity; + fargoType = hydro->data->fargo->type; + } + + // Rotation + haveRotation = hydro->haveRotation; + // disable rotation in cartesian geometry, as Coriolis is then treated as a source term + #if GEOMETRY == CARTESIAN + haveRotation = false; + #endif + Omega = hydro->OmegaZ; + + // Shearing box shear rate + sbS = hydro->sbS; + } + //***************************************************************** + // Functor Variables + //***************************************************************** + IdefixArray4D Uc; + IdefixArray4D Vc; + IdefixArray4D Flux; + IdefixArray3D A; + IdefixArray3D dV; + IdefixArray1D x1m; + IdefixArray1D x1; + + IdefixArray1D rt; + IdefixArray1D dmu; + IdefixArray1D sinx2m; + IdefixArray1D sinx2; + IdefixArray1D dx; + IdefixArray1D dx2; + IdefixArray3D invDt; + IdefixArray3D cMax; + IdefixArray3D dMax; + IdefixArray4D viscSrc; + + // Grid coarsening + bool haveGridCoarsening{false}; + IdefixArray2D coarseningLevel; + + // Gravitational potential + IdefixArray3D phiP; + bool needPotential{false}; + + // BodyForce + IdefixArray4D bodyForce; + bool needBodyForce{false}; + + // parabolic terms + bool haveParabolicTerms{false}; + + // Viscosity (source term only when non-cartesian geometry) + bool haveViscosity{false}; + + // BragViscosity (source term only when non-cartesian geometry) + IdefixArray4D bragViscSrc; + bool haveBragViscosity{false}; + + // Fargo + IdefixArray2D fargoVelocity; + Fargo::FargoType fargoType; + bool haveFargo; + + + //Rotation + bool haveRotation; + real Omega; + + // shearingBox + real sbS; + + // timestep + real dt; + + //***************************************************************** + // Functor Operator + //***************************************************************** + KOKKOS_INLINE_FUNCTION void operator() (const int k, const int j, const int i) const { + const int ioffset = (dir==IDIR) ? 1 : 0; + const int joffset = (dir==JDIR) ? 1 : 0; + const int koffset = (dir==KDIR) ? 1 : 0; + + real dtdV=dt / dV(k,j,i); + real rhs[Phys::nvar]; + + #pragma unroll + for(int nv = 0 ; nv < Phys::nvar ; nv++) { + rhs[nv] = - dtdV*(Flux(nv, k+koffset, j+joffset, i+ioffset) - Flux(nv, k, j, i)); + } + + #if GEOMETRY != CARTESIAN + if constexpr(dir==IDIR) { + #ifdef iMPHI + rhs[iMPHI] = rhs[iMPHI] / x1(i); + #endif + if constexpr(Phys::mhd) { + #if (GEOMETRY == POLAR || GEOMETRY == CYLINDRICAL) && (defined iBPHI) + rhs[iBPHI] = - dt / dx(i) * (Flux(iBPHI, k, j, i+1) - Flux(iBPHI, k, j, i) ); + + #elif (GEOMETRY == SPHERICAL) + real q = dt / (x1(i)*dx(i)); + EXPAND( , + rhs[iBTH] = -q * ((Flux(iBTH, k, j, i+1) - Flux(iBTH, k, j, i) )); , + rhs[iBPHI] = -q * ((Flux(iBPHI, k, j, i+1) - Flux(iBPHI, k, j, i) )); ) + #endif + } // MHD + } else if constexpr(dir==JDIR) { + #if (GEOMETRY == SPHERICAL) && (COMPONENTS == 3) + rhs[iMPHI] /= FABS(sinx2(j)); + if constexpr(Phys::mhd) { + rhs[iBPHI] = -dt / (rt(i)*dx(j)) * (Flux(iBPHI, k, j+1, i) - Flux(iBPHI, k, j, i)); + } // MHD + #endif // GEOMETRY + } + // Nothing for KDIR + #endif // GEOMETRY != CARTESIAN + + + // Fargo terms to enfore conservation (actually substract back what was added in + // Totalflux loop) + if(haveFargo || haveRotation) { + // fetch fargo velocity when required + real meanV = ZERO_F; + #if (GEOMETRY == POLAR || GEOMETRY == CARTESIAN) && DIMENSIONS >=2 + if((dir==IDIR || dir == KDIR) && haveFargo) { + if(fargoType==Fargo::userdef) { + meanV = fargoVelocity(k,i); + } else if(fargoType==Fargo::shearingbox) { + meanV = sbS * x1(i); + } + } + #if GEOMETRY != CARTESIAN + if((dir==IDIR) && haveRotation) { + meanV += Omega*x1(i); + } + #endif + const int meanDir = JDIR; + #elif GEOMETRY == SPHERICAL && DIMENSIONS ==3 + if((dir==IDIR || dir == JDIR) && haveFargo) meanV = fargoVelocity(j,i); + if((dir==IDIR || dir == JDIR) && haveRotation) { + meanV += Omega*x1(i)*sinx2(j); + } + const int meanDir = KDIR; + #else + const int meanDir = 0; + #endif + // NB: MX1+meanDir = iMPHI + // Mignone+2012, rhs of eq. 24, last term + rhs[MX1+meanDir] -= meanV*rhs[RHO]; + if constexpr(Phys::pressure) { + // Mignone+2012, rhs of eq.25, 4th and 5th term. NB: rhs[MX1+meanDir] is alreay the + // divergence of the *total* Momentum Flux F_my+w F_rho + rhs[ENG] -= meanV * ( HALF_F*meanV*rhs[RHO] + rhs[MX1+meanDir]); + } + } + + #if GEOMETRY != CARTESIAN + // Viscosity source terms + if(haveViscosity) { + #pragma unroll + for(int nv = 0 ; nv < COMPONENTS ; nv++) { + rhs[nv + VX1] += dt*viscSrc(nv,k,j,i); + } + } else if(haveBragViscosity) { + #pragma unroll + for(int nv = 0 ; nv < COMPONENTS ; nv++) { + rhs[nv + VX1] += dt*bragViscSrc(nv,k,j,i); + } + } + #endif // GEOMETRY != CARTESIAN + + // elmentary length for gradient computations + const int ig = ioffset*i + joffset*j + koffset*k; + real dl = dx(ig); + #if GEOMETRY == POLAR + if constexpr (dir==JDIR) + dl = dl*x1(i); + + #elif GEOMETRY == SPHERICAL + if constexpr(dir==JDIR) + dl = dl*rt(i); + else if constexpr(dir==KDIR) + dl = dl*rt(i)*dmu(j)/dx2(j); + #endif + + // Potential terms + if(needPotential) { + real dphi; + if constexpr (dir==IDIR) { + // Gravitational force in direction i + dphi = - 1.0/12.0 * ( + - phiP(k,j,i+2) + 8.0 * phiP(k,j,i+1) + - 8.0*phiP(k,j,i-1) + phiP(k,j,i-2)); + } + if constexpr (dir==JDIR) { + // Gravitational force in direction j + dphi = - 1.0/12.0 * ( + - phiP(k,j+2,i) + 8.0 * phiP(k,j+1,i) + - 8.0*phiP(k,j-1,i) + phiP(k,j-2,i)); + } + if constexpr (dir==KDIR) { + // Gravitational force in direction k + dphi = - 1.0/12.0 * ( + - phiP(k+2,j,i) + 8.0 * phiP(k+1,j,i) + - 8.0*phiP(k-1,j,i) + phiP(k-2,j,i)); + } + rhs[MX1+dir] += dt * Vc(RHO,k,j,i) * dphi /dl; + + if constexpr(Phys::pressure) { + // Add gravitational force work as a source term + // This is equivalent to rho * v . nabla(phi) + // (note that Flux has already been multiplied by A) + rhs[ENG] += HALF_F * dtdV * + (Flux(RHO,k,j,i) + Flux(RHO, k+koffset, j+joffset, i+ioffset)) * dphi; + } + } + + // Body force + if(needBodyForce) { + rhs[MX1+dir] += dt * Vc(RHO,k,j,i) * bodyForce(dir,k,j,i); + if constexpr(Phys::pressure) { + // rho * v . f, where rhov is taken as a volume average of Flux(RHO) + rhs[ENG] += HALF_F * dtdV * dl * + (Flux(RHO,k,j,i) + Flux(RHO, k+koffset, j+joffset, i+ioffset)) * + bodyForce(dir,k,j,i); + } // Pressure + + // Particular cases if we do not sweep all of the components + #if DIMENSIONS == 1 && COMPONENTS > 1 + EXPAND( , + rhs[MX2] += dt * Vc(RHO,k,j,i) * bodyForce(JDIR,k,j,i); , + rhs[MX3] += dt * Vc(RHO,k,j,i) * bodyForce(KDIR,k,j,i); ) + if constexpr(Phys::pressure) { + rhs[ENG] += dt * (EXPAND( ZERO_F , + + Vc(RHO,k,j,i) * Vc(VX2,k,j,i) * bodyForce(JDIR,k,j,i) , + + Vc(RHO,k,j,i) * Vc(VX3,k,j,i) * bodyForce(KDIR,k,j,i) )); + } + #endif + #if DIMENSIONS == 2 && COMPONENTS == 3 + // Only add this term once! + if constexpr (dir==JDIR) { + rhs[MX3] += dt * Vc(RHO,k,j,i) * bodyForce(KDIR,k,j,i); + if constexpr(Phys::pressure) { + rhs[ENG] += dt * Vc(RHO,k,j,i) * Vc(VX3,k,j,i) * bodyForce(KDIR,k,j,i); + } + } + #endif + } + + + // Timestep computation + // Change elementary grid spacing according to local coarsening level. + if(haveGridCoarsening) { + int factor; + //factor = 2^(coarsening-1) + if constexpr (dir==IDIR) { + factor = 1 << (coarseningLevel(k,j) - 1); + } + if constexpr (dir==JDIR) { + factor = 1 << (coarseningLevel(k,i) - 1); + } + if constexpr (dir==KDIR) { + factor = 1 << (coarseningLevel(j,i) - 1); + } + dl = dl * factor; + } + + // Compute dt from max signal speed + invDt(k,j,i) = invDt(k,j,i) + HALF_F*(cMax(k+koffset,j+joffset,i+ioffset) + + cMax(k,j,i)) / (dl); + + if(haveParabolicTerms) { + invDt(k,j,i) = invDt(k,j,i) + TWO_F* FMAX(dMax(k+koffset,j+joffset,i+ioffset), + dMax(k,j,i)) / (dl*dl); + } + + + // Evolve the field components + #pragma unroll + for(int nv = 0 ; nv < Phys::nvar ; nv++) { + // Do not evolve the field components if they are computed by CT (i.e. if they are in Vs) + D_EXPAND( if(nv == BX1) { continue; } , + if(nv == BX2) { continue; } , + if(nv == BX3) { continue; } ) + + + Uc(nv,k,j,i) = Uc(nv,k,j,i) + rhs[nv]; + } + } +}; + + + +// Compute the right handside in direction dir from conservative equation, with timestep dt +template +template +void Fluid::CalcRightHandSide(real t, real dt) { + idfx::pushRegion("Fluid::CalcRightHandSide"); + + // Update fargo velocity when needed + if(data->haveFargo && data->fargo->type == Fargo::userdef) { + data->fargo->GetFargoVelocity(t); + } + + auto fluxCorrection = Fluid_CorrectFluxFunctor(this,dt); + + ///////////////////////////////////////////////////////////////////////////// + // Flux correction (for fargo/non-cartesian geometry) + ///////////////////////////////////////////////////////////////////////////// + const int ioffset = (dir==IDIR) ? 1 : 0; + const int joffset = (dir==JDIR) ? 1 : 0; + const int koffset = (dir==KDIR) ? 1 : 0; + idefix_for("Correct Flux", + data->beg[KDIR],data->end[KDIR]+koffset, + data->beg[JDIR],data->end[JDIR]+joffset, + data->beg[IDIR],data->end[IDIR]+ioffset, + fluxCorrection); + + + // If user has requested specific flux functions for the boundaries, here they come + if(boundary->haveFluxBoundary) boundary->EnforceFluxBoundaries(dir); + + auto calcRHS = Fluid_CalcRHSFunctor(this,dt); + ///////////////////////////////////////////////////////////////////////////// + // Final conserved quantity budget from fluxes divergence + ///////////////////////////////////////////////////////////////////////////// + idefix_for("CalcRightHandSide", + data->beg[KDIR],data->end[KDIR], + data->beg[JDIR],data->end[JDIR], + data->beg[IDIR],data->end[IDIR], + calcRHS); + + + idfx::popRegion(); +} +#endif // FLUID_CALCRIGHTHANDSIDE_HPP_ diff --git a/src/hydro/checkDivB.cpp b/src/fluid/checkDivB.hpp similarity index 93% rename from src/hydro/checkDivB.cpp rename to src/fluid/checkDivB.hpp index d857926d..f9aac45c 100644 --- a/src/hydro/checkDivB.cpp +++ b/src/fluid/checkDivB.hpp @@ -5,11 +5,14 @@ // Licensed under CeCILL 2.1 License, see COPYING for more information // *********************************************************************************** -#include "hydro.hpp" -#include "dataBlock.hpp" +#ifndef FLUID_CHECKDIVB_HPP_ +#define FLUID_CHECKDIVB_HPP_ +#include "fluid.hpp" +#include "dataBlock.hpp" -real Hydro::CheckDivB() { +template +real Fluid::CheckDivB() { real divB; IdefixArray4D Vs = this->Vs; IdefixArray1D dx1 = data->dx[IDIR]; @@ -50,7 +53,7 @@ real Hydro::CheckDivB() { /* -real Hydro::CheckDivB(DataBlock &data) { +real Fluid::CheckDivB(DataBlock &data) { real divB=0; IdefixArray4D Vs = this->Vs; IdefixArray3D Ax1 = data->A[IDIR]; @@ -93,3 +96,5 @@ real Hydro::CheckDivB(DataBlock &data) { return(divB); } */ + +#endif // FLUID_CHECKDIVB_HPP_ diff --git a/src/fluid/checkNan.hpp b/src/fluid/checkNan.hpp new file mode 100644 index 00000000..71911c58 --- /dev/null +++ b/src/fluid/checkNan.hpp @@ -0,0 +1,130 @@ +// *********************************************************************************** +// Idefix MHD astrophysical code +// Copyright(C) Geoffroy R. J. Lesur +// and other code contributors +// Licensed under CeCILL 2.1 License, see COPYING for more information +// *********************************************************************************** + +#ifndef FLUID_CHECKNAN_HPP_ +#define FLUID_CHECKNAN_HPP_ +#include "dataBlock.hpp" +#include "dataBlockHost.hpp" +#include "fluid.hpp" + +// Check if current datablock has nans + +template +int Fluid::CheckNan() { + int nanVs=0; + int nanVc=0; + + idfx::pushRegion("Fluid::CheckNan"); + IdefixArray4D Vc=this->Vc; + + idefix_reduce("checkNanVc", + 0, Phys::nvar, + data->beg[KDIR], data->end[KDIR], + data->beg[JDIR], data->end[JDIR], + data->beg[IDIR], data->end[IDIR], + KOKKOS_LAMBDA (int n, int k, int j, int i, int &nnan) { + if(std::isnan(Vc(n,k,j,i))) nnan++; + }, Kokkos::Sum(nanVc) // reduction variable + ); + + if constexpr(Phys::mhd) { + IdefixArray4D Vs=this->Vs; + idefix_reduce("checkNanVs", + 0, DIMENSIONS, + data->beg[KDIR], data->end[KDIR]+KOFFSET, + data->beg[JDIR], data->end[JDIR]+JOFFSET, + data->beg[IDIR], data->end[IDIR]+IOFFSET, + KOKKOS_LAMBDA (int n, int k, int j, int i, int &nnan) { + if(std::isnan(Vs(n,k,j,i))) nnan++; + }, Kokkos::Sum(nanVs) // reduction variable + ); + } + + + int nanTot = nanVc+nanVs; + #ifdef WITH_MPI + MPI_Allreduce(MPI_IN_PLACE, &nanTot,1,MPI_INT, MPI_SUM, MPI_COMM_WORLD); + #endif + if(nanTot>0) { + idfx::cout << "Fluid<" << prefix << ">: Nans were found in the current calculation" + << std::endl; + if(nanVc+nanVs>0) { + // Each datablock shows its findings to stdout + std::cout << "Fluid<" << prefix << ">: rank " << idfx::prank + << " found " << nanVc << " Nans in Vc" + #if MHD == YES + << " and " << nanVs << " Nans in Vs" + #endif + << " in the current datablock. Details will be in corresponding process log file." + << std::endl; + + // We need to make copies to find exactly where the thing is wrong + + DataBlockHost dataHost(*data); + + IdefixHostArray4D VcHost = Kokkos::create_mirror_view(this->Vc); + Kokkos::deep_copy(VcHost,Vc); + + int nerrormax=10; + + for(int k = data->beg[KDIR] ; k < data->end[KDIR] ; k++) { + for(int j = data->beg[JDIR] ; j < data->end[JDIR] ; j++) { + for(int i = data->beg[IDIR] ; i < data->end[IDIR] ; i++) { + for(int n = 0 ; n < Phys::nvar ; n ++) { + if(std::isnan(VcHost(n,k,j,i)) && nerrormax>0) { + nerrormax--; + idfx::cout << "rank " << idfx::prank << ": Nan found in variable " + << this->VcName[n] << std::endl; + + idfx::cout << " global (i,j,k) = (" + << i-data->beg[IDIR]+data->gbeg[IDIR]-data->nghost[IDIR] + << ", " << j-data->beg[JDIR]+data->gbeg[JDIR]-data->nghost[JDIR] << ", " + << k-data->beg[KDIR]+data->gbeg[KDIR]-data->nghost[KDIR] << ")" << std::endl; + + idfx::cout << " global (x,y,z) = (" << dataHost.x[IDIR](i) << ", " + << dataHost.x[JDIR](j) << ", " << dataHost.x[KDIR](k) << ")" << std::endl; + } + } + } + } + } + + if constexpr(Phys::mhd) { + IdefixHostArray4D VsHost = Kokkos::create_mirror_view(this->Vs); + Kokkos::deep_copy(VsHost,Vs); + for(int k = data->beg[KDIR] ; k < data->end[KDIR]+KOFFSET ; k++) { + for(int j = data->beg[JDIR] ; j < data->end[JDIR]+JOFFSET ; j++) { + for(int i = data->beg[IDIR] ; i < data->end[IDIR]+IOFFSET ; i++) { + for(int n = 0 ; n < DIMENSIONS ; n ++) { + if(std::isnan(VsHost(n,k,j,i)) && nerrormax>0) { + nerrormax--; + idfx::cout << "rank " << idfx::prank << ": Nan found in variable " + << this->VsName[n] << std::endl; + idfx::cout << " global (i,j,k) = (" + << i-data->beg[IDIR]+data->gbeg[IDIR]-data->nghost[IDIR] + << ", " << j-data->beg[JDIR]+data->gbeg[JDIR]-data->nghost[JDIR] << ", " + << k-data->beg[KDIR]+data->gbeg[KDIR]-data->nghost[KDIR] << ")" << std::endl; + + idfx::cout << " global (x,y,z) = (" << dataHost.x[IDIR](i) << ", " + << dataHost.x[JDIR](j) << ", " << dataHost.x[KDIR](k) << ")" << std::endl; + } + } + } + } + } + } + if(nerrormax<=0) { + idfx::cout << "... " << std::endl << "*** More Nans have been found in current fluid. " + << "Only showing the first 10." << std::endl; + } + } + } + idfx::popRegion(); + return(nanTot); +} + +#endif // FLUID_CHECKNAN_HPP_ diff --git a/src/fluid/coarsenFlow.hpp b/src/fluid/coarsenFlow.hpp new file mode 100644 index 00000000..62e77095 --- /dev/null +++ b/src/fluid/coarsenFlow.hpp @@ -0,0 +1,331 @@ +// *********************************************************************************** +// Idefix MHD astrophysical code +// Copyright(C) Geoffroy R. J. Lesur +// and other code contributors +// Licensed under CeCILL 2.1 License, see COPYING for more information +// *********************************************************************************** + +#ifndef FLUID_COARSENFLOW_HPP_ +#define FLUID_COARSENFLOW_HPP_ + +#include "fluid.hpp" +#include "dataBlock.hpp" + +KOKKOS_INLINE_FUNCTION int Kmax(int n, int m) { + return( n>m ? n : m); +} +// This function coarsen the flow according to the grid coarsening array + +template +void Fluid::CoarsenFlow(IdefixArray4D &Vi) { + idfx::pushRegion("Fluid::CoarsenFlow"); + + IdefixArray3D dV = data->dV; + for(int dir = 0 ; dir < DIMENSIONS ; dir++) { + if(!data->coarseningDirection[dir]) continue; + int begDir = data->beg[dir]; + int endDir = data->end[dir]; + + IdefixArray2D coarseningLevel = data->coarseningLevel[dir]; + + idefix_for("FLUID_CoarsenFlow", + 0, Phys::nvar, + data->beg[KDIR],data->end[KDIR], + data->beg[JDIR],data->end[JDIR], + data->beg[IDIR],data->end[IDIR], + KOKKOS_LAMBDA (int n, int k, int j, int i) { + int factor, index; + int ioffset = 0; + int joffset = 0; + int koffset = 0; + //factor = 2^(coarsening-1) + if(dir==IDIR) { + factor = 1 << (coarseningLevel(k,j) - 1); + index = i; + ioffset = 1; + } + if(dir==JDIR) { + factor = 1 << (coarseningLevel(k,i) - 1); + index = j; + joffset = 1; + } + if(dir==KDIR) { + factor = 1 << (coarseningLevel(j,i) - 1); + index = k; + koffset = 1; + } + // check if coarsening is required in this region + if(factor>1) { + // We average the cells by groups of "factor" in direction "dir", + // so only the first element of each group will do the job. + if( (index-begDir)%factor == 0) { + real q = 0.0; + real V = 0.0; + for(int shift = 0 ; shift < factor ; shift++) { + q = q + Vi(n, k + shift*koffset, j + shift*joffset, i+shift*ioffset) + * dV(k + shift*koffset, j + shift*joffset, i+shift*ioffset); + V = V + dV(k + shift*koffset, j + shift*joffset, i+shift*ioffset); + } + // Average + q = q/V; + // Write back the cell elements + for(int shift = 0 ; shift < factor ; shift++) { + Vi(n, k + shift*koffset, j + shift*joffset, i+shift*ioffset) = q; + } + } + } + }); + } + idfx::popRegion(); +} + +template +void Fluid::CoarsenMagField(IdefixArray4D &Vsin) { + if constexpr(Phys::mhd) { + idfx::pushRegion("Fluid::CoarsenMagField"); + #if DIMENSIONS >= 2 + /********************************** + * MHD Part * + * ********************************/ + for(int dir = 0 ; dir < DIMENSIONS ; dir++) { + if(!data->coarseningDirection[dir]) continue; + int begDir = data->beg[dir]; + int endDir = data->end[dir]; + + IdefixArray2D coarseningLevel = data->coarseningLevel[dir]; + + const int BXn = dir; + const int BXt = (dir == IDIR ? BX2s : BX1s); + const int BXb = (dir == KDIR ? BX2s : BX3s); + + IdefixArray3D An = data->A[dir]; + IdefixArray3D At = data->A[BXt]; + IdefixArray3D Ab = data->A[BXb]; + + int it = 0, ib = 0; + int jt = 0, jb = 0; + int kt = 0, kb = 0; + int endk = data->end[KDIR]; + int endj = data->end[JDIR]; + int endi = data->end[IDIR]; + + // find the index over which the faces are off-centered + // (trick so that we can write a single loop) + if(dir==IDIR) { + jt=1; + kb=1; + endj++; + #if DIMENSIONS == 3 + endk++; + #endif + } + if(dir==JDIR) { + it=1; + kb=1; + endi++; + #if DIMENSIONS == 3 + endk++; + #endif + } + if(dir==KDIR) { + it=1; + jb=1; + endi++; + endj++; + } + #if DIMENSIONS < 3 + // force kb to 0 + kb = 0; + #endif + + // Coarsen components normal to the coarsening direction first (BXt and BXb) + idefix_for("FLUID_CoarsenFlow_BXsn", + data->beg[KDIR],endk, + data->beg[JDIR],endj, + data->beg[IDIR],endi, + KOKKOS_LAMBDA (int k, int j, int i) { + int coarsening_t =1; + [[maybe_unused]] int coarsening_b = 1; + int index; + int ioffset = 0; + int joffset = 0; + int koffset = 0; + if(dir==IDIR) { + coarsening_t = Kmax(coarseningLevel(k,j-1), coarseningLevel(k,j)); + #if DIMENSIONS == 3 + coarsening_b = Kmax(coarseningLevel(k-1,j), coarseningLevel(k,j)); + #endif + ioffset = 1; + index=i; + } + if(dir==JDIR) { + coarsening_t = Kmax(coarseningLevel(k,i-1), coarseningLevel(k,i)); + #if DIMENSIONS == 3 + coarsening_b = Kmax(coarseningLevel(k-1,i), coarseningLevel(k,i)); + #endif + joffset = 1; + index=j; + } + if(dir==KDIR) { + coarsening_t = Kmax(coarseningLevel(j,i-1), coarseningLevel(j,i)); + coarsening_b = Kmax(coarseningLevel(j-1,i), coarseningLevel(j,i)); + koffset = 1; + index=k; + } + + //coarsening_t=0; + //coarsening_b=0; + // Treat t component + if(coarsening_t>1) { + int factor_t = 1 << (coarsening_t-1); + // We average the cells by groups of "factor" in direction "dir", + // so only the first element of each group will do the job. + if( (index-begDir)%factor_t == 0) { + real q = 0.0; + real A = 0.0; + for(int shift = 0 ; shift < factor_t ; shift++) { + q = q + Vsin(BXt, k + shift*koffset, j + shift*joffset, i+shift*ioffset) + * At(k + shift*koffset, j + shift*joffset, i+shift*ioffset); + A = A + At(k + shift*koffset, j + shift*joffset, i+shift*ioffset); + } + + // If the Area is zero, do a point average instead (this happens on the axis instance) + if(FABS(A) < 1e-10) { + // do a point average instead of a surface average + q=0.0; + A=0.0; + for(int shift = 0 ; shift < factor_t ; shift++) { + q = q + Vsin(BXt, k + shift*koffset, j + shift*joffset, i+shift*ioffset); + A = A + 1.0; + } + } + q = q/A; + // Write back the cell elements + for(int shift = 0 ; shift < factor_t ; shift++) { + Vsin(BXt, k + shift*koffset, j + shift*joffset, i+shift*ioffset) = q; + } + } + } + // Treat b component + #if DIMENSIONS == 3 + if(coarsening_b>1) { + int factor_b = 1 << (coarsening_b-1); + // We average the cells by groups of "factor" in direction "dir", + // so only the first element of each group will do the job. + if( (index-begDir)%factor_b == 0) { + real q = 0.0; + real A = 0.0; + for(int shift = 0 ; shift < factor_b ; shift++) { + q = q + Vsin(BXb, k + shift*koffset, j + shift*joffset, i+shift*ioffset) + * Ab(k + shift*koffset, j + shift*joffset, i+shift*ioffset); + A = A + Ab(k + shift*koffset, j + shift*joffset, i+shift*ioffset); + } + // If the Area is zero, do a point average instead (this happens on the axis instance) + if(FABS(A) < 1e-10) { + // do a point average instead of a surface average + q=0.0; + A=0.0; + for(int shift = 0 ; shift < factor_b ; shift++) { + q = q + Vsin(BXb, k + shift*koffset, j + shift*joffset, i+shift*ioffset); + A = A + 1.0; + } + } + // Average + q = q/A; + // Write back the cell elements + for(int shift = 0 ; shift < factor_b ; shift++) { + Vsin(BXb, k + shift*koffset, j + shift*joffset, i+shift*ioffset) = q; + } + } + } + #endif + }); + + // Coarsen components parralel the coarsening direction (BXn) + idefix_for("FLUID_CoarsenFlow_BXsn", + data->beg[KDIR],data->end[KDIR], + data->beg[JDIR],data->end[JDIR], + data->beg[IDIR],data->end[IDIR], + KOKKOS_LAMBDA (int k, int j, int i) { + int coarsening; + int index; + int ioffset = 0; + int joffset = 0; + int koffset = 0; + // Pick the highest coarsening level of the current coordinates and its immediate neighbours + if(dir==IDIR) { + coarsening = coarseningLevel(k,j); + coarsening = Kmax(coarsening, coarseningLevel(k,j-1)); + coarsening = Kmax(coarsening, coarseningLevel(k,j+1)); + #if DIMENSIONS == 3 + coarsening = Kmax(coarsening, coarseningLevel(k-1,j)); + coarsening = Kmax(coarsening, coarseningLevel(k+1,j)); + #endif + ioffset = 1; + index=i; + } + if(dir==JDIR) { + coarsening = coarseningLevel(k,i); + coarsening = Kmax(coarsening, coarseningLevel(k,i-1)); + coarsening = Kmax(coarsening, coarseningLevel(k,i+1)); + #if DIMENSIONS == 3 + coarsening = Kmax(coarsening, coarseningLevel(k-1,i)); + coarsening = Kmax(coarsening, coarseningLevel(k+1,i)); + #endif + joffset = 1; + index=j; + } + if(dir==KDIR) { + coarsening = coarseningLevel(j,i); + coarsening = Kmax(coarsening, coarseningLevel(j,i-1)); + coarsening = Kmax(coarsening, coarseningLevel(j,i+1)); + coarsening = Kmax(coarsening, coarseningLevel(j-1,i)); + coarsening = Kmax(coarsening, coarseningLevel(j+1,i)); + koffset = 1; + index=k; + } + + if(coarsening > 1) { + // the current cell had tangential field components which have been coarsened. Hence, + // We reconstruct the parrallel field component with divB=0 + + int factor = 1 << (coarsening-1); + if( (index-begDir)%factor == 0) { + for(int shift = 0 ; shift < factor-1 ; shift++) { + real qt = Vsin(BXt,k+kt+shift*koffset, j+jt+shift*joffset, i+it+shift*ioffset) + * At(k+kt+shift*koffset, j+jt+shift*joffset, i+it+shift*ioffset) + - Vsin(BXt,k+shift*koffset, j+shift*joffset, i+shift*ioffset) + * At(k+shift*koffset, j+shift*joffset, i+shift*ioffset); + real qb = Vsin(BXb,k+kb+shift*koffset, j+jb+shift*joffset, i+ib+shift*ioffset) + * Ab(k+kb+shift*koffset, j+jb+shift*joffset, i+ib+shift*ioffset) + - Vsin(BXb,k+shift*koffset, j+shift*joffset, i+shift*ioffset) + * Ab(k+shift*koffset, j+shift*joffset, i+shift*ioffset); + /* + if(i==10 && j == 3 && k == 10) { + idfx::cout << "shift=" << shift << " ; oldBxn=" << Vsin(BXn, k+(shift+1)*koffset, j+(shift+1)*joffset, i+(shift+1)*ioffset) << std::endl; + idfx::cout << "qt=" << qt << " ; qb=" << qb << std::endl; + }*/ + + Vsin(BXn, k+(shift+1)*koffset, j+(shift+1)*joffset, i+(shift+1)*ioffset) = + 1.0/An(k+(shift+1)*koffset, j+(shift+1)*joffset, i+(shift+1)*ioffset) * + ( + Vsin(BXn, k+shift*koffset, j+shift*joffset, i+shift*ioffset) * + An(k+shift*koffset, j+shift*joffset, i+shift*ioffset) + - qt - qb + ); + /* + if(i==10 && j == 3 && k == 10) { + idfx::cout << "shift=" << shift << " ; newsBxn=" << Vsin(BXn, k+(shift+1)*koffset, j+(shift+1)*joffset, i+(shift+1)*ioffset) << std::endl; + }*/ + } + } + } + }); + } + #endif // DIMENSIONS>=2 + idfx::popRegion(); + } //MHD +} + + +#endif // FLUID_COARSENFLOW_HPP_ diff --git a/src/fluid/constrainedTransport/CMakeLists.txt b/src/fluid/constrainedTransport/CMakeLists.txt new file mode 100644 index 00000000..06efc782 --- /dev/null +++ b/src/fluid/constrainedTransport/CMakeLists.txt @@ -0,0 +1,10 @@ +target_sources(idefix + PUBLIC ${CMAKE_CURRENT_LIST_DIR}/calcCornerEmf.hpp + PUBLIC ${CMAKE_CURRENT_LIST_DIR}/calcNonidealEMF.hpp + PUBLIC ${CMAKE_CURRENT_LIST_DIR}/calcRiemannEmf.hpp + PUBLIC ${CMAKE_CURRENT_LIST_DIR}/constrainedTransport.hpp + PUBLIC ${CMAKE_CURRENT_LIST_DIR}/EMFexchange.hpp + PUBLIC ${CMAKE_CURRENT_LIST_DIR}/enforceEMFBoundary.hpp + PUBLIC ${CMAKE_CURRENT_LIST_DIR}/evolveMagField.hpp + PUBLIC ${CMAKE_CURRENT_LIST_DIR}/evolveVectorPotential.hpp + ) diff --git a/src/hydro/electromotiveforce/EMFexchange.cpp b/src/fluid/constrainedTransport/EMFexchange.hpp similarity index 94% rename from src/hydro/electromotiveforce/EMFexchange.cpp rename to src/fluid/constrainedTransport/EMFexchange.hpp index 51a04327..18718627 100644 --- a/src/hydro/electromotiveforce/EMFexchange.cpp +++ b/src/fluid/constrainedTransport/EMFexchange.hpp @@ -5,12 +5,16 @@ // Licensed under CeCILL 2.1 License, see COPYING for more information // *********************************************************************************** -#include "electroMotiveForce.hpp" -#include "hydro.hpp" +#ifndef FLUID_CONSTRAINEDTRANSPORT_EMFEXCHANGE_HPP_ +#define FLUID_CONSTRAINEDTRANSPORT_EMFEXCHANGE_HPP_ + +#include "constrainedTransport.hpp" +#include "fluid.hpp" #include "dataBlock.hpp" #ifdef WITH_MPI -void ElectroMotiveForce::ExchangeAll() { +template +void ConstrainedTransport::ExchangeAll() { if(data->mygrid->nproc[IDIR]>1) this->ExchangeX1(); if(data->mygrid->nproc[JDIR]>1) this->ExchangeX2(); if(data->mygrid->nproc[KDIR]>1) this->ExchangeX3(); @@ -18,7 +22,8 @@ void ElectroMotiveForce::ExchangeAll() { // Exchange EMFs in X1 -void ElectroMotiveForce::ExchangeX1() { +template +void ConstrainedTransport::ExchangeX1() { idfx::pushRegion("Emf::ExchangeX1"); @@ -118,7 +123,8 @@ void ElectroMotiveForce::ExchangeX1() { } // Exchange EMFs in X2 -void ElectroMotiveForce::ExchangeX2() { +template +void ConstrainedTransport::ExchangeX2() { idfx::pushRegion("Emf::ExchangeX2"); // Load the buffers with data @@ -213,7 +219,8 @@ void ElectroMotiveForce::ExchangeX2() { } // Exchange EMFs in X3 -void ElectroMotiveForce::ExchangeX3() { +template +void ConstrainedTransport::ExchangeX3() { idfx::pushRegion("Emf::ExchangeX3"); @@ -310,3 +317,4 @@ void ElectroMotiveForce::ExchangeX3() { } #endif +#endif // FLUID_CONSTRAINEDTRANSPORT_EMFEXCHANGE_HPP_ diff --git a/src/hydro/electromotiveforce/calcCornerEmf.cpp b/src/fluid/constrainedTransport/calcCornerEmf.hpp similarity index 93% rename from src/hydro/electromotiveforce/calcCornerEmf.cpp rename to src/fluid/constrainedTransport/calcCornerEmf.hpp index f0da98b7..0a998aa5 100644 --- a/src/hydro/electromotiveforce/calcCornerEmf.cpp +++ b/src/fluid/constrainedTransport/calcCornerEmf.hpp @@ -4,13 +4,16 @@ // and other code contributors // Licensed under CeCILL 2.1 License, see COPYING for more information // *********************************************************************************** +#ifndef FLUID_CONSTRAINEDTRANSPORT_CALCCORNEREMF_HPP_ +#define FLUID_CONSTRAINEDTRANSPORT_CALCCORNEREMF_HPP_ -#include "hydro.hpp" +#include "fluid.hpp" #include "dataBlock.hpp" // Compute Corner EMFs from the one stored in the Riemann step -void ElectroMotiveForce::CalcCornerEMF(real t) { - idfx::pushRegion("ElectroMotiveForce::CalcCornerEMF"); +template +void ConstrainedTransport::CalcCornerEMF(real t) { + idfx::pushRegion("ConstrainedTransport::CalcCornerEMF"); #if MHD == YES && DIMENSIONS >= 2 @@ -37,8 +40,9 @@ void ElectroMotiveForce::CalcCornerEMF(real t) { // Compute Corner EMFs from arithmetic averages -void ElectroMotiveForce::CalcArithmeticAverage() { - idfx::pushRegion("ElectroMotiveForce::CalcCornerEMF"); +template +void ConstrainedTransport::CalcArithmeticAverage() { + idfx::pushRegion("ConstrainedTransport::CalcCornerEMF"); // Corned EMFs IdefixArray3D ex = this->ex; @@ -77,8 +81,9 @@ void ElectroMotiveForce::CalcArithmeticAverage() { idfx::popRegion(); } -void ElectroMotiveForce::CalcCellCenteredEMF() { - idfx::pushRegion("ElectroMotiveForce::CalcCellCenteredEMF"); +template +void ConstrainedTransport::CalcCellCenteredEMF() { + idfx::pushRegion("ConstrainedTransport::CalcCellCenteredEMF"); IdefixArray4D Vc = hydro->Vc; // cell-centered EMFs IdefixArray3D Ex1 = this->Ex1; @@ -120,8 +125,9 @@ void ElectroMotiveForce::CalcCellCenteredEMF() { idfx::popRegion(); } -void ElectroMotiveForce::CalcUCT0Average() { - idfx::pushRegion("ElectroMotiveForce::CalcUCT0Average"); +template +void ConstrainedTransport::CalcUCT0Average() { + idfx::pushRegion("ConstrainedTransport::CalcUCT0Average"); // Corned EMFs IdefixArray3D ex = this->ex; IdefixArray3D ey = this->ey; @@ -187,8 +193,9 @@ void ElectroMotiveForce::CalcUCT0Average() { idfx::popRegion(); } -void ElectroMotiveForce::CalcContactAverage() { - idfx::pushRegion("ElectroMotiveForce::CalcContactAverage"); +template +void ConstrainedTransport::CalcContactAverage() { + idfx::pushRegion("ConstrainedTransport::CalcContactAverage"); // Corned EMFs IdefixArray3D ex = this->ex; IdefixArray3D ey = this->ey; @@ -404,3 +411,4 @@ void ElectroMotiveForce::CalcContactAverage() { #endif // MHD idfx::popRegion(); } +#endif // FLUID_CONSTRAINEDTRANSPORT_CALCCORNEREMF_HPP_ diff --git a/src/hydro/electromotiveforce/calcNonidealEMF.cpp b/src/fluid/constrainedTransport/calcNonidealEMF.hpp similarity index 93% rename from src/hydro/electromotiveforce/calcNonidealEMF.cpp rename to src/fluid/constrainedTransport/calcNonidealEMF.hpp index 59dac913..2253e33d 100644 --- a/src/hydro/electromotiveforce/calcNonidealEMF.cpp +++ b/src/fluid/constrainedTransport/calcNonidealEMF.hpp @@ -5,12 +5,16 @@ // Licensed under CeCILL 2.1 License, see COPYING for more information // *********************************************************************************** -#include "hydro.hpp" +#ifndef FLUID_CONSTRAINEDTRANSPORT_CALCNONIDEALEMF_HPP_ +#define FLUID_CONSTRAINEDTRANSPORT_CALCNONIDEALEMF_HPP_ + +#include "fluid.hpp" #include "dataBlock.hpp" // Compute Corner EMFs from nonideal MHD -void ElectroMotiveForce::CalcNonidealEMF(real t) { - idfx::pushRegion("ElectroMotiveForce::CalcNonidealEMF"); +template +void ConstrainedTransport::CalcNonidealEMF(real t) { + idfx::pushRegion("ConstrainedTransport::CalcNonidealEMF"); #if MHD == YES // Corned EMFs @@ -170,3 +174,5 @@ void ElectroMotiveForce::CalcNonidealEMF(real t) { idfx::popRegion(); } + +#endif //FLUID_CONSTRAINEDTRANSPORT_CALCNONIDEALEMF_HPP_ diff --git a/src/fluid/constrainedTransport/calcRiemannEmf.hpp b/src/fluid/constrainedTransport/calcRiemannEmf.hpp new file mode 100644 index 00000000..3c9479c2 --- /dev/null +++ b/src/fluid/constrainedTransport/calcRiemannEmf.hpp @@ -0,0 +1,435 @@ +// *********************************************************************************** +// Idefix MHD astrophysical code +// Copyright(C) Geoffroy R. J. Lesur +// and other code contributors +// Licensed under CeCILL 2.1 License, see COPYING for more information +// *********************************************************************************** + +#ifndef FLUID_CONSTRAINEDTRANSPORT_CALCRIEMANNEMF_HPP_ +#define FLUID_CONSTRAINEDTRANSPORT_CALCRIEMANNEMF_HPP_ + +#include "fluid.hpp" +#include "dataBlock.hpp" +#include "slopeLimiter.hpp" // Contains the main slope limiters called in this function +#include "physics.hpp" + +// A shortcut for the slopelimiter template parameters +// Note that we do not use the full class, so the template parameters are meaningless +// here +using SL = SlopeLimiter<>; + +template +void ConstrainedTransport::CalcRiemannAverage() { + idfx::pushRegion("ConstrainedTransport::calcRiemannAverage"); + +#if EMF_AVERAGE == UCT_HLLD || EMF_AVERAGE == UCT_HLL + + // Corned EMFs + IdefixArray3D ez = this->ez; +#if DIMENSIONS == 3 + IdefixArray3D ex = this->ex; + IdefixArray3D ey = this->ey; + + // Face-centered EMFs + IdefixArray3D exj = this->exj; + IdefixArray3D exk = this->exk; + IdefixArray3D eyi = this->eyi; + IdefixArray3D eyk = this->eyk; +#endif + + // Face-centered EMFs + IdefixArray3D ezi = this->ezi; + IdefixArray3D ezj = this->ezj; + + IdefixArray3D axL = this->axL; + IdefixArray3D axR = this->axR; + IdefixArray3D ayL = this->ayL; + IdefixArray3D ayR = this->ayR; +#if DIMENSIONS == 3 + IdefixArray3D azL = this->azL; + IdefixArray3D azR = this->azR; +#endif + + IdefixArray3D dxL = this->dxL; + IdefixArray3D dxR = this->dxR; + IdefixArray3D dyL = this->dyL; + IdefixArray3D dyR = this->dyR; +#if DIMENSIONS == 3 + IdefixArray3D dzL = this->dzL; + IdefixArray3D dzR = this->dzR; +#endif + + IdefixArray4D Vs = hydro->Vs; + + idefix_for("CalcCenterEMF", + data->beg[KDIR],data->end[KDIR]+KOFFSET, + data->beg[JDIR],data->end[JDIR]+JOFFSET, + data->beg[IDIR],data->end[IDIR]+IOFFSET, + KOKKOS_LAMBDA (int k, int j, int i) { + // IDIR +#if DIMENSIONS >= 2 + [[maybe_unused]] real phi, vL, vR, dv, bL, bR, db; + [[maybe_unused]] real aL, aR, dL, dR; + + [[maybe_unused]] const int im = i-1, jm = j-1, km = k-1; + + // EMF: Z component at (i-1/2, j-1/2, k) + aL = HALF_F*(axL(k,jm,i) + axL(k,jm+1,i)); + aR = HALF_F*(axR(k,jm,i) + axR(k,jm+1,i)); + dL = HALF_F*(dxL(k,jm,i) + dxL(k,jm+1,i)); + dR = HALF_F*(dxR(k,jm,i) + dxR(k,jm+1,i)); + + #if ORDER == 4 + SL::getPPMStates( Vs(BX2s,k,j,im-2), + Vs(BX2s,k,j,im-1), + Vs(BX2s,k,j,im), + Vs(BX2s,k,j,im+1), + Vs(BX2s,k,j,im+2), + db, bL); // We don't use the left value at m1 + + SL::getPPMStates( Vs(BX2s,k,j,i-2), + Vs(BX2s,k,j,i-1), + Vs(BX2s,k,j,i), + Vs(BX2s,k,j,i+1), + Vs(BX2s,k,j,i+2), + bR, db); // We don't use the right value at i + #else + db = SL::McLim(Vs(BX2s,k,j,im+1) - Vs(BX2s,k,j,im), + Vs(BX2s,k,j,im) - Vs(BX2s,k,j,im-1)); + bL = Vs(BX2s,k,j,im) + HALF_F*db; + + db = SL::McLim(Vs(BX2s,k,j,i+1) - Vs(BX2s,k,j,i), + Vs(BX2s,k,j,i) - Vs(BX2s,k,j,i-1)); + bR = Vs(BX2s,k,j,i) - HALF_F*db; + + #endif + + #if ORDER == 4 + SL::getPPMStates( ezj(k,j,im-2), + ezj(k,j,im-1), + ezj(k,j,im), + ezj(k,j,im+1), + ezj(k,j,im+2), + dv, vL); // We don't use the left value at m1 + + SL::getPPMStates( ezj(k,j,i-2), + ezj(k,j,i-1), + ezj(k,j,i), + ezj(k,j,i+1), + ezj(k,j,i+2), + vR, dv); // We don't use the right value at i + #else + dv = SL::McLim(ezj(k,j,im+1) - ezj(k,j,im), + ezj(k,j,im) - ezj(k,j,im-1)); + vL = ezj(k,j,im) + HALF_F*dv; + + dv = SL::McLim(ezj(k,j,i+1) - ezj(k,j,i), + ezj(k,j,i) - ezj(k,j,i-1)); + vR = ezj(k,j,i) - HALF_F*dv; + #endif + + phi = dR*bR - dL*bL; + ez(k,j,i) = (aL*vL*bL + aR*vR*bR) + phi; +#endif + +#if DIMENSIONS == 3 + // EMF: Y component at (i-1/2, j, k-1/2) + aL = HALF_F*(axL(km,j,i) + axL(km+1,j,i)); + aR = HALF_F*(axR(km,j,i) + axR(km+1,j,i)); + dL = HALF_F*(dxL(km,j,i) + dxL(km+1,j,i)); + dR = HALF_F*(dxR(km,j,i) + dxR(km+1,j,i)); + + #if ORDER == 4 + SL::getPPMStates( Vs(BX3s,k,j,im-2), + Vs(BX3s,k,j,im-1), + Vs(BX3s,k,j,im), + Vs(BX3s,k,j,im+1), + Vs(BX3s,k,j,im+2), + db, bL); // We don't use the left value at m1 + + SL::getPPMStates( Vs(BX3s,k,j,i-2), + Vs(BX3s,k,j,i-1), + Vs(BX3s,k,j,i), + Vs(BX3s,k,j,i+1), + Vs(BX3s,k,j,i+2), + bR, db); // We don't use the right value at i + #else + db = SL::McLim(Vs(BX3s,k,j,im+1) - Vs(BX3s,k,j,im), + Vs(BX3s,k,j,im) - Vs(BX3s,k,j,im-1)); + bL = Vs(BX3s,k,j,im) + HALF_F*db; + + db = SL::McLim(Vs(BX3s,k,j,i+1) - Vs(BX3s,k,j,i), + Vs(BX3s,k,j,i) - Vs(BX3s,k,j,i-1)); + bR = Vs(BX3s,k,j,i) - HALF_F*db; + #endif + + #if ORDER == 4 + SL::getPPMStates( eyk(k,j,im-2), + eyk(k,j,im-1), + eyk(k,j,im), + eyk(k,j,im+1), + eyk(k,j,im+2), + dv, vL); // We don't use the left value at m1 + + SL::getPPMStates( eyk(k,j,i-2), + eyk(k,j,i-1), + eyk(k,j,i), + eyk(k,j,i+1), + eyk(k,j,i+2), + vR, dv); // We don't use the right value at i + #else + dv = SL::McLim(eyk(k,j,im+1) - eyk(k,j,im), + eyk(k,j,im) - eyk(k,j,im-1)); + vL = eyk(k,j,im) + HALF_F*dv; + + dv = SL::McLim(eyk(k,j,i+1) - eyk(k,j,i), + eyk(k,j,i) - eyk(k,j,i-1)); + vR = eyk(k,j,i) - HALF_F*dv; + #endif + + phi = dR*bR - dL*bL; + ey(k,j,i) = (aL*vL*bL + aR*vR*bR) - phi; +#endif + + // JDIR +#if DIMENSIONS >= 2 + #if DIMENSIONS == 3 + // EMF: X component at (i, j-1/2, k-1/2) + aL = HALF_F*(ayL(km,j,i) + ayL(km+1,j,i)); + aR = HALF_F*(ayR(km,j,i) + ayR(km+1,j,i)); + dL = HALF_F*(dyL(km,j,i) + dyL(km+1,j,i)); + dR = HALF_F*(dyR(km,j,i) + dyR(km+1,j,i)); + + #if ORDER == 4 + SL::getPPMStates(Vs(BX3s,k,jm-2,i), + Vs(BX3s,k,jm-1,i), + Vs(BX3s,k,jm,i), + Vs(BX3s,k,jm+1,i), + Vs(BX3s,k,jm+2,i), + db, bL); + + SL::getPPMStates(Vs(BX3s,k,j-2,i), + Vs(BX3s,k,j-1,i), + Vs(BX3s,k,j,i), + Vs(BX3s,k,j+1,i), + Vs(BX3s,k,j+2,i), + bR, db); + #else + db = SL::McLim(Vs(BX3s,k,jm+1,i) - Vs(BX3s,k,jm,i), + Vs(BX3s,k,jm,i) - Vs(BX3s,k,jm-1,i)); + bL = Vs(BX3s,k,jm,i) + HALF_F*db; + + db = SL::McLim(Vs(BX3s,k,j+1,i) - Vs(BX3s,k,j,i), + Vs(BX3s,k,j,i) - Vs(BX3s,k,j-1,i)); + bR = Vs(BX3s,k,j,i) - HALF_F*db; + #endif + + #if ORDER == 4 + SL::getPPMStates(exk(k,jm-2,i), + exk(k,jm-1,i), + exk(k,jm,i), + exk(k,jm+1,i), + exk(k,jm+2,i), + dv, vL); + + SL::getPPMStates(exk(k,j-2,i), + exk(k,j-1,i), + exk(k,j,i), + exk(k,j+1,i), + exk(k,j+2,i), + vR, dv); + #else + dv = SL::McLim(exk(k,jm+1,i) - exk(k,jm,i), + exk(k,jm,i) - exk(k,jm-1,i)); + vL = exk(k,jm,i) + HALF_F*dv; + + dv = SL::McLim(exk(k,j+1,i) - exk(k,j,i), + exk(k,j,i) - exk(k,j-1,i)); + vR = exk(k,j,i) - HALF_F*dv; + #endif + + phi = dR*bR - dL*bL; + ex(k,j,i) = (aL*vL*bL + aR*vR*bR) + phi; + #endif + + // EMF: Z component at (i-1/2, j-1/2, k) + aL = HALF_F*(ayL(k,j,im) + ayL(k,j,im+1)); + aR = HALF_F*(ayR(k,j,im) + ayR(k,j,im+1)); + dL = HALF_F*(dyL(k,j,im) + dyL(k,j,im+1)); + dR = HALF_F*(dyR(k,j,im) + dyR(k,j,im+1)); + + #if ORDER == 4 + SL::getPPMStates(Vs(BX1s,k,jm-2,i), + Vs(BX1s,k,jm-1,i), + Vs(BX1s,k,jm,i), + Vs(BX1s,k,jm+1,i), + Vs(BX1s,k,jm+2,i), + db, bL); + + SL::getPPMStates(Vs(BX1s,k,j-2,i), + Vs(BX1s,k,j-1,i), + Vs(BX1s,k,j,i), + Vs(BX1s,k,j+1,i), + Vs(BX1s,k,j+2,i), + bR, db); + #else + db = SL::McLim(Vs(BX1s,k,jm+1,i) - Vs(BX1s,k,jm,i), + Vs(BX1s,k,jm,i) - Vs(BX1s,k,jm-1,i)); + bL = Vs(BX1s,k,jm,i) + HALF_F*db; + + db = SL::McLim(Vs(BX1s,k,j+1,i) - Vs(BX1s,k,j,i), + Vs(BX1s,k,j,i) - Vs(BX1s,k,j-1,i)); + bR = Vs(BX1s,k,j,i) - HALF_F*db; + #endif + + #if ORDER == 4 + SL::getPPMStates(ezi(k,jm-2,i), + ezi(k,jm-1,i), + ezi(k,jm,i), + ezi(k,jm+1,i), + ezi(k,jm+2,i), + dv, vL); + + SL::getPPMStates(ezi(k,j-2,i), + ezi(k,j-1,i), + ezi(k,j,i), + ezi(k,j+1,i), + ezi(k,j+2,i), + vR, dv); + #else + dv = SL::McLim(ezi(k,jm+1,i) - ezi(k,jm,i), + ezi(k,jm,i) - ezi(k,jm-1,i)); + vL = ezi(k,jm,i) + HALF_F*dv; + + dv = SL::McLim(ezi(k,j+1,i) - ezi(k,j,i), + ezi(k,j,i) - ezi(k,j-1,i)); + vR = ezi(k,j,i) - HALF_F*dv; + #endif + + phi = dR*bR - dL*bL; + ez(k,j,i) += (aL*vL*bL + aR*vR*bR) - phi; +#endif + + // KDIR +#if DIMENSIONS == 3 + // EMF: Y component at (i-1/2, j, k-1/2) + aL = HALF_F*(azL(k,j,im) + azL(k,j,im+1)); + aR = HALF_F*(azR(k,j,im) + azR(k,j,im+1)); + dL = HALF_F*(dzL(k,j,im) + dzL(k,j,im+1)); + dR = HALF_F*(dzR(k,j,im) + dzR(k,j,im+1)); + + #if ORDER == 4 + SL::getPPMStates(Vs(BX1s,km-2,j,i), + Vs(BX1s,km-1,j,i), + Vs(BX1s,km,j,i), + Vs(BX1s,km+1,j,i), + Vs(BX1s,km+2,j,i), + db, bL); + + SL::getPPMStates(Vs(BX1s,k-2,j,i), + Vs(BX1s,k-1,j,i), + Vs(BX1s,k,j,i), + Vs(BX1s,k+1,j,i), + Vs(BX1s,k+2,j,i), + bR, db); + #else + db = SL::McLim(Vs(BX1s,km+1,j,i) - Vs(BX1s,km,j,i), + Vs(BX1s,km,j,i) - Vs(BX1s,km-1,j,i)); + bL = Vs(BX1s,km,j,i) + HALF_F*db; + + db = SL::McLim(Vs(BX1s,k+1,j,i) - Vs(BX1s,k,j,i), + Vs(BX1s,k,j,i) - Vs(BX1s,k-1,j,i)); + bR = Vs(BX1s,k,j,i) - HALF_F*db; + #endif + + #if ORDER == 4 + SL::getPPMStates(eyi(km-2,j,i), + eyi(km-1,j,i), + eyi(km,j,i), + eyi(km+1,j,i), + eyi(km+2,j,i), + dv, vL); + + SL::getPPMStates(eyi(k-2,j,i), + eyi(k-1,j,i), + eyi(k,j,i), + eyi(k+1,j,i), + eyi(k+2,j,i), + vR, dv); + #else + dv = SL::McLim(eyi(km+1,j,i) - eyi(km,j,i), + eyi(km,j,i) - eyi(km-1,j,i)); + vL = eyi(km,j,i) + HALF_F*dv; + + dv = SL::McLim(eyi(k+1,j,i) - eyi(k,j,i), + eyi(k,j,i) - eyi(k-1,j,i)); + vR = eyi(k,j,i) - HALF_F*dv; + #endif + + phi = dR*bR - dL*bL; + ey(k,j,i) += (aL*vL*bL + aR*vR*bR) + phi; + + // EMF: X component at (i, j-1/2, k-1/2) + aL = HALF_F*(azL(k,jm,i) + azL(k,jm+1,i)); + aR = HALF_F*(azR(k,jm,i) + azR(k,jm+1,i)); + dL = HALF_F*(dzL(k,jm,i) + dzL(k,jm+1,i)); + dR = HALF_F*(dzR(k,jm,i) + dzR(k,jm+1,i)); + + #if ORDER == 4 + SL::getPPMStates(Vs(BX2s,km-2,j,i), + Vs(BX2s,km-1,j,i), + Vs(BX2s,km,j,i), + Vs(BX2s,km+1,j,i), + Vs(BX2s,km+2,j,i), + db, bL); + + SL::getPPMStates(Vs(BX2s,k-2,j,i), + Vs(BX2s,k-1,j,i), + Vs(BX2s,k,j,i), + Vs(BX2s,k+1,j,i), + Vs(BX2s,k+2,j,i), + bR, db); + #else + db = SL::McLim(Vs(BX2s,km+1,j,i) - Vs(BX2s,km,j,i), + Vs(BX2s,km,j,i) - Vs(BX2s,km-1,j,i)); + bL = Vs(BX2s,km,j,i) + HALF_F*db; + + db = SL::McLim(Vs(BX2s,k+1,j,i) - Vs(BX2s,k,j,i), + Vs(BX2s,k,j,i) - Vs(BX2s,k-1,j,i)); + bR = Vs(BX2s,k,j,i) - HALF_F*db; + #endif + + #if ORDER == 4 + SL::getPPMStates(exj(km-2,j,i), + exj(km-1,j,i), + exj(km,j,i), + exj(km+1,j,i), + exj(km+2,j,i), + dv, vL); + + SL::getPPMStates(exj(k-2,j,i), + exj(k-1,j,i), + exj(k,j,i), + exj(k+1,j,i), + exj(k+2,j,i), + vR, dv); + #else + dv = SL::McLim(exj(km+1,j,i) - exj(km,j,i), + exj(km,j,i) - exj(km-1,j,i)); + vL = exj(km,j,i) + HALF_F*dv; + + dv = SL::McLim(exj(k+1,j,i) - exj(k,j,i), + exj(k,j,i) - exj(k-1,j,i)); + vR = exj(k,j,i) - HALF_F*dv; + #endif + + phi = dR*bR - dL*bL; + ex(k,j,i) += (aL*vL*bL + aR*vR*bR) - phi; +#endif + } + ); +#endif // EMF_AVERAGE + + idfx::popRegion(); +} +#endif // FLUID_CONSTRAINEDTRANSPORT_CALCRIEMANNEMF_HPP_ diff --git a/src/hydro/electromotiveforce/electroMotiveForce.cpp b/src/fluid/constrainedTransport/constrainedTransport.hpp similarity index 71% rename from src/hydro/electromotiveforce/electroMotiveForce.cpp rename to src/fluid/constrainedTransport/constrainedTransport.hpp index 754b57b2..c32e4ec9 100644 --- a/src/hydro/electromotiveforce/electroMotiveForce.cpp +++ b/src/fluid/constrainedTransport/constrainedTransport.hpp @@ -5,22 +5,154 @@ // Licensed under CeCILL 2.1 License, see COPYING for more information // *********************************************************************************** -#include -#include "electroMotiveForce.hpp" -#include "hydro.hpp" -#include "dataBlock.hpp" +#ifndef FLUID_CONSTRAINEDTRANSPORT_CONSTRAINEDTRANSPORT_HPP_ +#define FLUID_CONSTRAINEDTRANSPORT_CONSTRAINEDTRANSPORT_HPP_ + +#include "idefix.hpp" #include "input.hpp" +#include "riemannSolver.hpp" + +// Forward declarations +#include "physics.hpp" +template class Fluid; + +class DataBlock; + +template +class ConstrainedTransport { + public: + enum AveragingType {none, arithmetic, uct0, uct_contact, uct_hll, uct_hlld}; + + // Type of averaging + AveragingType averaging{none}; + + // Face centered emf components + IdefixArray3D exj; + IdefixArray3D exk; + IdefixArray3D eyi; + IdefixArray3D eyk; + IdefixArray3D ezi; + IdefixArray3D ezj; + + // Edge centered emf components + IdefixArray3D ex; + IdefixArray3D ey; + IdefixArray3D ez; + +// Required by uct_contact averaging + IdefixArray3D svx; + IdefixArray3D svy; + IdefixArray3D svz; + +// required by uct_hll averaging + IdefixArray3D axL; + IdefixArray3D axR; + IdefixArray3D ayL; + IdefixArray3D ayR; + #if DIMENSIONS == 3 + IdefixArray3D azL; + IdefixArray3D azR; -// Initialisation of electromotive force object -ElectroMotiveForce::ElectroMotiveForce() { - // Do nothing -} + IdefixArray3D dzL; + IdefixArray3D dzR; + #endif + IdefixArray3D dxL; + IdefixArray3D dxR; + IdefixArray3D dyL; + IdefixArray3D dyR; + + + IdefixArray3D Ex1; + IdefixArray3D Ex2; + IdefixArray3D Ex3; + + // Helper arrays for shearing box + IdefixArray2D sbEyL; + IdefixArray2D sbEyR; + IdefixArray2D sbEyRL; + + // Range of existence + + // Init from Hydro class + ConstrainedTransport(Input &, Fluid *); + ~ConstrainedTransport(); + + void EvolveMagField(real, real, IdefixArray4D&); + void CalcCornerEMF(real ); + void ShowConfig(); + + // Different flavors of EMF average schemes + void CalcRiemannAverage(); + void CalcArithmeticAverage(); + void CalcCellCenteredEMF(); + void CalcUCT0Average(); + void CalcContactAverage(); + + // Enforce boundary conditions on the EMFs. + void EnforceEMFBoundary(); + void CalcNonidealEMF(real ); + + // Specific routines to symmetrize EMFs with shearing box BCs + void SymmetrizeEMFShearingBox(); + void ExtrapolateEMFShearingBox(BoundarySide, + IdefixArray2D, + IdefixArray2D); + + // Routines for evolving the magnetic potential (only available when EVOLVE_VECTOR_POTENTIAL) + void EvolveVectorPotential(real, IdefixArray4D &); + void ComputeMagFieldFromA(IdefixArray4D &Vein, IdefixArray4D &Vsout); + +#ifdef WITH_MPI + // Exchange surface EMFs to remove interprocess round off errors + void ExchangeAll(); + void ExchangeX1(); + void ExchangeX2(); + void ExchangeX3(); +#endif + + private: + DataBlock *data; + Fluid *hydro; + +#ifdef WITH_MPI + enum {faceRight, faceLeft}; + + // Buffers for MPI calls + IdefixArray1D BufferSendX1[2]; + IdefixArray1D BufferSendX2[2]; + IdefixArray1D BufferSendX3[2]; + IdefixArray1D BufferRecvX1[2]; + IdefixArray1D BufferRecvX2[2]; + IdefixArray1D BufferRecvX3[2]; + + IdefixArray1D mapVars; + + int bufferSizeX1; + int bufferSizeX2; + int bufferSizeX3; + + // Requests for MPI persistent communications + MPI_Request sendRequestX1[2]; + MPI_Request sendRequestX2[2]; + MPI_Request sendRequestX3[2]; + MPI_Request recvRequestX1[2]; + MPI_Request recvRequestX2[2]; + MPI_Request recvRequestX3[2]; + + Kokkos::Timer timer; // Internal MPI timer +#endif +}; + + +#include +#include "fluid.hpp" +#include "dataBlock.hpp" // Init the emf from a datablock pointer -void ElectroMotiveForce::Init(Input &input, Hydro *hydro) { - idfx::pushRegion("ElectroMotiveForce::Init"); +template +ConstrainedTransport::ConstrainedTransport(Input &input, Fluid *hydro) { + idfx::pushRegion("ConstrainedTransport::Init"); -#if MHD == YES if(input.CheckEntry("Hydro","emf")>=0) { std::string opType = input.Get("Hydro","emf",0); if(opType.compare("arithmetic")==0) { @@ -34,7 +166,7 @@ void ElectroMotiveForce::Init(Input &input, Hydro *hydro) { } else if(opType.compare("uct_hlld")==0) { this->averaging = uct_hlld; } else { - idfx::cout << "ElectroMotiveForce: unknown averaging scheme " << opType << std::endl; + idfx::cout << "ConstrainedTransport: unknown averaging scheme " << opType << std::endl; IDEFIX_ERROR("Unknown EMF averaging scheme"); } } else { @@ -120,7 +252,8 @@ void ElectroMotiveForce::Init(Input &input, Hydro *hydro) { data->np_tot[KDIR], data->np_tot[JDIR], data->np_tot[IDIR]); ) } if(averaging==uct_hlld) { - if(hydro->mySolver == Solver::HLL || hydro->mySolver == Solver::TVDLF) { + if( hydro->rSolver->GetSolver() == RiemannSolver::Solver::HLL_MHD + || hydro->rSolver->GetSolver() == RiemannSolver::Solver::TVDLF_MHD) { IDEFIX_ERROR("HLLD EMF reconstruction is only compatible with HLLD or ROE Riemann solvers"); } } @@ -256,7 +389,6 @@ void ElectroMotiveForce::Init(Input &input, Hydro *hydro) { #endif // WITH_MPI -#endif // MHD==YES idfx::popRegion(); } @@ -265,7 +397,8 @@ void ElectroMotiveForce::Init(Input &input, Hydro *hydro) { // Destructor (clean up persistent communication channels) -ElectroMotiveForce::~ElectroMotiveForce() { +template +ConstrainedTransport::~ConstrainedTransport() { #if MHD == YES #ifdef WITH_MPI // Properly clean up the mess @@ -288,24 +421,35 @@ ElectroMotiveForce::~ElectroMotiveForce() { #endif } -void ElectroMotiveForce::ShowConfig() { +template +void ConstrainedTransport::ShowConfig() { switch(averaging) { case arithmetic: - idfx::cout << "ElectroMotiveForce: Using ARITHMETIC averaging scheme." << std::endl; + idfx::cout << "ConstrainedTransport: Using ARITHMETIC averaging scheme." << std::endl; break; case uct0: - idfx::cout << "ElectroMotiveForce: Using UCT0 averaging scheme." << std::endl; + idfx::cout << "ConstrainedTransport: Using UCT0 averaging scheme." << std::endl; break; case uct_contact: - idfx::cout << "ElectroMotiveForce: Using UCT_CONTACT averaging scheme." << std::endl; + idfx::cout << "ConstrainedTransport: Using UCT_CONTACT averaging scheme." << std::endl; break; case uct_hll: - idfx::cout << "ElectroMotiveForce: Using 2D-HLL averaging scheme." << std::endl; + idfx::cout << "ConstrainedTransport: Using 2D-HLL averaging scheme." << std::endl; break; case uct_hlld: - idfx::cout << "ElectroMotiveForce: Using 2D-HLLD averaging scheme." << std::endl; + idfx::cout << "ConstrainedTransport: Using 2D-HLLD averaging scheme." << std::endl; break; default: IDEFIX_ERROR("Unknown averaging scheme"); } } + +#include "calcCornerEmf.hpp" +#include "calcNonidealEMF.hpp" +#include "calcRiemannEmf.hpp" +#include "EMFexchange.hpp" +#include "enforceEMFBoundary.hpp" +#include "evolveMagField.hpp" +#include "evolveVectorPotential.hpp" + +#endif // FLUID_CONSTRAINEDTRANSPORT_CONSTRAINEDTRANSPORT_HPP_ diff --git a/src/hydro/electromotiveforce/enforceEMFBoundary.cpp b/src/fluid/constrainedTransport/enforceEMFBoundary.hpp similarity index 63% rename from src/hydro/electromotiveforce/enforceEMFBoundary.cpp rename to src/fluid/constrainedTransport/enforceEMFBoundary.hpp index 44ce2122..192ec12e 100644 --- a/src/hydro/electromotiveforce/enforceEMFBoundary.cpp +++ b/src/fluid/constrainedTransport/enforceEMFBoundary.hpp @@ -5,95 +5,113 @@ // Licensed under CeCILL 2.1 License, see COPYING for more information // *********************************************************************************** -#include "electroMotiveForce.hpp" -#include "hydro.hpp" +#ifndef FLUID_CONSTRAINEDTRANSPORT_ENFORCEEMFBOUNDARY_HPP_ +#define FLUID_CONSTRAINEDTRANSPORT_ENFORCEEMFBOUNDARY_HPP_ + +#include "constrainedTransport.hpp" +#include "fluid.hpp" #include "dataBlock.hpp" +#include "axis.hpp" + +// In some cases, to avoid the divergence of the normal field of cells that should be identical +// in periodic boundary conditions or with domain decomposition, it is needed to average or +// periodise the EMFs of corresponding cells, as proposed by Stone et al. 2020 (p4, top left col.) +// This is because in some specific cases involving curvilinear coordinates, roundoff errors +// can accumulate, leading to a small drift of face-values that should be strictly equal. +// This behaviour is enabled using the flag below. +//#define ENFORCE_EMF_CONSISTENCY -void ElectroMotiveForce::EnforceEMFBoundary() { +template +void ConstrainedTransport::EnforceEMFBoundary() { idfx::pushRegion("Emf::EnforceEMFBoundary"); #if MHD == YES - if(data->hydro.haveEmfBoundary) - this->data->hydro.emfBoundaryFunc(*data, data->t); + if(data->hydro->haveEmfBoundary) + this->data->hydro->emfBoundaryFunc(*data, data->t); - if(this->data->hydro.haveAxis) { - this->data->hydro.myAxis.RegularizeEMFs(); + if(this->data->hydro->haveAxis) { + this->data->hydro->boundary->axis->RegularizeEMFs(); } - #ifdef WITH_MPI - // This average the EMFs at the domain surface with immediate neighbours - // to ensure the EMFs exactly match - this->ExchangeAll(); + #ifdef ENFORCE_EMF_CONSISTENCY + #ifdef WITH_MPI + // This average the EMFs at the domain surface with immediate neighbours + // to ensure the EMFs exactly match + this->ExchangeAll(); + #endif #endif IdefixArray3D ex = this->ex; IdefixArray3D ey = this->ey; IdefixArray3D ez = this->ez; - // Ensure EMF are strictly periodic to avoid accumulation of roundoff errors + // Enforce specific EMF regularisation for(int dir=0 ; dir < DIMENSIONS ; dir++ ) { - if(data->lbound[dir] == periodic && data->rbound[dir] == periodic) { - // If domain decomposed, periodicity is already enforced by ExchangeAll - if(data->mygrid->nproc[dir] == 1) { - int ioffset = (dir == IDIR) ? data->np_int[IDIR] : 0; - int joffset = (dir == JDIR) ? data->np_int[JDIR] : 0; - int koffset = (dir == KDIR) ? data->np_int[KDIR] : 0; - - int ibeg = (dir == IDIR) ? data->beg[IDIR] : 0; - int iend = (dir == IDIR) ? data->beg[IDIR]+1 : data->np_tot[IDIR]; - int jbeg = (dir == JDIR) ? data->beg[JDIR] : 0; - int jend = (dir == JDIR) ? data->beg[JDIR]+1 : data->np_tot[JDIR]; - int kbeg = (dir == KDIR) ? data->beg[KDIR] : 0; - int kend = (dir == KDIR) ? data->beg[KDIR]+1 : data->np_tot[KDIR]; - idefix_for("BoundaryEMFPeriodic",kbeg,kend,jbeg,jend,ibeg,iend, - KOKKOS_LAMBDA (int k, int j, int i) { - real em; - - if(dir==IDIR) { - em = HALF_F*(ez(k,j,i)+ez(k,j,i+ioffset)); - ez(k,j,i) = em; - ez(k,j,i+ioffset) = em; - - #if DIMENSIONS == 3 - em = HALF_F*(ey(k,j,i)+ey(k,j,i+ioffset)); - ey(k,j,i) = em; - ey(k,j,i+ioffset) = em; - #endif - } - - if(dir==JDIR) { - em = HALF_F*(ez(k,j,i)+ez(k,j+joffset,i)); - ez(k,j,i) = em; - ez(k,j+joffset,i) = em; - - #if DIMENSIONS == 3 - em = HALF_F*(ex(k,j,i)+ex(k,j+joffset,i)); - ex(k,j,i) = em; - ex(k,j+joffset,i) = em; - #endif - } - - if(dir==KDIR) { - em = HALF_F*(ex(k,j,i)+ex(k+koffset,j,i)); - ex(k,j,i) = em; - ex(k+koffset,j,i) = em; - - em = HALF_F*(ey(k,j,i)+ey(k+koffset,j,i)); - ey(k,j,i) = em; - ey(k+koffset,j,i) = em; - } - }); - } - } if(data->lbound[dir] == shearingbox || data->rbound[dir] == shearingbox) { SymmetrizeEMFShearingBox(); } + #ifdef ENFORCE_EMF_CONSISTENCY + if(data->lbound[dir] == periodic && data->rbound[dir] == periodic) { + // If domain decomposed, periodicity is already enforced by ExchangeAll + if(data->mygrid->nproc[dir] == 1) { + int ioffset = (dir == IDIR) ? data->np_int[IDIR] : 0; + int joffset = (dir == JDIR) ? data->np_int[JDIR] : 0; + int koffset = (dir == KDIR) ? data->np_int[KDIR] : 0; + + int ibeg = (dir == IDIR) ? data->beg[IDIR] : 0; + int iend = (dir == IDIR) ? data->beg[IDIR]+1 : data->np_tot[IDIR]; + int jbeg = (dir == JDIR) ? data->beg[JDIR] : 0; + int jend = (dir == JDIR) ? data->beg[JDIR]+1 : data->np_tot[JDIR]; + int kbeg = (dir == KDIR) ? data->beg[KDIR] : 0; + int kend = (dir == KDIR) ? data->beg[KDIR]+1 : data->np_tot[KDIR]; + idefix_for("BoundaryEMFPeriodic",kbeg,kend,jbeg,jend,ibeg,iend, + KOKKOS_LAMBDA (int k, int j, int i) { + real em; + + if(dir==IDIR) { + em = HALF_F*(ez(k,j,i)+ez(k,j,i+ioffset)); + ez(k,j,i) = em; + ez(k,j,i+ioffset) = em; + + #if DIMENSIONS == 3 + em = HALF_F*(ey(k,j,i)+ey(k,j,i+ioffset)); + ey(k,j,i) = em; + ey(k,j,i+ioffset) = em; + #endif + } + + if(dir==JDIR) { + em = HALF_F*(ez(k,j,i)+ez(k,j+joffset,i)); + ez(k,j,i) = em; + ez(k,j+joffset,i) = em; + + #if DIMENSIONS == 3 + em = HALF_F*(ex(k,j,i)+ex(k,j+joffset,i)); + ex(k,j,i) = em; + ex(k,j+joffset,i) = em; + #endif + } + + if(dir==KDIR) { + em = HALF_F*(ex(k,j,i)+ex(k+koffset,j,i)); + ex(k,j,i) = em; + ex(k+koffset,j,i) = em; + + em = HALF_F*(ey(k,j,i)+ey(k+koffset,j,i)); + ey(k,j,i) = em; + ey(k+koffset,j,i) = em; + } + }); + } + } + #endif //ENFORCE_EMF_CONSISTENCY } #endif // MHD==YES idfx::popRegion(); } -void ElectroMotiveForce::SymmetrizeEMFShearingBox() { +template +void ConstrainedTransport::SymmetrizeEMFShearingBox() { idfx::pushRegion("Emf::EnforceEMFBoundary"); #if MHD == YES @@ -170,7 +188,8 @@ void ElectroMotiveForce::SymmetrizeEMFShearingBox() { } -void ElectroMotiveForce::ExtrapolateEMFShearingBox(BoundarySide side, +template +void ConstrainedTransport::ExtrapolateEMFShearingBox(BoundarySide side, IdefixArray2D Ein, IdefixArray2D Eout) { const int nxi = data->np_int[IDIR]; @@ -213,7 +232,7 @@ void ElectroMotiveForce::ExtrapolateEMFShearingBox(BoundarySide side, const int jom2 = jghost + ((jo-2-jghost)%nxj+nxj)%nxj; // Define Left and right fluxes - // Fluxes are defined from slop-limited interpolation + // Fluxes are defined from slope-limited interpolation // Using Van-leer slope limiter (consistently with the main advection scheme) real Fl,Fr; real dqm, dqp, dq; @@ -248,3 +267,4 @@ void ElectroMotiveForce::ExtrapolateEMFShearingBox(BoundarySide side, Eout(k,j) = Ein(k,jo) - eps*(Fr - Fl); }); } +#endif // FLUID_CONSTRAINEDTRANSPORT_ENFORCEEMFBOUNDARY_HPP_ diff --git a/src/hydro/electromotiveforce/evolveMagField.cpp b/src/fluid/constrainedTransport/evolveMagField.hpp similarity index 92% rename from src/hydro/electromotiveforce/evolveMagField.cpp rename to src/fluid/constrainedTransport/evolveMagField.hpp index 8be30ee8..01aa8a6e 100644 --- a/src/hydro/electromotiveforce/evolveMagField.cpp +++ b/src/fluid/constrainedTransport/evolveMagField.hpp @@ -4,13 +4,16 @@ // and other code contributors // Licensed under CeCILL 2.1 License, see COPYING for more information // *********************************************************************************** +#ifndef FLUID_CONSTRAINEDTRANSPORT_EVOLVEMAGFIELD_HPP_ +#define FLUID_CONSTRAINEDTRANSPORT_EVOLVEMAGFIELD_HPP_ -#include "hydro.hpp" +#include "fluid.hpp" #include "dataBlock.hpp" // Evolve the magnetic field in Vs according to Constranied transport -void ElectroMotiveForce::EvolveMagField(real t, real dt, IdefixArray4D &Vsin) { - idfx::pushRegion("ElectroMotiveForce::EvolveMagField"); +template +void ConstrainedTransport::EvolveMagField(real t, real dt, IdefixArray4D &Vsin) { + idfx::pushRegion("ConstrainedTransport::EvolveMagField"); #if MHD == YES // Corned EMFs IdefixArray3D Ex1 = this->ex; @@ -130,3 +133,5 @@ void ElectroMotiveForce::EvolveMagField(real t, real dt, IdefixArray4D &Vs #endif idfx::popRegion(); } + +#endif //FLUID_CONSTRAINEDTRANSPORT_EVOLVEMAGFIELD_HPP_ diff --git a/src/hydro/electromotiveforce/evolveVectorPotential.cpp b/src/fluid/constrainedTransport/evolveVectorPotential.hpp similarity index 90% rename from src/hydro/electromotiveforce/evolveVectorPotential.cpp rename to src/fluid/constrainedTransport/evolveVectorPotential.hpp index dee21534..0887d2e1 100644 --- a/src/hydro/electromotiveforce/evolveVectorPotential.cpp +++ b/src/fluid/constrainedTransport/evolveVectorPotential.hpp @@ -5,15 +5,19 @@ // Licensed under CeCILL 2.1 License, see COPYING for more information // *********************************************************************************** +#ifndef FLUID_CONSTRAINEDTRANSPORT_EVOLVEVECTORPOTENTIAL_HPP_ +#define FLUID_CONSTRAINEDTRANSPORT_EVOLVEVECTORPOTENTIAL_HPP_ + #include #include "idefix.hpp" -#include "hydro.hpp" +#include "fluid.hpp" #include "dataBlock.hpp" -void ElectroMotiveForce::EvolveVectorPotential(real dt, IdefixArray4D &Vein) { +template +void ConstrainedTransport::EvolveVectorPotential(real dt, IdefixArray4D &Vein) { #ifdef EVOLVE_VECTOR_POTENTIAL - idfx::pushRegion("ElectroMotiveForce::EvolveVectorPotential"); + idfx::pushRegion("ConstrainedTransport::EvolveVectorPotential"); IdefixArray4D Ve = Vein; // Corned EMFs IdefixArray3D Ex1 = this->ex; @@ -39,10 +43,11 @@ void ElectroMotiveForce::EvolveVectorPotential(real dt, IdefixArray4D &Vei -void ElectroMotiveForce::ComputeMagFieldFromA(IdefixArray4D &Vein, +template +void ConstrainedTransport::ComputeMagFieldFromA(IdefixArray4D &Vein, IdefixArray4D &Vsout) { #ifdef EVOLVE_VECTOR_POTENTIAL - idfx::pushRegion("ElectroMotiveForce::ComputeMagFieldfromA"); + idfx::pushRegion("ConstrainedTransport::ComputeMagFieldfromA"); // Corned EMFs IdefixArray3D Ex1 = this->ex; @@ -157,3 +162,4 @@ void ElectroMotiveForce::ComputeMagFieldFromA(IdefixArray4D &Vein, idfx::popRegion(); #endif } +#endif //FLUID_CONSTRAINEDTRANSPORT_EVOLVEVECTORPOTENTIAL_HPP_ diff --git a/src/fluid/convertConsToPrim.hpp b/src/fluid/convertConsToPrim.hpp new file mode 100644 index 00000000..45607b86 --- /dev/null +++ b/src/fluid/convertConsToPrim.hpp @@ -0,0 +1,191 @@ +// *********************************************************************************** +// Idefix MHD astrophysical code +// Copyright(C) Geoffroy R. J. Lesur +// and other code contributors +// Licensed under CeCILL 2.1 License, see COPYING for more information +// *********************************************************************************** + +#ifndef FLUID_CONVERTCONSTOPRIM_HPP_ +#define FLUID_CONVERTCONSTOPRIM_HPP_ + +#include "fluid.hpp" +#include "dataBlock.hpp" +#include "tracer.hpp" + +template +KOKKOS_INLINE_FUNCTION void K_ConsToPrim(real Vc[], real Uc[], const EquationOfState *eos) { + Vc[RHO] = Uc[RHO]; + + EXPAND( Vc[VX1] = Uc[MX1]/Uc[RHO]; , + Vc[VX2] = Uc[MX2]/Uc[RHO]; , + Vc[VX3] = Uc[MX3]/Uc[RHO]; ) + + if constexpr(Phys::mhd) { + EXPAND( Vc[BX1] = Uc[BX1]; , + Vc[BX2] = Uc[BX2]; , + Vc[BX3] = Uc[BX3]; ) + } + + + if constexpr(Phys::pressure) { + real kin = HALF_F / Uc[RHO] * (EXPAND( Uc[MX1]*Uc[MX1] , + + Uc[MX2]*Uc[MX2] , + + Uc[MX3]*Uc[MX3] )); + + if constexpr(Phys::mhd) { + real mag = HALF_F * (EXPAND( Uc[BX1]*Uc[BX1] , + + Uc[BX2]*Uc[BX2] , + + Uc[BX3]*Uc[BX3] )); + + Vc[PRS] = eos->GetPressure(Uc[ENG] - kin - mag, Uc[RHO]); + + // Check pressure positivity + if(Vc[PRS]<= ZERO_F) { + #ifdef SMALL_PRESSURE_TEMPERATURE + Vc[PRS] = SMALL_PRESSURE_TEMPERATURE*Vc[RHO]; + #else + Vc[PRS] = SMALL_PRESSURE_FIX; + #endif + + Uc[ENG] = eos->GetInternalEnergy(Vc[PRS],Vc[RHO]) + kin + mag; + } + + } else { // Hydro case + Vc[PRS] = eos->GetPressure(Uc[ENG] - kin, Uc[RHO]); + // Check pressure positivity + if(Vc[PRS]<= ZERO_F) { + #ifdef SMALL_PRESSURE_TEMPERATURE + Vc[PRS] = SMALL_PRESSURE_TEMPERATURE*Vc[RHO]; + #else + Vc[PRS] = SMALL_PRESSURE_FIX; + #endif + + Uc[ENG] = eos->GetInternalEnergy(Vc[PRS],Vc[RHO]) + kin; + } + } // MHD + } // Have Energy +} + +template +KOKKOS_INLINE_FUNCTION void K_PrimToCons(real Uc[], real Vc[], const EquationOfState *eos) { + Uc[RHO] = Vc[RHO]; + + EXPAND( Uc[MX1] = Vc[VX1]*Vc[RHO]; , + Uc[MX2] = Vc[VX2]*Vc[RHO]; , + Uc[MX3] = Vc[VX3]*Vc[RHO]; ) + + if constexpr(Phys::mhd) { + EXPAND( Uc[BX1] = Vc[BX1]; , + Uc[BX2] = Vc[BX2]; , + Uc[BX3] = Vc[BX3]; ) + } + + + if constexpr(Phys::pressure) { + if constexpr(Phys::mhd) { + Uc[ENG] = eos->GetInternalEnergy(Vc[PRS],Vc[RHO]) + + HALF_F * Vc[RHO] * (EXPAND( Vc[VX1]*Vc[VX1] , + + Vc[VX2]*Vc[VX2] , + + Vc[VX3]*Vc[VX3] )) + + HALF_F * (EXPAND( Uc[BX1]*Uc[BX1] , + + Uc[BX2]*Uc[BX2] , + + Uc[BX3]*Uc[BX3] )); + } else { + Uc[ENG] = eos->GetInternalEnergy(Vc[PRS],Vc[RHO]) + + HALF_F * Vc[RHO] * (EXPAND( Vc[VX1]*Vc[VX1] , + + Vc[VX2]*Vc[VX2] , + + Vc[VX3]*Vc[VX3] )); + } //MHD + } // Energy +} + + + +// Convect Conservative to Primitive variable +template +void Fluid::ConvertConsToPrim() { + idfx::pushRegion("Fluid::ConvertConsToPrim"); + + IdefixArray4D Vc = this->Vc; + IdefixArray4D Uc = this->Uc; + EquationOfState eos; + if constexpr(Phys::eos) { + eos = *(this->eos.get()); + } + + if constexpr(Phys::mhd) { + #ifdef EVOLVE_VECTOR_POTENTIAL + emf->ComputeMagFieldFromA(Ve,Vs); + #endif + boundary->ReconstructVcField(Uc); + } + + idefix_for("ConsToPrim", + 0,data->np_tot[KDIR], + 0,data->np_tot[JDIR], + 0,data->np_tot[IDIR], + KOKKOS_LAMBDA (int k, int j, int i) { + real U[Phys::nvar]; + real V[Phys::nvar]; + +#pragma unroll + for(int nv = 0 ; nv < Phys::nvar; nv++) { + U[nv] = Uc(nv,k,j,i); + } + + K_ConsToPrim(V,U,&eos); + +#pragma unroll + for(int nv = 0 ; nvConvertConsToPrim(); + } + + idfx::popRegion(); +} + +// Convert Primitive to conservative variables +template +void Fluid::ConvertPrimToCons() { + idfx::pushRegion("Fluid::ConvertPrimToCons"); + + IdefixArray4D Vc = this->Vc; + IdefixArray4D Uc = this->Uc; + EquationOfState eos; + if constexpr(Phys::eos) { + eos = *(this->eos.get()); + } + + idefix_for("ConvertPrimToCons", + 0,data->np_tot[KDIR], + 0,data->np_tot[JDIR], + 0,data->np_tot[IDIR], + KOKKOS_LAMBDA (int k, int j, int i) { + real U[Phys::nvar]; + real V[Phys::nvar]; + +#pragma unroll + for(int nv = 0 ; nv < Phys::nvar; nv++) { + V[nv] = Vc(nv,k,j,i); + } + + K_PrimToCons(U,V,&eos); + +#pragma unroll + for(int nv = 0 ; nvConvertPrimToCons(); + } + + idfx::popRegion(); +} + +#endif //FLUID_CONVERTCONSTOPRIM_HPP_ diff --git a/src/fluid/drag.cpp b/src/fluid/drag.cpp new file mode 100644 index 00000000..d24e0bea --- /dev/null +++ b/src/fluid/drag.cpp @@ -0,0 +1,112 @@ +// *********************************************************************************** +// Idefix MHD astrophysical code +// Copyright(C) Geoffroy R. J. Lesur +// and other code contributors +// Licensed under CeCILL 2.1 License, see COPYING for more information +// *********************************************************************************** +#include "drag.hpp" +#include "physics.hpp" +void Drag::AddDragForce(const real dt) { + idfx::pushRegion("Drag::AddDragForce"); + + auto UcGas = this->UcGas; + auto VcGas = this->VcGas; + auto UcDust = this->UcDust; + auto VcDust = this->VcDust; + auto InvDt = this->InvDt; + + const Type type = this->type; + real dragCoeff = this->dragCoeff; + bool feedback = this->feedback; + + EquationOfState eos = *(this->eos); + + auto userGammai = this->gammai; + if(type == Type::Userdef) { + if(userDrag != NULL) { + idfx::pushRegion("Drag::UserDrag"); + userDrag(data, dragCoeff, userGammai); + idfx::popRegion(); + } else { + IDEFIX_ERROR("No User-defined drag function has been enrolled"); + } + } + // Compute a drag force fd = - gamma*rhod*rhog*(vd-vg) + // Where gamma is computed according to the choice of drag type + idefix_for("DragForce",0,data->np_tot[KDIR],0,data->np_tot[JDIR],0,data->np_tot[IDIR], + KOKKOS_LAMBDA (int k, int j, int i) { + real gamma; // The drag coefficient + if(type == Type::Gamma) { + gamma = dragCoeff; + + } else if(type == Type::Tau) { + // In this case, the coefficient is the stopping time (assumed constant) + gamma = 1/(dragCoeff*VcGas(RHO,k,j,i)); + } else if(type == Type::Size) { + real cs; + // Assume a fixed size, hence for both Epstein or Stokes, gamma~1/rho_g/cs + // Get the sound speed + #if HAVE_ENERGY == 1 + cs = std::sqrt(eos.GetGamma(VcGas(PRS,k,j,i),VcGas(RHO,k,j,i) + *VcGas(PRS,k,j,i)/VcGas(RHO,k,j,i))); + #else + cs = eos.GetWaveSpeed(k,j,i); + #endif + gamma = cs/dragCoeff; + } else if(type == Type::Userdef) { + gamma = userGammai(k,j,i); + } + + real dp = dt * gamma * VcDust(RHO,k,j,i) * VcGas(RHO,k,j,i); + for(int n = MX1 ; n < MX1+COMPONENTS ; n++) { + real dv = VcDust(n,k,j,i) - VcGas(n,k,j,i); + UcDust(n,k,j,i) -= dp*dv; + if(feedback) UcGas(n,k,j,i) += dp*dv; + #if HAVE_ENERGY == 1 + // We add back the energy dissipated for the dust which is not accounted for + // (since there is no energy equation for dust grains) + + // TODO(GL): this should be disabled in the case of a true multifluid system where + // both fluids have a proper energy equation + UcGas(ENG,k,j,i) += dp*dv*VcDust(n,k,j,i); + #endif + } + // Cfl constraint + real idt = gamma*VcGas(RHO,k,j,i); + if(feedback) idt += gamma*VcDust(RHO,k,j,i); + InvDt(k,j,i) += idt; + }); + idfx::popRegion(); +} + +void Drag::ShowConfig() { + idfx::cout << "Drag: Using "; + switch(type) { + case Type::Gamma: + idfx::cout << "constant gamma"; + break; + case Type::Tau: + idfx::cout << "constant stopping time"; + break; + case Type::Size: + idfx::cout << "constant dust size"; + break; + case Type::Userdef: + idfx::cout << "user-defined"; + break; + } + + idfx::cout << " drag law"; + if(feedback) { + idfx::cout << " with feedback." << std::endl; + } else { + idfx::cout << " without feedback." << std::endl; + } +} + +void Drag::EnrollUserDrag(UserDefDragFunc func) { + if(type != Type::Userdef) { + IDEFIX_ERROR("User-defined drag function requires drag entry to be set to \"userdef\""); + } + this->userDrag = func; +} diff --git a/src/fluid/drag.hpp b/src/fluid/drag.hpp new file mode 100644 index 00000000..d5f7a5a7 --- /dev/null +++ b/src/fluid/drag.hpp @@ -0,0 +1,107 @@ +// *********************************************************************************** +// Idefix MHD astrophysical code +// Copyright(C) Geoffroy R. J. Lesur +// and other code contributors +// Licensed under CeCILL 2.1 License, see COPYING for more information +// *********************************************************************************** +#ifndef FLUID_DRAG_HPP_ +#define FLUID_DRAG_HPP_ + +#include +#include "idefix.hpp" +#include "input.hpp" +#include "fluid_defs.hpp" +#include "eos.hpp" + +using UserDefDragFunc = void (*) (DataBlock *, real beta, IdefixArray3D &gammai); + + +class Drag { + public: + enum class Type{Gamma, Tau, Size, Userdef}; + // Different types of implementation for the drag force. + template + Drag(Input &, Fluid *); + void ShowConfig(); // print configuration + void AddDragForce(const real); + void EnrollUserDrag(UserDefDragFunc); // User defined drag function enrollment + + IdefixArray4D UcDust; // Dust conservative quantities + IdefixArray4D UcGas; // Gas conservative quantities + IdefixArray4D VcDust; // Gas primitive quantities + IdefixArray4D VcGas; // Gas primitive quantities + IdefixArray3D InvDt; // The InvDt of current dust specie + IdefixArray3D gammai; // the drag coefficient (only used for user-defined dust grains) + Type type; + + private: + DataBlock* data; + real dragCoeff; + bool feedback{false}; + + UserDefDragFunc userDrag{NULL}; + + // Sound speed computation + EquationOfState *eos; +}; + +#include "fluid.hpp" + +template +Drag::Drag(Input &input, Fluid *hydroin): + UcDust{hydroin->Uc}, + UcGas{hydroin->data->hydro->Uc}, + VcDust{hydroin->Vc}, + VcGas{hydroin->data->hydro->Vc}, + InvDt{hydroin->InvDt} { + idfx::pushRegion("Drag::Drag"); + // Save the parent hydro object + + this->data = hydroin->data; + // We use the EOS of the basic fluid instance, as there is no EOS for dust! + this->eos = hydroin->data->hydro->eos.get(); + + // Check in which block we should fetch our information + std::string BlockName; + if(Phys::dust) { + BlockName = "Dust"; + } else { + IDEFIX_ERROR("Drag is currently implemented only for dusty fuids"); + } + + if(input.CheckEntry(BlockName,"drag")>=0) { + std::string dragType = input.Get(BlockName,"drag",0); + if(dragType.compare("gamma") == 0) { + this->type = Type::Gamma; + } else if(dragType.compare("tau") == 0) { + this->type = Type::Tau; + } else if(dragType.compare("size") == 0) { + this->type = Type::Size; + } else if(dragType.compare("userdef") == 0) { + this->type = Type::Userdef; + this->gammai = IdefixArray3D("UserDrag", + data->np_tot[KDIR], + data->np_tot[JDIR], + data->np_tot[IDIR]); + } else { + std::stringstream msg; + msg << "Unknown drag type \"" << dragType + << "\" in your input file." << std::endl + << "Allowed values are: gamma, tau, epstein, stokes, userdef." << std::endl; + + IDEFIX_ERROR(msg); + } + // Fetch the drag coefficient for the current specie. + const int n = hydroin->instanceNumber; + this->dragCoeff = input.Get(BlockName,"drag",n+1); + + // Feedback is true by default, but can be switched off. + this->feedback = input.GetOrSet(BlockName,"drag_feedback",0,true); + } else { + IDEFIX_ERROR("A [Drag] block is required in your input file to define the drag force."); + } + + + idfx::popRegion(); +} +#endif // FLUID_DRAG_HPP_ diff --git a/src/fluid/enroll.hpp b/src/fluid/enroll.hpp new file mode 100644 index 00000000..77e8876f --- /dev/null +++ b/src/fluid/enroll.hpp @@ -0,0 +1,131 @@ +// *********************************************************************************** +// Idefix MHD astrophysical code +// Copyright(C) Geoffroy R. J. Lesur +// and other code contributors +// Licensed under CeCILL 2.1 License, see COPYING for more information +// *********************************************************************************** + +#ifndef FLUID_ENROLL_HPP_ +#define FLUID_ENROLL_HPP_ + +#include "dataBlock.hpp" +template +void Fluid::EnrollIsoSoundSpeed(IsoSoundSpeedFunc myFunc) { + if constexpr(!Phys::isothermal) { + IDEFIX_ERROR("Isothermal sound speed enrollment requires ISOTHERMAL to be defined in" + "definitions.hpp"); + } else { + #ifdef ISOTHERMAL + eos->EnrollIsoSoundSpeed(myFunc); + #endif + } +} + +template +template +void Fluid::EnrollUserDefBoundary(T myFunc) { + // This is a proxy for userdef enrollment + boundary->EnrollUserDefBoundary(myFunc); +} + +template +template +void Fluid::EnrollInternalBoundary(T myFunc) { + // This is a proxy for internal boundary enrollment + boundary->EnrollInternalBoundary(myFunc); +} + +template +template +void Fluid::EnrollFluxBoundary(T myFunc) { + // This is a proxy for userdef enrollment + boundary->EnrollFluxBoundary(myFunc); +} + +template +void Fluid::EnrollUserSourceTerm(SrcTermFunc myFunc) { + this->userSourceTerm = myFunc; + this->haveUserSourceTerm = true; + this->haveSourceTerms = true; +} + +// Deprecated enrollment function +template +void Fluid::EnrollUserSourceTerm(SrcTermFuncOld myFunc) { + std::stringstream msg; + msg << "The old signature for user-defined source terms " << std::endl + << "(DataBlock &, const real t, const real dt)" << std::endl + << "is deprecated. You should now use "<< std::endl + << "(Fluid *, const real t, const real dt)" << std::endl + << "With the Phys of your choice (DefaultPhysics, DustPhysics...)" << std::endl; + + IDEFIX_WARNING(msg); + + this->userSourceTermOld = myFunc; + this->haveUserSourceTerm = true; + this->haveSourceTerms = true; +} + + +template +void Fluid::EnrollEmfBoundary(EmfBoundaryFunc myFunc) { + if constexpr(!Phys::mhd) { + IDEFIX_ERROR("This function can only be used with the MHD solver."); + } + this->emfBoundaryFunc = myFunc; + this->haveEmfBoundary = true; +} + +template +void Fluid::EnrollOhmicDiffusivity(DiffusivityFunc myFunc) { + if constexpr(!Phys::mhd) { + IDEFIX_ERROR("This function can only be used with the MHD solver."); + } + if(this->resistivityStatus.status < UserDefFunction) { + IDEFIX_WARNING("Ohmic diffusivity enrollment requires Hydro/Resistivity " + "to be set to userdef in .ini file"); + } + this->ohmicDiffusivityFunc = myFunc; +} + +template +void Fluid::EnrollAmbipolarDiffusivity(DiffusivityFunc myFunc) { + if constexpr(!Phys::mhd) { + IDEFIX_ERROR("This function can only be used with the MHD solver."); + } + if(this->ambipolarStatus.status < UserDefFunction) { + IDEFIX_WARNING("Ambipolar diffusivity enrollment requires Hydro/Ambipolar " + "to be set to userdef in .ini file"); + } + this->ambipolarDiffusivityFunc = myFunc; +} + +template +void Fluid::EnrollHallDiffusivity(DiffusivityFunc myFunc) { + if constexpr(!Phys::mhd) { + IDEFIX_ERROR("This function can only be used with the MHD solver."); + } + if(this->hallStatus.status < UserDefFunction) { + IDEFIX_WARNING("Hall diffusivity enrollment requires Hydro/Hall " + "to be set to userdef in .ini file"); + } + this->hallDiffusivityFunc = myFunc; +} + +template +void Fluid::ResetStage() { + // Reset variables required at the beginning of each stage + // (essentially linked to timestep evaluation) + idfx::pushRegion("Fluid::ResetStage"); + + IdefixArray3D InvDt=this->InvDt; + + idefix_for("HydroResetStage",0,data->np_tot[KDIR],0,data->np_tot[JDIR],0,data->np_tot[IDIR], + KOKKOS_LAMBDA (int k, int j, int i) { + InvDt(k,j,i) = ZERO_F; + }); + + idfx::popRegion(); +} + +#endif //FLUID_ENROLL_HPP_ diff --git a/src/fluid/eos/CMakeLists.txt b/src/fluid/eos/CMakeLists.txt new file mode 100644 index 00000000..f939d9d5 --- /dev/null +++ b/src/fluid/eos/CMakeLists.txt @@ -0,0 +1,5 @@ +target_sources(idefix + PUBLIC ${CMAKE_CURRENT_LIST_DIR}/eos_adiabatic.hpp + PUBLIC ${CMAKE_CURRENT_LIST_DIR}/eos_isothermal.hpp + PUBLIC ${CMAKE_CURRENT_LIST_DIR}/eos.hpp + ) diff --git a/src/hydro/shockFlattening.hpp b/src/fluid/eos/eos.hpp similarity index 50% rename from src/hydro/shockFlattening.hpp rename to src/fluid/eos/eos.hpp index dabb1fdc..7e0a490a 100644 --- a/src/hydro/shockFlattening.hpp +++ b/src/fluid/eos/eos.hpp @@ -5,23 +5,20 @@ // Licensed under CeCILL 2.1 License, see COPYING for more information // *********************************************************************************** -#ifndef HYDRO_SHOCKFLATTENING_HPP_ -#define HYDRO_SHOCKFLATTENING_HPP_ +#ifndef FLUID_EOS_EOS_HPP_ +#define FLUID_EOS_EOS_HPP_ -#include "hydro.hpp" +// This is a wrapper which decides which eos .hpp file should be chosen according to +// Idefix configuration -enum class FlagShock{None, Shock}; +#ifndef EOS_FILE + #ifdef ISOTHERMAL + #include "eos_isothermal.hpp" + #else + #include "eos_adiabatic.hpp" + #endif +#else + #include EOS_FILE +#endif -class ShockFlattening { - public: - ShockFlattening(Hydro*, real); - ShockFlattening() {} - - void FindShock(); - - Hydro *hydro; - IdefixArray3D flagArray; - bool isActive{false}; - real smoothing{0}; -}; -#endif //HYDRO_SHOCKFLATTENING_HPP_ +#endif // FLUID_EOS_EOS_HPP_ diff --git a/src/fluid/eos/eos_adiabatic.hpp b/src/fluid/eos/eos_adiabatic.hpp new file mode 100644 index 00000000..88d4f0f6 --- /dev/null +++ b/src/fluid/eos/eos_adiabatic.hpp @@ -0,0 +1,50 @@ +// *********************************************************************************** +// Idefix MHD astrophysical code +// Copyright(C) Geoffroy R. J. Lesur +// and other code contributors +// Licensed under CeCILL 2.1 License, see COPYING for more information +// *********************************************************************************** + +#ifndef FLUID_EOS_EOS_ADIABATIC_HPP_ +#define FLUID_EOS_EOS_ADIABATIC_HPP_ + +#include +#include "idefix.hpp" +#include "input.hpp" + +// This is the adiabatic (constant gamma) implementation of the equation of state +class EquationOfState { + public: + EquationOfState() = default; + + EquationOfState(Input & input, DataBlock *, std::string prefix) { + this->gamma = input.GetOrSet(prefix,"gamma",0, 5.0/3.0); + } + + void ShowConfig() { + idfx::cout << "EquationOfState: ideal with gamma=" << this->gamma << std::endl; + } + + // First adiabatic exponent. In the ideal EOS, gamma does not depend on the gas state, + // So we add default values to 0 here so that GetGamma can be called without any argument + KOKKOS_INLINE_FUNCTION real GetGamma(real P = 0.0, real rho = 0.0) const {return gamma;} + void Refresh(DataBlock &, real) {} // Refresh the eos (recompute coefficients and tables) + + KOKKOS_INLINE_FUNCTION + real GetWaveSpeed(int k, int j, int i) const { + Kokkos::abort("GetWaveSpeed should be used only for isothermal EOS"); + return 0; + } + KOKKOS_INLINE_FUNCTION + real GetInternalEnergy(real P, real rho) const { + return P/(gamma-1.0); + } + KOKKOS_INLINE_FUNCTION + real GetPressure(real Eint, real rho) const { + return Eint * (gamma-1.0); + } + + private: + real gamma; +}; +#endif // FLUID_EOS_EOS_ADIABATIC_HPP_ diff --git a/src/fluid/eos/eos_isothermal.hpp b/src/fluid/eos/eos_isothermal.hpp new file mode 100644 index 00000000..04efc9b2 --- /dev/null +++ b/src/fluid/eos/eos_isothermal.hpp @@ -0,0 +1,100 @@ +// *********************************************************************************** +// Idefix MHD astrophysical code +// Copyright(C) Geoffroy R. J. Lesur +// and other code contributors +// Licensed under CeCILL 2.1 License, see COPYING for more information +// *********************************************************************************** + +#ifndef FLUID_EOS_EOS_ISOTHERMAL_HPP_ +#define FLUID_EOS_EOS_ISOTHERMAL_HPP_ + +#include +#include "idefix.hpp" +#include "input.hpp" +#include "fluid_defs.hpp" +#include "dataBlock.hpp" + + +// This is the isothermal implementation of the equation of state +class EquationOfState { + public: + EquationOfState() = default; + + EquationOfState(Input & input, DataBlock *data, std::string prefix) { + std::string isoString = input.Get(prefix,"csiso",0); + if(isoString.compare("constant") == 0) { + this->haveIsoSoundSpeed = Constant; + this->isoSoundSpeed = input.Get(prefix,"csiso",1); + } else if(isoString.compare("userdef") == 0) { + this->haveIsoSoundSpeed = UserDefFunction; + this->isoSoundSpeedArray = IdefixArray3D(prefix+"_csIso", + data->np_tot[KDIR], data->np_tot[JDIR], data->np_tot[IDIR]); + } else { + IDEFIX_ERROR("csiso admits only constant or userdef entries"); + } + } + + void ShowConfig() { + if(haveIsoSoundSpeed == Constant) { + idfx::cout << "EquationOfState: isothermal with cs=" << isoSoundSpeed << "." + << std::endl; + } else if(haveIsoSoundSpeed == UserDefFunction) { + idfx::cout << "EquationOfState: isothermal with user-defined cs function." + << std::endl; + if(!isoSoundSpeedFunc) { + IDEFIX_ERROR("No user-defined isothermal sound speed function has been enrolled."); + } + } + } + + // In the isothermal EOS, gamma does not depend on the gas state, + // So we add default values to 0 here so that GetGamma can be called without any argument + KOKKOS_INLINE_FUNCTION real GetGamma(real P = 0.0, real rho = 0.0) const { + return 1.0; + } + + void Refresh(DataBlock &data, real t) { // Refresh the coefficients (and tables) + idfx::pushRegion("EquationOfState::Refresh"); + if(haveIsoSoundSpeed == UserDefFunction) { + if(isoSoundSpeedFunc) { + idfx::pushRegion("EquationOfState::UserDefSoundSpeed"); + isoSoundSpeedFunc(data, t, isoSoundSpeedArray); + idfx::popRegion(); + } else { + IDEFIX_ERROR("No user-defined isothermal sound speed function has been enrolled"); + } + } + idfx::popRegion(); + } + + KOKKOS_INLINE_FUNCTION real GetWaveSpeed(const int k, const int j, const int i) const { + if(haveIsoSoundSpeed == UserDefFunction) { + return isoSoundSpeedArray(k,j,i); + } else { + return isoSoundSpeed; + } + } + KOKKOS_INLINE_FUNCTION real GetInternalEnergy(real P, real rho) const { + Kokkos::abort("Internal energy is not defined in the isothermal EOS"); + } + KOKKOS_INLINE_FUNCTION real GetPressure(real Eint, real rho) const { + Kokkos::abort("Pressure is not defined in the isothermal EOS"); + } + + // Enroll user-defined isothermal sound speed + void EnrollIsoSoundSpeed(IsoSoundSpeedFunc func) { + if(this->haveIsoSoundSpeed != UserDefFunction) { + IDEFIX_WARNING("Isothermal sound speed enrollment requires Hydro/csiso " + " to be set to userdef in .ini file"); + } + this->isoSoundSpeedFunc = func; + } + + private: + real isoSoundSpeed; + HydroModuleStatus haveIsoSoundSpeed{Disabled}; + IdefixArray3D isoSoundSpeedArray; + IsoSoundSpeedFunc isoSoundSpeedFunc{NULL}; +}; + +#endif // FLUID_EOS_EOS_ISOTHERMAL_HPP_ diff --git a/src/fluid/eos/eos_template.hpp b/src/fluid/eos/eos_template.hpp new file mode 100644 index 00000000..3d615081 --- /dev/null +++ b/src/fluid/eos/eos_template.hpp @@ -0,0 +1,69 @@ +// *********************************************************************************** +// Idefix MHD astrophysical code +// Copyright(C) Geoffroy R. J. Lesur +// and other code contributors +// Licensed under CeCILL 2.1 License, see COPYING for more information +// *********************************************************************************** + +#ifndef FLUID_EOS_EOS_TEMPLATE_HPP_ +#define FLUID_EOS_EOS_TEMPLATE_HPP_ + +#include +#include "idefix.hpp" +#include "input.hpp" + +// This is a template for a generalized custom equation of state +class EquationOfState { + public: + EquationOfState() = default; + + // Add here the required steps to initialize your equation of state + EquationOfState(Input & input, DataBlock *, std::string prefix) { + // .... + } + + // Information message displayed before entering the main loop + void ShowConfig() { + idfx::cout << "EquationOfState: Custom." << std::endl; + } + + // First adiabatic exponent. + KOKKOS_INLINE_FUNCTION real GetGamma(real P , real rho ) const { + real gamma; + // Compute gamma (needed for sound speed estimations, 5/3 would work in general) + return gamma; + } + + // Refresh the eos (recompute coefficients and tables) + void Refresh(DataBlock &data, real t) { + // .... + } + + // This function is used only when the isothermal approximation is enabled. Not needed here + KOKKOS_INLINE_FUNCTION + real GetWaveSpeed(int k, int j, int i) const { + Kokkos::abort("GetWaveSpeed should be used only for isothermal EOS"); + return 0; + } + + // Compute the internal energy from pressure and density + KOKKOS_INLINE_FUNCTION + real GetInternalEnergy(real P, real rho) const { + real eint; // = ... + return eint; + } + + // Compute the pressure from internal energy and density + KOKKOS_INLINE_FUNCTION + real GetPressure(real Eint, real rho) const { + real P; // = ... + return P; + } + + private: + real gamma; + // Add here the internal variables required by your equation of state. +}; + + +#endif // FLUID_EOS_EOS_TEMPLATE_HPP_ diff --git a/src/fluid/evolveStage.hpp b/src/fluid/evolveStage.hpp new file mode 100644 index 00000000..06754d0b --- /dev/null +++ b/src/fluid/evolveStage.hpp @@ -0,0 +1,87 @@ +// *********************************************************************************** +// Idefix MHD astrophysical code +// Copyright(C) Geoffroy R. J. Lesur +// and other code contributors +// Licensed under CeCILL 2.1 License, see COPYING for more information +// *********************************************************************************** + +#ifndef FLUID_EVOLVESTAGE_HPP_ +#define FLUID_EVOLVESTAGE_HPP_ + +#include "fluid.hpp" +#include "riemannSolver.hpp" +template +template +void Fluid::LoopDir(const real t, const real dt) { + // Step 2: compute the intercell flux with our Riemann solver, store the resulting InvDt + this->rSolver->template CalcFlux(this->FluxRiemann); + + // Step 2.5: compute intercell parabolic flux when needed + if(haveExplicitParabolicTerms) CalcParabolicFlux(t); + + // If we have tracers, compute the tracer intercell flux + if(haveTracer) { + this->tracer->template CalcFlux(this->FluxRiemann); + } + + // Step 3: compute the resulting evolution of the conserved variables, stored in Uc + CalcRightHandSide(t,dt); + if(haveTracer) { + this->tracer->template CalcRightHandSide(this->FluxRiemann,t ,dt); + } + + // Recursive: do next dimension + if constexpr (dir+1 < DIMENSIONS) LoopDir(t, dt); +} + + + +// Evolve one step forward in time of hydro +template +void Fluid::EvolveStage(const real t, const real dt) { + idfx::pushRegion("Fluid::EvolveStage"); + // Compute current when needed + if(needExplicitCurrent) CalcCurrent(); + + if(hallStatus.status == UserDefFunction) { + if(hallDiffusivityFunc) + hallDiffusivityFunc(*data, t, xHall); + else + IDEFIX_ERROR("No user-defined Hall diffusivity function has been enrolled"); + } + + if constexpr(Phys::eos) { + eos->Refresh(*data, t); + } + + // Loop on all of the directions + LoopDir(t,dt); + + // Step 4: add source terms to the conserved variables (curvature, rotation, etc) + if(haveSourceTerms) AddSourceTerms(t, dt); + + // Step 5: add drag when needed + if(haveDrag) drag->AddDragForce(dt); + + if constexpr(Phys::mhd) { + #if DIMENSIONS >= 2 + // Compute the field evolution according to CT + emf->CalcCornerEMF(t); + if(resistivityStatus.isExplicit || ambipolarStatus.isExplicit) { + emf->CalcNonidealEMF(t); + } + emf->EnforceEMFBoundary(); + #ifdef EVOLVE_VECTOR_POTENTIAL + emf->EvolveVectorPotential(dt, Ve); + emf->ComputeMagFieldFromA(Ve, Vs); + #else + emf->EvolveMagField(t, dt, Vs); + #endif + + boundary->ReconstructVcField(Uc); + #endif + } + + idfx::popRegion(); +} +#endif //FLUID_EVOLVESTAGE_HPP_ diff --git a/src/fluid/fluid.hpp b/src/fluid/fluid.hpp new file mode 100644 index 00000000..ae809b48 --- /dev/null +++ b/src/fluid/fluid.hpp @@ -0,0 +1,764 @@ +// *********************************************************************************** +// Idefix MHD astrophysical code +// Copyright(C) Geoffroy R. J. Lesur +// and other code contributors +// Licensed under CeCILL 2.1 License, see COPYING for more information +// *********************************************************************************** + +#ifndef FLUID_FLUID_HPP_ +#define FLUID_FLUID_HPP_ + +#include +#include +#include + +#include "idefix.hpp" +#include "grid.hpp" +#include "fluid_defs.hpp" +#include "eos.hpp" +#include "thermalDiffusion.hpp" +#include "bragThermalDiffusion.hpp" +#include "selfGravity.hpp" +#include "vtk.hpp" +#include "dump.hpp" +#ifdef WITH_HDF5 +#include "xdmf.hpp" +#endif + +// forward class declaration +class DataBlock; +template +class Boundary; + +template +class ConstrainedTransport; + +template +class RKLegendre; + +template +class RiemannSolver; + +template +class ShockFlattening; + +class Viscosity; +class ThermalDiffusion; +class BragViscosity; +class BragThermalDiffusion; +class Drag; +class Tracer; + + +template +class Fluid { + public: + Fluid( Grid &, Input&, DataBlock *, int n = 0); + void ConvertConsToPrim(); + void ConvertPrimToCons(); + template void CalcParabolicFlux(const real); + template void AddNonIdealMHDFlux(const real); + template void CalcRightHandSide(real, real ); + void CalcCurrent(); + void AddSourceTerms(real, real ); + void CoarsenFlow(IdefixArray4D&); + void CoarsenMagField(IdefixArray4D&); + real CheckDivB(); + void EvolveStage(const real, const real); + void ResetStage(); + void ShowConfig(); + IdefixArray4D GetFlux() {return this->FluxRiemann;} + int CheckNan(); + + // Our boundary conditions + std::unique_ptr> boundary; + + // EOS + std::unique_ptr eos; + + // Source terms + bool haveSourceTerms{false}; + + // Parabolic terms + bool haveExplicitParabolicTerms{false}; + bool haveRKLParabolicTerms{false}; + + std::unique_ptr> rkl; + + // Current + bool haveCurrent{false}; + bool needExplicitCurrent{false}; + bool needRKLCurrent{false}; + + // Nonideal MHD effects coefficients + ParabolicModuleStatus resistivityStatus, ambipolarStatus, hallStatus; + + // Whether or not we have viscosity + ParabolicModuleStatus viscosityStatus; + + // Whether or not we have thermal diffusion + ParabolicModuleStatus thermalDiffusionStatus; + + // Viscosity object + std::unique_ptr viscosity; + + // Thermal Diffusion object + std::unique_ptr thermalDiffusion; + + // Whether or not we have braginskii viscosity + ParabolicModuleStatus bragViscosityStatus; + + // Whether or not we have braginskii thermal diffusion + ParabolicModuleStatus bragThermalDiffusionStatus; + + // BragViscosity object + std::unique_ptr bragViscosity; + + // Braginskii Thermal Diffusion object + std::unique_ptr bragThermalDiffusion; + + // Drag object + bool haveDrag{false}; + std::unique_ptr drag; + + // Whether or not we have to treat the axis + bool haveAxis{false}; + + // Rotation vector + bool haveRotation{false}; + real OmegaZ; + + bool haveShearingBox{false}; + // Shear rate for shearing box problems + real sbS; + // Box width for shearing box problems + real sbLx; + + // Tracers treatment + std::unique_ptr tracer; + bool haveTracer{false}; + int nTracer{0}; + + + // Enroll user-defined boundary conditions (proxies for boundary class functions) + template + void EnrollUserDefBoundary(T); + template + void EnrollInternalBoundary(T); + template + void EnrollFluxBoundary(T); + + void EnrollEmfBoundary(EmfBoundaryFunc); + + // Add some user source terms + void EnrollUserSourceTerm(SrcTermFunc); + void EnrollUserSourceTerm(SrcTermFuncOld); // Deprecated + + // Enroll user-defined ohmic, ambipolar and Hall diffusivities + void EnrollOhmicDiffusivity(DiffusivityFunc); + void EnrollAmbipolarDiffusivity(DiffusivityFunc); + void EnrollHallDiffusivity(DiffusivityFunc); + + // Enroll user-defined isothermal sound speed + void EnrollIsoSoundSpeed(IsoSoundSpeedFunc); + + + // Arrays required by the Hydro object + IdefixArray4D Vc; // Main cell-centered primitive variables index + IdefixArray4D Vs; // Main face-centered varariables + IdefixArray4D Ve; // Main edge-centered varariables (only when EVOLVE_VECTOR_POTENTIAL) + IdefixArray4D Uc; // Main cell-centered conservative variables + IdefixArray4D J; // Electrical current + // (only defined when non-ideal MHD effects are enabled) + + // Name of the fields (used in outputs) + std::vector VcName; + std::vector VsName; + std::vector VeName; + + // Storing all of the electromotive forces + std::unique_ptr> emf; + + // Required by time integrator + IdefixArray3D InvDt; + + IdefixArray4D FluxRiemann; + IdefixArray3D dMax; // Maximum diffusion speed + + std::unique_ptr> rSolver; + + DataBlock *data; + + // Data related to current instance of the Fluid object + std::string prefix; + int instanceNumber; + + private: + friend class ConstrainedTransport; + friend class Fargo; + friend class RKLegendre; + friend class Boundary; + friend class ShockFlattening; + friend class RiemannSolver; + friend class Viscosity; + friend class ThermalDiffusion; + friend class BragViscosity; + friend class BragThermalDiffusion; + friend class Drag; + + template + friend struct Fluid_AddSourceTermsFunctor; + + template + friend struct Fluid_CorrectFluxFunctor; + + template + friend struct Fluid_CalcRHSFunctor; + + template + friend struct ShockFlattening_FindShockFunctor; + + // Emf boundary conditions + bool haveEmfBoundary{false}; + EmfBoundaryFunc emfBoundaryFunc{NULL}; + + // User defined source term + SrcTermFunc userSourceTerm{NULL}; + SrcTermFuncOld userSourceTermOld{NULL}; + bool haveUserSourceTerm{false}; + + real etaO, xH, xA; // Ohmic resistivity, Hall, ambipolar (when constant) + + // Ohmic, Hall and ambipolar diffusivity (when function-defined) + DiffusivityFunc ohmicDiffusivityFunc{NULL}; + DiffusivityFunc ambipolarDiffusivityFunc{NULL}; + DiffusivityFunc hallDiffusivityFunc{NULL}; + + IdefixArray3D cMax; // Maximum propagation speed + + // Nonideal effect diffusion coefficient (only allocated when needed) + IdefixArray3D etaOhmic; + IdefixArray3D xHall; + IdefixArray3D xAmbipolar; + + // Loop on dimensions + template + void LoopDir(const real, const real); +}; + +#include "physics.hpp" +#include "dataBlock.hpp" +#include "boundary.hpp" +#include "constrainedTransport.hpp" +#include "axis.hpp" +#include "rkl.hpp" +#include "riemannSolver.hpp" +#include "viscosity.hpp" +#include "bragViscosity.hpp" +#include "drag.hpp" +#include "checkNan.hpp" +#include "tracer.hpp" + + +template +Fluid::Fluid(Grid &grid, Input &input, DataBlock *datain, int n) { + idfx::pushRegion("Fluid::Fluid"); + // Save the datablock to which we are attached from now on + this->data = datain; + + // Create our own prefix + prefix = std::string(Phys::prefix); + + // When dealing with dust, add the specie number + if(Phys::prefix.compare("Dust") == 0) prefix += std::to_string(n); + + // Keep the instance # for later use + instanceNumber = n; + + #if ORDER < 1 || ORDER > 4 + IDEFIX_ERROR("Reconstruction at chosen order is not implemented. Check your definitions file"); + #endif + + // Source terms (always activated when non-cartesian geometry because of curvature source terms) +#if GEOMETRY == CARTESIAN + this->haveSourceTerms = false; +#else + this->haveSourceTerms = true; +#endif + + // Check whether we have rotation + int rotation = input.CheckEntry(std::string(Phys::prefix),"rotation"); + + if(rotation>=0 ) { + this->haveSourceTerms = true; + this->haveRotation = true; + if(rotation >1 ) IDEFIX_ERROR("Rotation needs a a single rotation velocity (Omega_Z) \ + in idefix.ini"); + this->OmegaZ = input.Get(std::string(Phys::prefix),"rotation",0); + } + + // Check whether we have shearing box + int shearingbox = input.CheckEntry(std::string(Phys::prefix),"shearingBox"); + + if(shearingbox>=0 ) { + this->haveShearingBox = true; + this->haveSourceTerms = true; + if(shearingbox != 1) { + IDEFIX_ERROR("Shearing box needs a scalar value for the shear rate in idefix.ini"); + } + + this->sbS = input.Get(std::string(Phys::prefix),"shearingBox",0); + // Get box size + this->sbLx = grid.xend[IDIR] - grid.xbeg[IDIR]; + } + + // Passive tracers + if(input.CheckEntry(std::string(Phys::prefix),"tracer")>=0) { + this->haveTracer = true; + // nTracer is initialised before instanciation of the Tracer object + // Because we need to know the number of tracers to allocate Vc, Uc and Flux. + this->nTracer = input.Get(std::string(Phys::prefix),"tracer",0); + if(this->nTracer < 1) { + IDEFIX_ERROR("The number of passive tracers should be >= 1"); + } + } + + // If we are not the primary hydro object, we copy the properties of the primary hydro object + // so that we solve for consistant physics + if(prefix.compare("Hydro") != 0) { + this->haveSourceTerms = data->hydro->haveSourceTerms; + this->haveRotation = data->hydro->haveRotation; + this->OmegaZ = data->hydro->OmegaZ; + this->haveShearingBox = data->hydro->haveShearingBox; + this->sbS = data->hydro->sbS; + this->sbLx = data->hydro->sbLx; + } + + + + + /////////////////////// + // Parabolic terms + /////////////////////// + + + // Check whether viscosity is enabled, if so, init a viscosity object + if(input.CheckEntry(std::string(Phys::prefix),"viscosity")>=0) { + std::string opType = input.Get(std::string(Phys::prefix),"viscosity",0); + if(opType.compare("explicit") == 0 ) { + haveExplicitParabolicTerms = true; + viscosityStatus.isExplicit = true; + } else if(opType.compare("rkl") == 0 ) { + haveRKLParabolicTerms = true; + viscosityStatus.isRKL = true; + } else { + std::stringstream msg; + msg << "Unknown integration type for viscosity: " << opType; + IDEFIX_ERROR(msg); + } + if(input.Get(std::string(Phys::prefix),"viscosity",1) + .compare("constant") == 0) { + viscosityStatus.status = Constant; + } else if(input.Get(std::string(Phys::prefix),"viscosity",1) + .compare("userdef") == 0) { + viscosityStatus.status = UserDefFunction; + } else { + IDEFIX_ERROR("Unknown viscosity definition in idefix.ini. " + "Can only be constant or userdef."); + } + } + + // Check whether thermal diffusion is enabled, if so, init a thermal diffusion object + if(input.CheckEntry(std::string(Phys::prefix),"TDiffusion")>=0) { + std::string opType = input.Get(std::string(Phys::prefix),"TDiffusion",0); + if(opType.compare("explicit") == 0 ) { + haveExplicitParabolicTerms = true; + thermalDiffusionStatus.isExplicit = true; + } else if(opType.compare("rkl") == 0 ) { + haveRKLParabolicTerms = true; + thermalDiffusionStatus.isRKL = true; + } else { + std::stringstream msg; + msg << "Unknown integration type for thermal diffusion: " << opType; + IDEFIX_ERROR(msg); + } + if(input.Get(std::string(Phys::prefix),"TDiffusion",1).compare("constant") == 0) { + thermalDiffusionStatus.status = Constant; + } else if(input.Get(std::string(Phys::prefix), + "TDiffusion",1).compare("userdef") == 0) { + thermalDiffusionStatus.status = UserDefFunction; + + } else { + IDEFIX_ERROR("Unknown thermal diffusion definition in idefix.ini. " + "Can only be constant or userdef."); + } + } + + // Check whether braginskii viscosity is enabled, if so, init a braginskii viscosity object + if(input.CheckEntry(std::string(Phys::prefix),"bragViscosity")>=0) { + std::string opType = input.Get(std::string(Phys::prefix),"bragViscosity",0); + if(opType.compare("explicit") == 0 ) { + haveExplicitParabolicTerms = true; + bragViscosityStatus.isExplicit = true; + } else if(opType.compare("rkl") == 0 ) { + haveRKLParabolicTerms = true; + bragViscosityStatus.isRKL = true; + } else { + std::stringstream msg; + msg << "Unknown integration type for braginskii viscosity: " << opType; + IDEFIX_ERROR(msg); + } + this->bragViscosity = std::make_unique(input, grid, this); + } + + // Check whether braginskii thermal diffusion is enabled, + // if so, init a braginskii thermal diffusion object + if(input.CheckEntry(std::string(Phys::prefix),"bragTDiffusion")>=0) { + std::string opType = input.Get(std::string(Phys::prefix),"bragTDiffusion",0); + if(opType.compare("explicit") == 0 ) { + haveExplicitParabolicTerms = true; + bragThermalDiffusionStatus.isExplicit = true; + } else if(opType.compare("rkl") == 0 ) { + haveRKLParabolicTerms = true; + bragThermalDiffusionStatus.isRKL = true; + } else { + std::stringstream msg; + msg << "Unknown integration type for braginskii thermal diffusion: " << opType; + IDEFIX_ERROR(msg); + } + this->bragThermalDiffusion = std::make_unique(input, grid, this); + } + + if(input.CheckEntry(std::string(Phys::prefix),"drag")>=0) { + haveDrag = true; + } + + if constexpr(Phys::mhd) { + if(input.CheckEntry(std::string(Phys::prefix),"resistivity")>=0 || + input.CheckEntry(std::string(Phys::prefix),"ambipolar")>=0 || + input.CheckEntry(std::string(Phys::prefix),"hall")>=0 ) { + // + this->haveCurrent = true; + + if(input.CheckEntry(std::string(Phys::prefix),"resistivity")>=0) { + std::string opType = input.Get(std::string(Phys::prefix),"resistivity",0); + if(opType.compare("explicit") == 0 ) { + haveExplicitParabolicTerms = true; + resistivityStatus.isExplicit = true; + needExplicitCurrent = true; + } else if(opType.compare("rkl") == 0 ) { + haveRKLParabolicTerms = true; + resistivityStatus.isRKL = true; + needRKLCurrent = true; + } else { + std::stringstream msg; + msg << "Unknown integration type for resistivity: " << opType; + IDEFIX_ERROR(msg); + } + if(input.Get( + std::string(Phys::prefix),"resistivity",1).compare("constant") == 0) { + this->etaO = input.Get(std::string(Phys::prefix),"resistivity",2); + resistivityStatus.status = Constant; + } else if(input.Get( + std::string(Phys::prefix),"resistivity",1).compare("userdef") == 0) { + resistivityStatus.status = UserDefFunction; + } else { + IDEFIX_ERROR("Unknown resistivity definition in idefix.ini. " + "Can only be constant or userdef."); + } + } + + if(input.CheckEntry(std::string(Phys::prefix),"ambipolar")>=0) { + std::string opType = input.Get(std::string(Phys::prefix),"ambipolar",0); + if(opType.compare("explicit") == 0 ) { + haveExplicitParabolicTerms = true; + ambipolarStatus.isExplicit = true; + needExplicitCurrent = true; + } else if(opType.compare("rkl") == 0 ) { + haveRKLParabolicTerms = true; + ambipolarStatus.isRKL = true; + needRKLCurrent = true; + } else { + std::stringstream msg; + msg << "Unknown integration type for ambipolar: " << opType; + IDEFIX_ERROR(msg); + } + if(input.Get( + std::string(Phys::prefix),"ambipolar",1).compare("constant") == 0) { + this->xA = input.Get(std::string(Phys::prefix),"ambipolar",2); + ambipolarStatus.status = Constant; + } else if(input.Get( + std::string(Phys::prefix),"ambipolar",1).compare("userdef") == 0) { + ambipolarStatus.status = UserDefFunction; + } else { + IDEFIX_ERROR("Unknown ambipolar definition in idefix.ini. " + "Can only be constant or userdef."); + } + } + + if(input.CheckEntry(std::string(Phys::prefix),"hall")>=0) { + std::string opType = input.Get(std::string(Phys::prefix),"hall",0); + if(opType.compare("explicit") == 0 ) { + hallStatus.isExplicit = true; + needExplicitCurrent = true; + } else if(opType.compare("rkl") == 0 ) { + IDEFIX_ERROR("RKL inegration is incompatible with Hall"); + } else { + std::stringstream msg; + msg << "Unknown integration type for hall: " << opType; + IDEFIX_ERROR(msg); + } + if(input.Get(std::string(Phys::prefix),"hall",1).compare("constant") == 0) { + this->xH = input.Get(std::string(Phys::prefix),"hall",2); + hallStatus.status = Constant; + } else if(input.Get( + std::string(Phys::prefix),"hall",1).compare("userdef") == 0) { + hallStatus.status = UserDefFunction; + } else { + IDEFIX_ERROR("Unknown Hall definition in idefix.ini. Can only be constant or userdef."); + } + } + } + } // MHD + + ///////////////////////////////////////// + // ALLOCATION SECION /////////////////// + ///////////////////////////////////////// + + // We now allocate the fields required by the hydro solver + Vc = IdefixArray4D(prefix+"_Vc", Phys::nvar+nTracer, + data->np_tot[KDIR], data->np_tot[JDIR], data->np_tot[IDIR]); + Uc = IdefixArray4D(prefix+"_Uc", Phys::nvar+nTracer, + data->np_tot[KDIR], data->np_tot[JDIR], data->np_tot[IDIR]); + + data->states["current"].PushArray(Uc, State::center, prefix+"_Uc"); + + InvDt = IdefixArray3D(prefix+"_InvDt", + data->np_tot[KDIR], data->np_tot[JDIR], data->np_tot[IDIR]); + cMax = IdefixArray3D(prefix+"_cMax", + data->np_tot[KDIR], data->np_tot[JDIR], data->np_tot[IDIR]); + dMax = IdefixArray3D(prefix+"_dMax", + data->np_tot[KDIR], data->np_tot[JDIR], data->np_tot[IDIR]); + FluxRiemann = IdefixArray4D(prefix+"_FluxRiemann", Phys::nvar+nTracer, + data->np_tot[KDIR], data->np_tot[JDIR], data->np_tot[IDIR]); + + if constexpr(Phys::mhd) { + Vs = IdefixArray4D(prefix+"_Vs", DIMENSIONS, + data->np_tot[KDIR]+KOFFSET, data->np_tot[JDIR]+JOFFSET, data->np_tot[IDIR]+IOFFSET); + #ifdef EVOLVE_VECTOR_POTENTIAL + #if DIMENSIONS == 1 + IDEFIX_ERROR("EVOLVE_VECTOR_POTENTIAL is not compatible with 1D MHD"); + #else + Ve = IdefixArray4D(prefix+"_Ve", AX3e+1, + data->np_tot[KDIR]+KOFFSET, data->np_tot[JDIR]+JOFFSET, data->np_tot[IDIR]+IOFFSET); + + data->states["current"].PushArray(Ve, State::center, prefix+"_Ve"); + #endif + #else // EVOLVE_VECTOR_POTENTIAL + data->states["current"].PushArray(Vs, State::center, prefix+"_Vs"); + #endif // EVOLVE_VECTOR_POTENTIAL + } + + if(this->haveCurrent) { + // Allocate current (when hydro needs it) + J = IdefixArray4D(prefix+"_J", 3, + data->np_tot[KDIR], data->np_tot[JDIR], data->np_tot[IDIR]); + } + + // Allocate nonideal MHD effects array when a user-defined function is used + if(this->resistivityStatus.status == UserDefFunction) + etaOhmic = IdefixArray3D(prefix+"_etaOhmic", + data->np_tot[KDIR], data->np_tot[JDIR], data->np_tot[IDIR]); + if(this->ambipolarStatus.status == UserDefFunction) + xAmbipolar = IdefixArray3D(prefix+"_xAmbipolar", + data->np_tot[KDIR], data->np_tot[JDIR], data->np_tot[IDIR]); + if(this->hallStatus.status == UserDefFunction) + xHall = IdefixArray3D(prefix+"_xHall", + data->np_tot[KDIR], data->np_tot[JDIR], data->np_tot[IDIR]); + + // Fill the names of the fields + std::string outputPrefix(""); + // If we have hydro, the output prefix is "" for backward compatibility + if(prefix.compare("Hydro") != 0) { + outputPrefix = prefix; + outputPrefix += "_"; + } + + for(int i = 0 ; i < Phys::nvar+nTracer ; i++) { + switch(i) { + case RHO: + VcName.push_back("RHO"); + data->dump->RegisterVariable(Vc, outputPrefix+"Vc-RHO", RHO); + break; + case VX1: + VcName.push_back("VX1"); + data->dump->RegisterVariable(Vc, outputPrefix+"Vc-VX1", VX1, IDIR); + break; + case VX2: + VcName.push_back("VX2"); + data->dump->RegisterVariable(Vc, outputPrefix+"Vc-VX2", VX2, JDIR); + break; + case VX3: + VcName.push_back("VX3"); + data->dump->RegisterVariable(Vc, outputPrefix+"Vc-VX3", VX3, KDIR); + break; + case BX1: + VcName.push_back("BX1"); + // never save cell-centered BX1 in dumps + break; + case BX2: + VcName.push_back("BX2"); + #if DIMENSIONS < 2 + data->dump->RegisterVariable(Vc, outputPrefix+"Vc-BX2", BX2, JDIR); + #endif + break; + case BX3: + VcName.push_back("BX3"); + #if DIMENSIONS < 3 + data->dump->RegisterVariable(Vc, outputPrefix+"Vc-BX3", BX3, KDIR); + #endif + break; + case PRS: + VcName.push_back("PRS"); + data->dump->RegisterVariable(Vc, outputPrefix+"Vc-PRS", PRS); + break; + default: + if(i>=Phys::nvar) { + std::string tracerLabel = std::string("TR")+std::to_string(i-Phys::nvar); // ="TRn" + VcName.push_back(tracerLabel); + data->dump->RegisterVariable(Vc, outputPrefix+"Vc-"+tracerLabel, i); + } else { + VcName.push_back("Vc-"+std::to_string(i)); + data->vtk->RegisterVariable(Vc, outputPrefix+"Vc-"+std::to_string(i), i); + } + } + data->vtk->RegisterVariable(Vc, outputPrefix+VcName[i], i); + #ifdef WITH_HDF5 + data->xdmf->RegisterVariable(Vc, outputPrefix+VcName[i], i); + #endif + } + + if constexpr(Phys::mhd) { + for(int i = 0 ; i < DIMENSIONS ; i++) { + switch(i) { + case 0: + VsName.push_back("BX1s"); + data->dump->RegisterVariable(Vs, outputPrefix+"Vs-BX1s", + BX1s, IDIR, DumpField::ArrayLocation::Face); + break; + case 1: + VsName.push_back("BX2s"); + data->dump->RegisterVariable(Vs, outputPrefix+"Vs-BX2s", + BX2s, JDIR, DumpField::ArrayLocation::Face); + break; + case 2: + VsName.push_back("BX3s"); + data->dump->RegisterVariable(Vs, outputPrefix+"Vs-BX3s", + BX3s, KDIR, DumpField::ArrayLocation::Face); + break; + default: + VsName.push_back("Vs-"+std::to_string(i)); + } + } + + #ifdef EVOLVE_VECTOR_POTENTIAL + #if DIMENSIONS < 3 + VeName.push_back("AX3e"); + data->dump->RegisterVariable(Ve, outputPrefix+"Ve-AX3e", + AX3e, KDIR, DumpField::ArrayLocation::Edge); + #else + for(int i = 0 ; i < DIMENSIONS ; i++) { + switch(i) { + case 0: + VeName.push_back("AX1e"); + data->dump->RegisterVariable(Ve, outputPrefix+"Ve-AX1e", + AX1e, IDIR, DumpField::ArrayLocation::Edge); + break; + case 1: + VeName.push_back("AX2e"); + data->dump->RegisterVariable(Ve, outputPrefix+"Ve-AX2e", + AX2e, JDIR, DumpField::ArrayLocation::Edge); + break; + case 2: + VeName.push_back("AX3e"); + data->dump->RegisterVariable(Ve, outputPrefix+"Ve-AX3e", + AX3e, KDIR, DumpField::ArrayLocation::Edge); + break; + default: + VeName.push_back("Ve-"+std::to_string(i)); + } + } + #endif + #endif + } + + + //******************************************* + //** Child object allocation section + //********************************************* + + // Initialise Riemann Solver + this->rSolver = std::make_unique>(input, this); + + if constexpr(Phys::mhd) { + this->emf = std::make_unique>(input, this); + } + + // Initialise the EOS + if constexpr(Phys::eos) { + this->eos = std::make_unique(input, data, this->prefix); + } + + // Initialise boundary conditions + boundary = std::make_unique>(this); + this->haveAxis = data->haveAxis; + + if(haveRKLParabolicTerms) { + this->rkl = std::make_unique>(input,this); + } + + // Thermal diffusion + if(thermalDiffusionStatus.status != Disabled ) { + this->thermalDiffusion = std::make_unique(input, grid, this); + } + + // Viscosity + if(viscosityStatus.status != Disabled) { + this->viscosity = std::make_unique(input, grid, this); + } + + // Braginskii Thermal diffusion + if(bragThermalDiffusionStatus.status != Disabled ) { + this->bragThermalDiffusion = std::make_unique(input, grid, this); + } + + // Braginskii Viscosity + if(bragViscosityStatus.status != Disabled) { + this->bragViscosity = std::make_unique(input, grid, this); + } + + + // Drag force when needed + if(haveDrag) { + this->drag = std::make_unique(input, this); + } + + // Tracers when needed + if(haveTracer) { + this->tracer= std::make_unique(this, nTracer); + } + + idfx::popRegion(); +} + + +#include "addSourceTerms.hpp" +#include "calcRightHandSide.hpp" +#include "enroll.hpp" +#include "calcCurrent.hpp" +#include "coarsenFlow.hpp" +#include "convertConsToPrim.hpp" +#include "checkDivB.hpp" +#include "evolveStage.hpp" +#include "showConfig.hpp" +#endif // FLUID_FLUID_HPP_ diff --git a/src/hydro/hydro_defs.hpp b/src/fluid/fluid_defs.hpp similarity index 74% rename from src/hydro/hydro_defs.hpp rename to src/fluid/fluid_defs.hpp index b48521ed..2269fc80 100644 --- a/src/hydro/hydro_defs.hpp +++ b/src/fluid/fluid_defs.hpp @@ -5,25 +5,24 @@ // Licensed under CeCILL 2.1 License, see COPYING for more information // *********************************************************************************** -#ifndef HYDRO_HYDRO_DEFS_HPP_ -#define HYDRO_HYDRO_DEFS_HPP_ +#ifndef FLUID_FLUID_DEFS_HPP_ +#define FLUID_FLUID_DEFS_HPP_ #include "../idefix.hpp" + // Common definitions for all of the objects dependent on hydro // forward class declaration class DataBlock; +template +class Fluid; + #define SMALL_PRESSURE_FIX (1.0e-5) #define eps_UCT_CONTACT (1.0e-6) -// Riemann Solver type -#if MHD == YES -enum Solver {TVDLF=1, HLL, HLLD, ROE}; -#else -enum Solver {TVDLF=1, HLL, HLLC, ROE}; -#endif + // Parabolic terms can have different status enum HydroModuleStatus {Disabled, Constant, UserDefFunction }; @@ -36,18 +35,25 @@ struct ParabolicModuleStatus { }; -using UserDefBoundaryFunc = void (*) (DataBlock &, int dir, BoundarySide side, - const real t); using GravPotentialFunc = void (*) (DataBlock &, const real t, IdefixArray1D&, IdefixArray1D&, IdefixArray1D&, IdefixArray3D &); using BodyForceFunc = void (*) (DataBlock &, const real t, IdefixArray4D&); -using SrcTermFunc = void (*) (DataBlock &, const real t, const real dt); -using InternalBoundaryFunc = void (*) (DataBlock &, const real t); +using IsoSoundSpeedFunc = void (*) (DataBlock &, const real t, IdefixArray3D &); + +template +using SrcTermFunc = void (*) (Fluid *, const real t, const real dt); + + using EmfBoundaryFunc = void (*) (DataBlock &, const real t); using DiffusivityFunc = void (*) (DataBlock &, const real t, IdefixArray3D &); -using IsoSoundSpeedFunc = void (*) (DataBlock &, const real t, IdefixArray3D &); +using BragDiffusivityFunc = void (*) (DataBlock &, const real t, + IdefixArray3D &, IdefixArray3D &); + +// Deprecated signatures +using SrcTermFuncOld = void (*) (DataBlock &, const real t, const real dt); + -#endif //HYDRO_HYDRO_DEFS_HPP_ +#endif //FLUID_FLUID_DEFS_HPP_ diff --git a/src/fluid/showConfig.hpp b/src/fluid/showConfig.hpp new file mode 100644 index 00000000..f3a14430 --- /dev/null +++ b/src/fluid/showConfig.hpp @@ -0,0 +1,168 @@ +// *********************************************************************************** +// Idefix MHD astrophysical code +// Copyright(C) Geoffroy R. J. Lesur +// and other code contributors +// Licensed under CeCILL 2.1 License, see COPYING for more information +// *********************************************************************************** + +#ifndef FLUID_SHOWCONFIG_HPP_ +#define FLUID_SHOWCONFIG_HPP_ + +#include + +#include "idefix.hpp" +#include "fluid.hpp" +#include "dataBlock.hpp" + +template +void Fluid::ShowConfig() { + idfx::cout << Phys::prefix << ": "; + if constexpr(!Phys::dust) { + if constexpr(Phys::mhd) { + idfx::cout << "solving MHD equations." << std::endl; + #ifdef EVOLVE_VECTOR_POTENTIAL + idfx::cout << Phys::prefix << ": Using EXPERIMENTAL vector potential formulation for MHD." + << std::endl; + #endif + } else { + idfx::cout << "solving HD equations." << std::endl; + } + } else { + idfx::cout << "solving pressure-less dust equations." << std::endl; + } + idfx::cout << Phys::prefix << ": Reconstruction: "; + #if ORDER == 1 + idfx::cout << "1st order (donor cell)" << std::endl; + #elif ORDER == 2 + idfx::cout << "2nd order (PLM Van Leer)" << std::endl; + #elif ORDER == 3 + idfx::cout << "3rd order (LimO3)" << std::endl; + #elif ORDER == 4 + idfx::cout << "4th order (PPM)" << std::endl; + #endif + + + + if(haveRotation) { + idfx::cout << Phys::prefix << ": Rotation ENABLED with Omega=" << this->OmegaZ << std::endl; + } + if(haveShearingBox) { + idfx::cout << Phys::prefix << ": ShearingBox ENABLED with shear rate= " << this->sbS + << " and Lx= " << sbLx << std::endl; + } + + if(resistivityStatus.status != Disabled) { + if(resistivityStatus.status == Constant) { + idfx::cout << Phys::prefix << ": Ohmic resistivity ENABLED with constant resistivity eta=" + << etaO << std::endl; + } else if(resistivityStatus.status == UserDefFunction) { + idfx::cout << Phys::prefix + << ": Ohmic resistivity ENABLED with user-defined resistivity function." + << std::endl; + if(!ohmicDiffusivityFunc) { + IDEFIX_ERROR("No user-defined Ihmic resistivity function has been enrolled."); + } + } else { + IDEFIX_ERROR("Unknown Ohmic resistivity mode"); + } + if(resistivityStatus.isExplicit) { + idfx::cout << Phys::prefix << ": Ohmic resistivity uses an explicit time integration." + << std::endl; + } else if(resistivityStatus.isRKL) { + idfx::cout << Phys::prefix + << ": Ohmic resistivity uses a Runge-Kutta-Legendre time integration." + << std::endl; + } else { + IDEFIX_ERROR("Unknown time integrator for Ohmic resistivity"); + } + } + + if(ambipolarStatus.status != Disabled) { + if(ambipolarStatus.status == Constant) { + idfx::cout << Phys::prefix << ": Ambipolar diffusion ENABLED with constant diffusivity xA=" + << xA << std::endl; + } else if(ambipolarStatus.status == UserDefFunction) { + idfx::cout << Phys::prefix + << ": Ambipolar diffusion ENABLED with user-defined diffusivity function." + << std::endl; + if(!ambipolarDiffusivityFunc) { + IDEFIX_ERROR("No user-defined ambipolar diffusion function has been enrolled."); + } + } else { + IDEFIX_ERROR("Unknown Ambipolar diffusion mode"); + } + if(ambipolarStatus.isExplicit) { + idfx::cout << Phys::prefix << ": Ambipolar diffusion uses an explicit time integration." + << std::endl; + } else if(ambipolarStatus.isRKL) { + idfx::cout << Phys::prefix + << ": Ambipolar diffusion uses a Runge-Kutta-Legendre time integration." + << std::endl; + } else { + IDEFIX_ERROR("Unknown time integrator for ambipolar diffusion"); + } + } + + if(hallStatus.status != Disabled) { + if(hallStatus.status == Constant) { + idfx::cout << Phys::prefix << ": Hall effect ENABLED with constant diffusivity xH=" + << xH << std::endl; + } else if(hallStatus.status == UserDefFunction) { + idfx::cout << Phys::prefix << ": Hall effect ENABLED with user-defined diffusivity function." + << std::endl; + if(!hallDiffusivityFunc) { + IDEFIX_ERROR("No user-defined Hall diffusivity function has been enrolled."); + } + } else { + IDEFIX_ERROR("Unknown Hall effect mode"); + } + if(hallStatus.isExplicit) { + idfx::cout << Phys::prefix << ": Hall effect uses an explicit time integration." << std::endl; + } else { + IDEFIX_ERROR("Unknown time integrator for Hall effect"); + } + } + + if(haveTracer) { + idfx::cout << Phys::prefix << ": " << this->nTracer << " tracers ENABLED for this fluid." + << std::endl; + } + + if(emfBoundaryFunc) { + idfx::cout << Phys::prefix << ": user-defined EMF boundaries ENABLED." << std::endl; + } + if(userSourceTerm) { + idfx::cout << Phys::prefix << ": user-defined source terms ENABLED." << std::endl; + } + + if constexpr(Phys::eos) { + eos->ShowConfig(); + } + rSolver->ShowConfig(); + + if constexpr(Phys::mhd) { + emf->ShowConfig(); + } + if(haveRKLParabolicTerms) { + rkl->ShowConfig(); + } + if(viscosityStatus.isExplicit || viscosityStatus.isRKL) { + viscosity->ShowConfig(); + } + if(thermalDiffusionStatus.status != Disabled) { + thermalDiffusion->ShowConfig(); + } + if(bragViscosityStatus.isExplicit || bragViscosityStatus.isRKL) { + bragViscosity->ShowConfig(); + } + if(bragThermalDiffusionStatus.status != Disabled) { + bragThermalDiffusion->ShowConfig(); + } + if(haveAxis) { + boundary->axis->ShowConfig(); + } + if(haveDrag) { + drag->ShowConfig(); + } +} +#endif //FLUID_SHOWCONFIG_HPP_ diff --git a/src/hydro/thermalDiffusion.cpp b/src/fluid/thermalDiffusion.cpp similarity index 58% rename from src/hydro/thermalDiffusion.cpp rename to src/fluid/thermalDiffusion.cpp index 3d657e11..344e0d16 100644 --- a/src/hydro/thermalDiffusion.cpp +++ b/src/fluid/thermalDiffusion.cpp @@ -8,51 +8,19 @@ #include + #include "thermalDiffusion.hpp" #include "dataBlock.hpp" +#include "fluid.hpp" +#include "eos.hpp" -ThermalDiffusion::ThermalDiffusion() { - // Default constructor -} - -void ThermalDiffusion::Init(Input &input, Grid &grid, Hydro *hydroin) { - idfx::pushRegion("ThermalDiffusion::Init"); - // Save the parent hydro object - this->hydro = hydroin; - - if(input.CheckEntry("Hydro","TDiffusion")>=0) { - if(input.Get("Hydro","TDiffusion",1).compare("constant") == 0) { - this->kappa = input.Get("Hydro","TDiffusion",2); - this->haveThermalDiffusion = Constant; - } else if(input.Get("Hydro","TDiffusion",1).compare("userdef") == 0) { - this->haveThermalDiffusion = UserDefFunction; - this->kappaArr = IdefixArray3D("ThermalDiffusionKappaArray",hydro->data->np_tot[KDIR], - hydro->data->np_tot[JDIR], - hydro->data->np_tot[IDIR]); - - } else { - IDEFIX_ERROR("Unknown thermal diffusion definition in idefix.ini. " - "Can only be constant or userdef."); - } - } else { - IDEFIX_ERROR("I cannot create a ThermalDiffusion object without TDiffusion defined" - "in the .ini file"); - } - - #ifdef ISOTHERMAL - IDEFIX_ERROR("Thermal diffusion is not compatible with the ISOTHERMAL approximation"); - #endif - - idfx::popRegion(); -} - void ThermalDiffusion::ShowConfig() { - if(haveThermalDiffusion==Constant) { - idfx::cout << "Thermal Diffusion: ENEABLED with constant diffusivity kappa=" + if(status.status==Constant) { + idfx::cout << "Thermal Diffusion: ENABLED with constant diffusivity kappa=" << this->kappa << " ."<< std::endl; - } else if (haveThermalDiffusion==UserDefFunction) { + } else if (status.status==UserDefFunction) { idfx::cout << "Thermal Diffusion: ENABLED with user-defined diffusivity function." << std::endl; if(!diffusivityFunc) { @@ -61,9 +29,9 @@ void ThermalDiffusion::ShowConfig() { } else { IDEFIX_ERROR("Unknown thermal diffusion mode"); } - if(hydro->thermalDiffusionStatus.isExplicit) { + if(status.isExplicit) { idfx::cout << "Thermal Diffusion: uses an explicit time integration." << std::endl; - } else if(hydro->thermalDiffusionStatus.isRKL) { + } else if(status.isRKL) { idfx::cout << "Thermal Diffusion: uses a Runge-Kutta-Legendre time integration." << std::endl; } else { @@ -72,43 +40,41 @@ void ThermalDiffusion::ShowConfig() { } void ThermalDiffusion::EnrollThermalDiffusivity(DiffusivityFunc myFunc) { - if(this->haveThermalDiffusion < UserDefFunction) { + if(this->status.status != UserDefFunction) { IDEFIX_WARNING("Thermal diffusivity enrollment requires Hydro/ThermalDiffusion " "to be set to userdef in .ini file"); } this->diffusivityFunc = myFunc; - idfx::cout << "ThermalDiffusion: User-defined thermal diffusion has been enrolled." << std::endl; } // This function computes the viscous flux and stores it in hydro->fluxRiemann // (this avoids an extra array) // Associated source terms, present in non-cartesian geometry are also computed // and stored in this->viscSrc for later use (in calcRhs). -void ThermalDiffusion::AddDiffusiveFlux(int dir, const real t) { -#if HAVE_ENERGY == 1 +void ThermalDiffusion::AddDiffusiveFlux(int dir, const real t, const IdefixArray4D &Flux) { idfx::pushRegion("ThermalDiffusion::AddDiffusiveFlux"); - IdefixArray4D Vc = this->hydro->Vc; - IdefixArray4D Flux = this->hydro->FluxRiemann; - IdefixArray3D dMax = this->hydro->dMax; + IdefixArray4D Vc = this->Vc; + IdefixArray3D dMax = this->dMax; IdefixArray3D kappaArr = this->kappaArr; - IdefixArray1D dx = this->hydro->data->dx[dir]; + IdefixArray1D dx = this->data->dx[dir]; + EquationOfState eos = *(this->eos); #if GEOMETRY == POLAR - IdefixArray1D x1 = this->hydro->data->x[IDIR]; + IdefixArray1D x1 = this->data->x[IDIR]; #endif #if GEOMETRY == SPHERICAL - IdefixArray1D rt = this->hydro->data->rt; - IdefixArray1D dmu = this->hydro->data->dmu; - IdefixArray1D dx2 = this->hydro->data->dx[JDIR]; + IdefixArray1D rt = this->data->rt; + IdefixArray1D dmu = this->data->dmu; + IdefixArray1D dx2 = this->data->dx[JDIR]; #endif - HydroModuleStatus haveThermalDiffusion = this->haveThermalDiffusion; + HydroModuleStatus haveThermalDiffusion = this->status.status; // Compute thermal diffusion if needed if(haveThermalDiffusion == UserDefFunction && dir == IDIR) { if(diffusivityFunc) { idfx::pushRegion("UserDef::ThermalDiffusivityFunction"); - diffusivityFunc(*this->hydro->data, t, kappaArr); + diffusivityFunc(*this->data, t, kappaArr); idfx::popRegion(); } else { IDEFIX_ERROR("No user-defined thermal diffusion function has been enrolled"); @@ -117,15 +83,14 @@ void ThermalDiffusion::AddDiffusiveFlux(int dir, const real t) { int ibeg, iend, jbeg, jend, kbeg, kend; - ibeg = this->hydro->data->beg[IDIR]; - iend = this->hydro->data->end[IDIR]; - jbeg = this->hydro->data->beg[JDIR]; - jend = this->hydro->data->end[JDIR]; - kbeg = this->hydro->data->beg[KDIR]; - kend = this->hydro->data->end[KDIR]; + ibeg = this->data->beg[IDIR]; + iend = this->data->end[IDIR]; + jbeg = this->data->beg[JDIR]; + jend = this->data->end[JDIR]; + kbeg = this->data->beg[KDIR]; + kend = this->data->end[KDIR]; real kappaConstant = this->kappa; - real gamma = hydro->gamma; if(dir==IDIR) iend++; if(dir==JDIR) jend++; @@ -174,10 +139,11 @@ void ThermalDiffusion::AddDiffusiveFlux(int dir, const real t) { Flux(ENG,k,j,i) += -kappa*gradT; // Compute total diffusion coefficient + real gamma = eos.GetGamma(Vc(PRS,k,j,i),Vc(RHO,k,j,i)); + real locdmax = kappa * (gamma-ONE_F) / (HALF_F * ( Vc(RHO,k,j,i) + Vc(RHO,k-koffset,j-joffset,i-ioffset))); dMax(k,j,i) = FMAX(dMax(k,j,i) , locdmax); }); idfx::popRegion(); -#endif // HAVE_ENERGY } diff --git a/src/fluid/thermalDiffusion.hpp b/src/fluid/thermalDiffusion.hpp new file mode 100644 index 00000000..4e40f3a7 --- /dev/null +++ b/src/fluid/thermalDiffusion.hpp @@ -0,0 +1,93 @@ +// *********************************************************************************** +// Idefix MHD astrophysical code +// Copyright(C) Geoffroy R. J. Lesur +// and other code contributors +// Licensed under CeCILL 2.1 License, see COPYING for more information +// *********************************************************************************** + +#ifndef FLUID_THERMALDIFFUSION_HPP_ +#define FLUID_THERMALDIFFUSION_HPP_ + +#include + +#include "idefix.hpp" +#include "input.hpp" +#include "grid.hpp" +#include "fluid_defs.hpp" +#include "eos.hpp" + + +// Forward class hydro declaration +template class Fluid; + +class DataBlock; + +class ThermalDiffusion { + public: + template + ThermalDiffusion(Input &, Grid &, Fluid *); // Initialisation + + void ShowConfig(); // display configuration + + void AddDiffusiveFlux(int, const real, const IdefixArray4D &); + + // Enroll user-defined viscous diffusivity + void EnrollThermalDiffusivity(DiffusivityFunc); + + IdefixArray4D viscSrc; // Source terms of the viscous operator + IdefixArray3D kappaArr; + + // pre-computed geometrical factors in non-cartesian geometry + IdefixArray1D one_dmu; + + private: + DataBlock *data; + + // status of the module + ParabolicModuleStatus &status; + + DiffusivityFunc diffusivityFunc; + + // helper array + IdefixArray4D &Vc; + IdefixArray3D &dMax; + + // constant diffusion coefficient (when needed) + real kappa; + + // equation of state (required to get the heat capacity) + EquationOfState *eos; +}; + +#include "fluid.hpp" +#include "dataBlock.hpp" + +template +ThermalDiffusion::ThermalDiffusion(Input &input, Grid &grid, Fluid *hydroin): + Vc{hydroin->Vc}, + dMax{hydroin->dMax}, + eos{hydroin->eos.get()}, + data{hydroin->data}, + status{hydroin->thermalDiffusionStatus} { + idfx::pushRegion("ThermalDiffusion::ThermalDiffusion"); + + if(status.status == Constant) { + this->kappa = input.Get(std::string(Phys::prefix),"TDiffusion",2); + } else if(status.status == UserDefFunction) { + this->kappaArr = IdefixArray3D("ThermalDiffusionKappaArray",data->np_tot[KDIR], + data->np_tot[JDIR], + data->np_tot[IDIR]); + } else { + IDEFIX_ERROR("Unknown thermal diffusion definition in idefix.ini. " + "Can only be constant or userdef."); + } + + + #ifdef ISOTHERMAL + IDEFIX_ERROR("Thermal diffusion is not compatible with the ISOTHERMAL approximation"); + #endif + + idfx::popRegion(); +} + +#endif // FLUID_THERMALDIFFUSION_HPP_ diff --git a/src/fluid/tracer/CMakeLists.txt b/src/fluid/tracer/CMakeLists.txt new file mode 100644 index 00000000..cb8e3575 --- /dev/null +++ b/src/fluid/tracer/CMakeLists.txt @@ -0,0 +1,4 @@ +target_sources(idefix + PUBLIC ${CMAKE_CURRENT_LIST_DIR}/tracer.cpp + PUBLIC ${CMAKE_CURRENT_LIST_DIR}/tracer.hpp + ) diff --git a/src/fluid/tracer/tracer.cpp b/src/fluid/tracer/tracer.cpp new file mode 100644 index 00000000..7ee95c6c --- /dev/null +++ b/src/fluid/tracer/tracer.cpp @@ -0,0 +1,49 @@ +// *********************************************************************************** +// Idefix MHD astrophysical code +// Copyright(C) Geoffroy R. J. Lesur +// and other code contributors +// Licensed under CeCILL 2.1 License, see COPYING for more information +// *********************************************************************************** + +#include "idefix.hpp" +#include "tracer.hpp" + +// Convect Conservative to Primitive variable +void Tracer::ConvertConsToPrim() { + idfx::pushRegion("Tracer::ConvertConsToPrim"); + + IdefixArray4D Vc = this->Vc; + IdefixArray4D Uc = this->Uc; + + idefix_for("ConsToPrimScalar", + nVar, nVar+nTracer, // Loop on the index where scalars are lying + 0,data->np_tot[KDIR], + 0,data->np_tot[JDIR], + 0,data->np_tot[IDIR], + KOKKOS_LAMBDA (int n, int k, int j, int i) { + Vc(n,k,j,i) = Uc(n,k,j,i) / Uc(RHO,k,j,i); + }); + + idfx::popRegion(); +} + + + +// Convect Conservative to Primitive variable +void Tracer::ConvertPrimToCons() { + idfx::pushRegion("Tracer::ConvertPrimToCons"); + + IdefixArray4D Vc = this->Vc; + IdefixArray4D Uc = this->Uc; + + idefix_for("PrimToConsScalar", + nVar, nVar+nTracer, // Loop on the index where scalars are lying + 0,data->np_tot[KDIR], + 0,data->np_tot[JDIR], + 0,data->np_tot[IDIR], + KOKKOS_LAMBDA (int n, int k, int j, int i) { + Uc(n,k,j,i) = Vc(n,k,j,i) * Vc(RHO,k,j,i); + }); + + idfx::popRegion(); +} diff --git a/src/fluid/tracer/tracer.hpp b/src/fluid/tracer/tracer.hpp new file mode 100644 index 00000000..57889131 --- /dev/null +++ b/src/fluid/tracer/tracer.hpp @@ -0,0 +1,125 @@ +// *********************************************************************************** +// Idefix MHD astrophysical code +// Copyright(C) Geoffroy R. J. Lesur +// and other code contributors +// Licensed under CeCILL 2.1 License, see COPYING for more information +// *********************************************************************************** + +#ifndef FLUID_TRACER_TRACER_HPP_ +#define FLUID_TRACER_TRACER_HPP_ + +#include +#include "idefix.hpp" +#include "slopeLimiter.hpp" + +// Forward class hydro declaration +template class Fluid; +class DataBlock; + + +class Tracer { + public: + template Tracer(Fluid *, int n); + void ConvertConsToPrim(); + void ConvertPrimToCons(); + template void CalcFlux(IdefixArray4D &); + template void CalcRightHandSide(IdefixArray4D &, real, real); + + private: + IdefixArray4D Vc; // Vector of primitive variables for the passive tracer + IdefixArray4D Uc; // Vector of conservative variables for the passive tracer + + std::string prefix; + + DataBlock *data; + + int nTracer; + int nVar; +}; + + +#include "fluid.hpp" +#include "dataBlock.hpp" + +template +Tracer::Tracer(Fluid *fluid, int n) { + idfx::pushRegion("Tracer::Tracer"); + nTracer = n; + Vc = fluid->Vc; + Uc = fluid->Uc; + data = fluid->data; + nVar = Phys::nvar; + idfx::popRegion(); +} + + +// Compute the upwinded flux +template +void Tracer::CalcFlux(IdefixArray4D &Flux) { + idfx::pushRegion("Tracer::CalcFlux"); + + IdefixArray4D Vc = this->Vc; + IdefixArray4D Uc = this->Uc; + IdefixArray3D A = data->A[dir]; + + constexpr int ioffset = (dir==IDIR ? 1 : 0); + constexpr int joffset = (dir==JDIR ? 1 : 0); + constexpr int koffset = (dir==KDIR ? 1 : 0); + + idefix_for("ComputeTracerFlux", + Phys::nvar, Phys::nvar+nTracer, // Loop on the index where tracers are lying + data->beg[KDIR],data->end[KDIR]+koffset, + data->beg[JDIR],data->end[JDIR]+joffset, + data->beg[IDIR],data->end[IDIR]+ioffset, + KOKKOS_LAMBDA (int nv, int k, int j, int i) { + real vface; + if(Flux(RHO,k,j,i) > 0) { + // Interpolate from the left + real dvm = Vc(nv,k-koffset,j-joffset,i-ioffset) + -Vc(nv,k-2*koffset,j-2*joffset,i-2*ioffset); + real dvp = Vc(nv,k,j,i)-Vc(nv,k-koffset,j-joffset,i-ioffset); + + real dv = SlopeLimiter<>::PLMLim(dvp,dvm); + + vface = Vc(nv,k-koffset,j-joffset,i-ioffset) + HALF_F*dv; + } else { + // interpolation from the right + real dvm = Vc(nv,k,j,i) + -Vc(nv,k-koffset,j-joffset,i-ioffset); + real dvp = Vc(nv,k+koffset,j+joffset,i+ioffset) - Vc(nv,k,j,i); + + real dv = SlopeLimiter<>::PLMLim(dvp,dvm); + + vface = Vc(nv,k,j,i) - HALF_F*dv; + } + + // Compute flux*Area (! different from the fluid flux) + Flux(nv,k,j,i) = Flux(RHO,k,j,i) * vface * A(k,j,i); + }); + idfx::popRegion(); +} + +template +void Tracer::CalcRightHandSide(IdefixArray4D &Flux, real t, real dt) { + idfx::pushRegion("Tracer::ComputeRHS"); + + IdefixArray4D Uc = this->Uc; + IdefixArray3D dV = data->dV; + + constexpr int ioffset = (dir==IDIR ? 1 : 0); + constexpr int joffset = (dir==JDIR ? 1 : 0); + constexpr int koffset = (dir==KDIR ? 1 : 0); + + idefix_for("ComputeTracerRHS", + Phys::nvar, Phys::nvar+nTracer, // Loop on the index where tracers are lying + data->beg[KDIR],data->end[KDIR], + data->beg[JDIR],data->end[JDIR], + data->beg[IDIR],data->end[IDIR], + KOKKOS_LAMBDA (int nv, int k, int j, int i) { + Uc(nv,k,j,i) += -dt / dV(k,j,i) * (Flux(nv,k+koffset,j+joffset,i+ioffset) - Flux(nv,k,j,i)); + }); + + idfx::popRegion(); +} + +#endif // FLUID_TRACER_TRACER_HPP_ diff --git a/src/hydro/viscosity.cpp b/src/fluid/viscosity.cpp similarity index 78% rename from src/hydro/viscosity.cpp rename to src/fluid/viscosity.cpp index 30a660ef..038976dd 100644 --- a/src/hydro/viscosity.cpp +++ b/src/fluid/viscosity.cpp @@ -12,6 +12,8 @@ #include "viscosity.hpp" #include "dataBlock.hpp" +#include "fluid.hpp" +#include "fargo.hpp" #define D_DX_I(q,n) (q(n,k,j,i) - q(n,k,j,i - 1)) @@ -36,62 +38,35 @@ #define D_DY_K(q,n) ( 0.25*(q(n,k,j + 1,i) + q(n,k - 1,j + 1,i)) \ - 0.25*(q(n,k,j - 1,i) + q(n,k - 1,j - 1,i))) -Viscosity::Viscosity() { - // Default constructor -} -void Viscosity::Init(Input &input, Grid &grid, Hydro *hydroin) { - idfx::pushRegion("Viscosity::Init"); - // Save the parent hydro object - this->hydro = hydroin; - - if(input.CheckEntry("Hydro","viscosity")>=0) { - if(input.Get("Hydro","viscosity",1).compare("constant") == 0) { - this->eta1 = input.Get("Hydro","viscosity",2); - // second viscosity? - this->eta2 = input.GetOrSet("Hydro","viscosity",3, 0.0); - this->haveViscosity = Constant; - } else if(input.Get("Hydro","viscosity",1).compare("userdef") == 0) { - this->haveViscosity = UserDefFunction; - this->eta1Arr = IdefixArray3D("ViscosityEta1Array",hydro->data->np_tot[KDIR], - hydro->data->np_tot[JDIR], - hydro->data->np_tot[IDIR]); - this->eta2Arr = IdefixArray3D("ViscosityEta1Array",hydro->data->np_tot[KDIR], - hydro->data->np_tot[JDIR], - hydro->data->np_tot[IDIR]); - } else { - IDEFIX_ERROR("Unknown viscosity definition in idefix.ini. " - "Can only be constant or userdef."); - } - } else { - IDEFIX_ERROR("I cannot create a Viscosity object without viscosity defined" - "in the .ini file"); - } +// This function is technically part of the constructor, +// but since constructors cannot be Lambda-captured by cuda +// we need an extra public function for array initialisation in the constructor + +void Viscosity::InitArrays() { // Allocate and fill arrays when needed #if GEOMETRY != CARTESIAN - one_dmu = IdefixArray1D("Viscosity_1dmu", hydro->data->np_tot[JDIR]); + one_dmu = IdefixArray1D("Viscosity_1dmu", data->np_tot[JDIR]); IdefixArray1D dmu = one_dmu; - IdefixArray1D th = hydro->data->x[JDIR]; - idefix_for("ViscousInitGeometry",1,hydro->data->np_tot[JDIR], + IdefixArray1D th = data->x[JDIR]; + idefix_for("ViscousInitGeometry",1,data->np_tot[JDIR], KOKKOS_LAMBDA(int j) { real scrch = FABS((1.0-cos(th(j)))*(sin(th(j)) >= 0.0 ? 1.0:-1.0) -(1.0-cos(th(j-1))) * (sin(th(j-1)) > 0.0 ? 1.0:-1.0)); dmu(j) = 1.0/scrch; }); #endif - viscSrc = IdefixArray4D("Viscosity_source", COMPONENTS, hydro->data->np_tot[KDIR], - hydro->data->np_tot[JDIR], - hydro->data->np_tot[IDIR]); - idfx::popRegion(); + viscSrc = IdefixArray4D("Viscosity_source", COMPONENTS, data->np_tot[KDIR], + data->np_tot[JDIR], + data->np_tot[IDIR]); } - void Viscosity::ShowConfig() { - if(haveViscosity==Constant) { - idfx::cout << "Viscosity: ENEABLED with constant viscosity eta1=" + if(status.status==Constant) { + idfx::cout << "Viscosity: ENABLED with constant viscosity eta1=" << this->eta1 << " and eta2=" << this->eta2 << " ."<< std::endl; - } else if (haveViscosity==UserDefFunction) { + } else if (status.status==UserDefFunction) { idfx::cout << "Viscosity: ENABLED with user-defined viscosity function." << std::endl; if(!viscousDiffusivityFunc) { @@ -100,9 +75,9 @@ void Viscosity::ShowConfig() { } else { IDEFIX_ERROR("Unknown viscosity mode"); } - if(hydro->viscosityStatus.isExplicit) { + if(this->status.isExplicit) { idfx::cout << "Viscosity: uses an explicit time integration." << std::endl; - } else if(hydro->viscosityStatus.isRKL) { + } else if(this->status.isRKL) { idfx::cout << "Viscosity: uses a Runge-Kutta-Legendre time integration." << std::endl; } else { @@ -111,63 +86,78 @@ void Viscosity::ShowConfig() { } void Viscosity::EnrollViscousDiffusivity(ViscousDiffusivityFunc myFunc) { - if(this->haveViscosity < UserDefFunction) { + if(this->status.status < UserDefFunction) { IDEFIX_WARNING("Viscous diffusivity enrollment requires Hydro/Viscosity " "to be set to userdef in .ini file"); } this->viscousDiffusivityFunc = myFunc; } -// This function computes the viscous flux and stores it in hydro->fluxRiemann +// This function computes the viscous flux and stores it in Flux // (this avoids an extra array) // Associated source terms, present in non-cartesian geometry are also computed // and stored in this->viscSrc for later use (in calcRhs). -void Viscosity::AddViscousFlux(int dir, const real t) { +void Viscosity::AddViscousFlux(int dir, const real t, const IdefixArray4D &Flux) { idfx::pushRegion("Viscosity::AddViscousFlux"); - IdefixArray4D Vc = this->hydro->Vc; - IdefixArray4D Flux = this->hydro->FluxRiemann; + IdefixArray4D Vc = this->Vc; IdefixArray4D viscSrc = this->viscSrc; - IdefixArray3D dMax = this->hydro->dMax; + IdefixArray3D dMax = this->dMax; IdefixArray3D eta1Arr = this->eta1Arr; IdefixArray3D eta2Arr = this->eta2Arr; IdefixArray1D one_dmu = this->one_dmu; - IdefixArray1D x1 = this->hydro->data->x[IDIR]; - IdefixArray1D x2 = this->hydro->data->x[JDIR]; - IdefixArray1D x3 = this->hydro->data->x[KDIR]; - IdefixArray1D x1l = this->hydro->data->xl[IDIR]; - IdefixArray1D x2l = this->hydro->data->xl[JDIR]; - IdefixArray1D x3l = this->hydro->data->xl[KDIR]; - IdefixArray1D dx1 = this->hydro->data->dx[IDIR]; - IdefixArray1D dx2 = this->hydro->data->dx[JDIR]; - IdefixArray1D dx3 = this->hydro->data->dx[KDIR]; + IdefixArray1D x1 = this->data->x[IDIR]; + IdefixArray1D x2 = this->data->x[JDIR]; + IdefixArray1D x3 = this->data->x[KDIR]; + IdefixArray1D x1l = this->data->xl[IDIR]; + IdefixArray1D x2l = this->data->xl[JDIR]; + IdefixArray1D x3l = this->data->xl[KDIR]; + IdefixArray1D dx1 = this->data->dx[IDIR]; + IdefixArray1D dx2 = this->data->dx[JDIR]; + IdefixArray1D dx3 = this->data->dx[KDIR]; + + // Fargo variables use to correct energy fluxes + #if HAVE_ENERGY + [[maybe_unused]] IdefixArray2D fargoVelocity; + [[maybe_unused]] Fargo::FargoType fargoType; + bool haveFargo; + #endif #if GEOMETRY == SPHERICAL - IdefixArray1D sinx2 = this->hydro->data->sinx2; - IdefixArray1D tanx2 = this->hydro->data->tanx2; - IdefixArray1D sinx2m = this->hydro->data->sinx2m; - IdefixArray1D tanx2m = this->hydro->data->tanx2m; + IdefixArray1D sinx2 = this->data->sinx2; + IdefixArray1D tanx2 = this->data->tanx2; + IdefixArray1D sinx2m = this->data->sinx2m; + IdefixArray1D tanx2m = this->data->tanx2m; #endif - HydroModuleStatus haveViscosity = this->haveViscosity; + HydroModuleStatus haveViscosity = this->status.status; // Compute viscosity if needed if(haveViscosity == UserDefFunction && dir == IDIR) { if(viscousDiffusivityFunc) { - viscousDiffusivityFunc(*this->hydro->data, t, eta1Arr, eta2Arr); + viscousDiffusivityFunc(*data, t, eta1Arr, eta2Arr); } else { IDEFIX_ERROR("No user-defined viscosity function has been enrolled"); } } + #if HAVE_ENERGY + haveFargo = this->data->haveFargo; + if(haveFargo) { + fargoVelocity = data->fargo->meanVelocity; + fargoType = data->fargo->type; + } + [[maybe_unused]] real sbS = this->sbS; + #endif + int ibeg, iend, jbeg, jend, kbeg, kend; - ibeg = this->hydro->data->beg[IDIR]; - iend = this->hydro->data->end[IDIR]; - jbeg = this->hydro->data->beg[JDIR]; - jend = this->hydro->data->end[JDIR]; - kbeg = this->hydro->data->beg[KDIR]; - kend = this->hydro->data->end[KDIR]; + ibeg = this->data->beg[IDIR]; + iend = this->data->end[IDIR]; + jbeg = this->data->beg[JDIR]; + jend = this->data->end[JDIR]; + kbeg = this->data->beg[KDIR]; + kend = this->data->end[KDIR]; real eta1Constant = this->eta1; @@ -367,9 +357,30 @@ void Viscosity::AddViscousFlux(int dir, const real t) { Flux(MX3, k, j, i) -= tau_xz; ) #if HAVE_ENERGY - Flux(ENG, k, j, i) -= EXPAND( 0.5*(Vc(VX1,k,j,i) + Vc(VX1,k,j,i-1))*tau_xx , - + 0.5*(Vc(VX2,k,j,i) + Vc(VX2,k,j,i-1))*tau_xy , - + 0.5*(Vc(VX3,k,j,i) + Vc(VX3,k,j,i-1))*tau_xz); + EXPAND( real vx1 = 0.5*(Vc(VX1,k,j,i) + Vc(VX1,k,j,i-1)); , + real vx2 = 0.5*(Vc(VX2,k,j,i) + Vc(VX2,k,j,i-1)); , + real vx3 = 0.5*(Vc(VX3,k,j,i) + Vc(VX3,k,j,i-1)); ) + + if(haveFargo) { + // fetch fargo velocity when required and substract from vxj since + // energy equation is computed for the deviations + [[maybe_unused]] real meanV = 0.0; + #if (GEOMETRY == POLAR || GEOMETRY == CARTESIAN) && DIMENSIONS >=2 + if(fargoType==Fargo::userdef) { + meanV = HALF_F*(fargoVelocity(k,i-1)+fargoVelocity(k,i)); + } else if(fargoType==Fargo::shearingbox) { + meanV = sbS * x1l(i); + } + vx2 = vx2 - meanV; + #elif GEOMETRY == SPHERICAL && DIMENSIONS ==3 + meanV = fargoVelocity(j,i); + vx3 = vx3 - meanV; + #endif + } + + Flux(ENG, k, j, i) -= EXPAND( vx1*tau_xx , + + vx2*tau_xy , + + vx3*tau_xz); #endif real locdmax = (FMAX(eta1,eta2))/(0.5*(Vc(RHO,k,j,i)+Vc(RHO,k,j,i-1))); @@ -498,6 +509,7 @@ void Viscosity::AddViscousFlux(int dir, const real t) { // tau_xy is initially cell centered since it is involved in the source term tau_xy = etaC1*( 0.5*(Vc(VX1,k,j+1,i)-Vc(VX1,k,j-1,i))/x1(i)/dx2(j) + +0.5*(Vc(VX2,k,j,i+1)-Vc(VX2,k,j,i-1))/dx1(i) - Vc(VX2,k,j,i)/x1(i)); tau_yy = 2.0*eta1*(dVyj/x1(i) + vx1i/x1(i)) + (eta2 - (2.0/3.0)*eta1)*divV; @@ -557,9 +569,31 @@ void Viscosity::AddViscousFlux(int dir, const real t) { Flux(MX3, k, j, i) -= tau_yz; ) #if HAVE_ENERGY - Flux(ENG, k, j, i) -= EXPAND( 0.5*(Vc(VX1,k,j,i) + Vc(VX1,k,j-1,i))*tau_xy , - + 0.5*(Vc(VX2,k,j,i) + Vc(VX2,k,j-1,i))*tau_yy , - + 0.5*(Vc(VX3,k,j,i) + Vc(VX3,k,j-1,i))*tau_yz); + + EXPAND( real vx1 = 0.5*(Vc(VX1,k,j-1,i) + Vc(VX1,k,j,i)); , + real vx2 = 0.5*(Vc(VX2,k,j-1,i) + Vc(VX2,k,j,i)); , + real vx3 = 0.5*(Vc(VX3,k,j-1,i) + Vc(VX3,k,j,i)); ) + + if(haveFargo) { + // fetch fargo velocity when required and substract from vxj since + // energy equation is computed for the deviations + [[maybe_unused]] real meanV = 0.0; + #if (GEOMETRY == POLAR || GEOMETRY == CARTESIAN) && DIMENSIONS >=2 + if(fargoType==Fargo::userdef) { + meanV = fargoVelocity(k,i); + } else if(fargoType==Fargo::shearingbox) { + meanV = sbS * x1(i); + } + vx2 = vx2 - meanV; + #elif GEOMETRY == SPHERICAL && DIMENSIONS ==3 + meanV = HALF_F*(fargoVelocity(j,i)+fargoVelocity(j-1,i)); + vx3 = vx3 - meanV; + #endif + } + + Flux(ENG, k, j, i) -= EXPAND( vx1*tau_xy , + + vx2*tau_yy , + + vx3*tau_yz); #endif real locdmax = (FMAX(eta1,eta2))/(0.5*(Vc(RHO,k,j,i)+Vc(RHO,k,j-1,i))); @@ -679,9 +713,31 @@ void Viscosity::AddViscousFlux(int dir, const real t) { Flux(MX3, k, j, i) -= tau_zz; ) #if HAVE_ENERGY - Flux(ENG, k, j, i) -= EXPAND( 0.5*(Vc(VX1,k,j,i) + Vc(VX1,k-1,j,i))*tau_xz , - + 0.5*(Vc(VX2,k,j,i) + Vc(VX2,k-1,j,i))*tau_yz , - + 0.5*(Vc(VX3,k,j,i) + Vc(VX3,k-1,j,i))*tau_zz); + + EXPAND( real vx1 = 0.5*(Vc(VX1,k-1,j,i) + Vc(VX1,k,j,i)); , + real vx2 = 0.5*(Vc(VX2,k-1,j,i) + Vc(VX2,k,j,i)); , + real vx3 = 0.5*(Vc(VX3,k-1,j,i) + Vc(VX3,k,j,i)); ) + + if(haveFargo) { + // fetch fargo velocity when required and substract from vxj since + // energy equation is computed for the deviations + [[maybe_unused]] real meanV = 0.0; + #if (GEOMETRY == POLAR || GEOMETRY == CARTESIAN) && DIMENSIONS >=2 + if(fargoType==Fargo::userdef) { + meanV = HALF_F*(fargoVelocity(k-1,i)+fargoVelocity(k,i)); + } else if(fargoType==Fargo::shearingbox) { + meanV = sbS * x1l(i); + } + vx2 = vx2 - meanV; + #elif GEOMETRY == SPHERICAL && DIMENSIONS ==3 + meanV = fargoVelocity(j,i); + vx3 = vx3 - meanV; + #endif + } + + Flux(ENG, k, j, i) -= EXPAND( vx1*tau_xz , + + vx2*tau_yz , + + vx3*tau_zz); #endif real locdmax = (FMAX(eta1,eta2))/(0.5*(Vc(RHO,k,j,i)+Vc(RHO,k-1,j,i))); diff --git a/src/fluid/viscosity.hpp b/src/fluid/viscosity.hpp new file mode 100644 index 00000000..6d1637f3 --- /dev/null +++ b/src/fluid/viscosity.hpp @@ -0,0 +1,97 @@ +// *********************************************************************************** +// Idefix MHD astrophysical code +// Copyright(C) Geoffroy R. J. Lesur +// and other code contributors +// Licensed under CeCILL 2.1 License, see COPYING for more information +// *********************************************************************************** + +#ifndef FLUID_VISCOSITY_HPP_ +#define FLUID_VISCOSITY_HPP_ + +#include + +#include "idefix.hpp" +#include "input.hpp" +#include "grid.hpp" +#include "fluid_defs.hpp" + + + +// Forward class hydro declaration +template class Fluid; +class DataBlock; + +using ViscousDiffusivityFunc = void (*) (DataBlock &, const real t, + IdefixArray3D &, IdefixArray3D &); + +class Viscosity { + public: + template + Viscosity(Input &, Grid &, Fluid *); + void ShowConfig(); // print configuration + void AddViscousFlux(int, const real, const IdefixArray4D &); + + // Enroll user-defined viscous diffusivity + void EnrollViscousDiffusivity(ViscousDiffusivityFunc); + + // Function for internal use (but public to allow for Cuda lambda capture) + void InitArrays(); + + IdefixArray4D viscSrc; // Source terms of the viscous operator + IdefixArray3D eta1Arr; + IdefixArray3D eta2Arr; + + // pre-computed geometrical factors in non-cartesian geometry + IdefixArray1D one_dmu; + + private: + DataBlock* data; + + // Viscosity status + ParabolicModuleStatus &status; + + ViscousDiffusivityFunc viscousDiffusivityFunc; + + IdefixArray4D &Vc; + IdefixArray3D &dMax; + + // constant diffusion coefficient (when needed) + real eta1, eta2; + + // Shearing box + real sbS; +}; + +#include "fluid.hpp" + +template +Viscosity::Viscosity(Input &input, Grid &grid, Fluid *hydroin): + Vc{hydroin->Vc}, + dMax{hydroin->dMax}, + status{hydroin->viscosityStatus} { + idfx::pushRegion("Viscosity::Viscosity"); + // Save the parent hydro object + this->data = hydroin->data; + this->sbS = hydroin->sbS; + + if(status.status == Constant) { + this->eta1 = input.Get(std::string(Phys::prefix),"viscosity",2); + // second viscosity? + this->eta2 = input.GetOrSet(std::string(Phys::prefix),"viscosity",3, 0.0); + } else if(status.status == UserDefFunction) { + this->eta1Arr = IdefixArray3D("ViscosityEta1Array",data->np_tot[KDIR], + data->np_tot[JDIR], + data->np_tot[IDIR]); + this->eta2Arr = IdefixArray3D("ViscosityEta1Array",data->np_tot[KDIR], + data->np_tot[JDIR], + data->np_tot[IDIR]); + } else { + IDEFIX_ERROR("Unknown viscosity definition in idefix.ini. " + "Can only be constant or userdef."); + } + + InitArrays(); + + idfx::popRegion(); +} +#endif // FLUID_VISCOSITY_HPP_ diff --git a/src/global.cpp b/src/global.cpp index ea82afd3..08091ca2 100644 --- a/src/global.cpp +++ b/src/global.cpp @@ -62,7 +62,10 @@ int initialize() { void pushRegion(const std::string& kName) { Kokkos::Profiling::pushRegion(kName); - + if(prof.perfEnabled) { + prof.currentRegion = prof.currentRegion->GetChild(kName); + prof.currentRegion->Start(); + } #ifdef DEBUG regionIndent=regionIndent+4; for(int i=0; i < regionIndent ; i++) { @@ -74,6 +77,10 @@ void pushRegion(const std::string& kName) { void popRegion() { Kokkos::Profiling::popRegion(); + if(prof.perfEnabled) { + prof.currentRegion->Stop(); + prof.currentRegion = prof.currentRegion->parent; + } #ifdef DEBUG for(int i=0; i < regionIndent ; i++) { cout << "-"; @@ -124,4 +131,18 @@ real randm(void) { return static_cast(static_cast(q) / static_cast(m)); } +void safeExit(int retCode) { + if(retCode != 0) { + #ifdef WITH_MPI + MPI_Abort(MPI_COMM_WORLD,retCode); + #endif + } else { + Kokkos::finalize(); + #ifdef WITH_MPI + MPI_Finalize(); + #endif + } + exit(retCode); +} + } // namespace idfx diff --git a/src/global.hpp b/src/global.hpp index 4a208b08..47fca42e 100644 --- a/src/global.hpp +++ b/src/global.hpp @@ -14,6 +14,7 @@ namespace idfx { int initialize(); // Initialisation routine for idefix real randm(); // Custom random number generator +void safeExit(int ); // Exit the code class IdefixOutStream; class IdefixErrStream; class Profiler; @@ -22,7 +23,7 @@ extern int prank; //< parallel rank extern int psize; extern IdefixOutStream cout; //< custom cout for idefix extern IdefixErrStream cerr; //< custom cerr for idefix -extern Profiler prof; //< profiler (for memory usage) +extern Profiler prof; //< profiler (for memory & performance usage) extern double mpiCallsTimer; //< time significant MPI calls extern LoopPattern defaultLoopPattern; //< default loop patterns (for idefix_for loops) extern bool warningsAreErrors; //< whether warnings should be considered as errors diff --git a/src/gravity/CMakeLists.txt b/src/gravity/CMakeLists.txt index 7babb289..ad258579 100644 --- a/src/gravity/CMakeLists.txt +++ b/src/gravity/CMakeLists.txt @@ -1,4 +1,8 @@ target_sources(idefix PUBLIC ${CMAKE_CURRENT_LIST_DIR}/gravity.hpp PUBLIC ${CMAKE_CURRENT_LIST_DIR}/gravity.cpp + PUBLIC ${CMAKE_CURRENT_LIST_DIR}/laplacian.cpp + PUBLIC ${CMAKE_CURRENT_LIST_DIR}/laplacian.hpp + PUBLIC ${CMAKE_CURRENT_LIST_DIR}/selfGravity.hpp + PUBLIC ${CMAKE_CURRENT_LIST_DIR}/selfGravity.cpp ) diff --git a/src/gravity/gravity.cpp b/src/gravity/gravity.cpp index f59a1a36..2492ce09 100644 --- a/src/gravity/gravity.cpp +++ b/src/gravity/gravity.cpp @@ -8,12 +8,16 @@ #include #include "gravity.hpp" +#include "planetarySystem.hpp" #include "dataBlock.hpp" #include "input.hpp" -void Gravity::Init(Input &input, DataBlock *datain) { +Gravity::Gravity(Input &input, DataBlock *datain) { + idfx::pushRegion("Gravity::Gravity"); this->data = datain; - data->haveGravity = true; + + // Gravitational constant G + this->gravCst = input.GetOrSet("Gravity","gravCst",0, 1.0); // Gravitational potential int nPotential = input.CheckEntry("Gravity","potential"); if(nPotential >=0) { @@ -27,12 +31,24 @@ void Gravity::Init(Input &input, DataBlock *datain) { this->centralMass = input.GetOrSet("Gravity","Mcentral",0, 1.0); } else if (potentialString.compare("selfgravity") == 0) { this->haveSelfGravityPotential = true; + } else if (potentialString.compare("planet") == 0) { + this->havePlanetsPotential = true; + if(!datain->haveplanetarySystem) { + IDEFIX_ERROR("You need to define a [Planet] block if" + "you want to work with embedded planets"); + } } else { IDEFIX_ERROR("Unknown type of gravitational potential in idefix.ini. "); } } } + // Automatically enables gravity if a planetary system was initialised. + if(datain->haveplanetarySystem) { + this->havePlanetsPotential = true; + this->havePotential = true; + } + // Body Force if(input.CheckEntry("Gravity","bodyForce")>=0) { std::string potentialString = input.Get("Gravity","bodyForce",0); @@ -44,8 +60,10 @@ void Gravity::Init(Input &input, DataBlock *datain) { } } + + // Allocate required arrays - if(havePotential && !haveInitialisedPotential) { + if(havePotential && (!haveInitialisedPotential)) { phiP = IdefixArray3D("Gravity_PhiP", data->np_tot[KDIR], data->np_tot[JDIR], data->np_tot[IDIR]); haveInitialisedPotential = true; @@ -55,11 +73,30 @@ void Gravity::Init(Input &input, DataBlock *datain) { data->np_tot[KDIR], data->np_tot[JDIR], data->np_tot[IDIR]); haveInitialisedBodyForce = true; } + + // TODO(mauxionj): the following should consider the possibility to enable SelfGravity + // even tought it is not explicitly asked by the user, to handle dependencies of other modules + // (e.g. planet module ?). Those dependencies are to be handled in the previous conditional block + // such that from this point, if SelfGravity is needed, haveSelfGravityPotential is true and + // we build the corresponding object if not done yet. + + // Check SelfGravity object + if(haveSelfGravityPotential) { + selfGravity.Init(input, this->data); + haveInitialisedSelfGravity = true; + } + + this->skipGravity = input.GetOrSet("Gravity","skip",0,1); + if(skipGravity<1) { + IDEFIX_ERROR("[Gravity]:skip should be a strictly positive integer"); + } + idfx::popRegion(); } void Gravity::ShowConfig() { if(data->haveGravity) { idfx::cout << "Gravity: ENABLED." << std::endl; + idfx::cout << "Gravity: G=" << gravCst << "." << std::endl; if(haveUserDefPotential) { idfx::cout << "Gravity: User-defined gravitational potential ENABLED." << std::endl; if(!gravPotentialFunc) { @@ -72,6 +109,10 @@ void Gravity::ShowConfig() { } if(haveSelfGravityPotential) { idfx::cout << "Gravity: self-gravity ENABLED." << std::endl; + selfGravity.ShowConfig(); + } + if(havePlanetsPotential) { + idfx::cout << "Gravity: planet(s) potential ENABLED." << std::endl; } if(haveBodyForce) { idfx::cout << "Gravity: user-defined body force ENABLED." << std::endl; @@ -79,10 +120,14 @@ void Gravity::ShowConfig() { IDEFIX_ERROR("No user-defined body force has been enrolled."); } } + if(skipGravity>1) { + idfx::cout << "Gravity: gravity field will be updated every " << skipGravity + << " cycles." << std::endl; + } } } // This function compute the gravitational field, using both body force and potential -void Gravity::ComputeGravity() { +void Gravity::ComputeGravity(int stepNumber) { idfx::pushRegion("Gravity::ComputeGravity"); if(havePotential) { if(haveUserDefPotential) { @@ -100,10 +145,14 @@ void Gravity::ComputeGravity() { AddCentralMassPotential(); } if(havePlanetsPotential) { - IDEFIX_ERROR("Planet potential not implemented. Ask GWF."); + data->planetarySystem->AddPlanetsPotential(phiP, data->t); } if(haveSelfGravityPotential) { - IDEFIX_ERROR("Self gravity potential not implemented. Ask JM."); + // Solving Poisson for the current gas density distribution + if(stepNumber % selfGravity.skipSelfGravity == 0) selfGravity.SolvePoisson(); + + // Adding gas self-gravity contribution to global gravity potential + selfGravity.AddSelfGravityPotential(phiP); } } if(haveBodyForce) { @@ -116,6 +165,14 @@ void Gravity::ComputeGravity() { bodyForceFunc(*data, data->t, bodyForceVector); idfx::popRegion(); } + + // For debug purpose + #ifdef DEBUG_GRAVITY + potTotFile.open("potTot.dat",std::ios::trunc); + this->selfGravity.WriteField(potTotFile,this->phiP); + potTotFile.close(); + #endif + idfx::popRegion(); } @@ -158,6 +215,7 @@ void Gravity::AddCentralMassPotential() { IdefixArray1D x3 = data->x[KDIR]; IdefixArray3D phiP = this->phiP; real mass = this->centralMass; + real gravCst = this->gravCst; #if GEOMETRY == CARTESIAN IDEFIX_ERROR("Central mass potential is not defined in cartesian geometry"); #endif @@ -178,7 +236,7 @@ void Gravity::AddCentralMassPotential() { #else r = ONE_F; // Make sure this one is initialized #endif - phiP(k,j,i) += -mass/r; + phiP(k,j,i) += -gravCst*mass/r; }); idfx::popRegion(); } diff --git a/src/gravity/gravity.hpp b/src/gravity/gravity.hpp index 4f12100b..aa0184aa 100644 --- a/src/gravity/gravity.hpp +++ b/src/gravity/gravity.hpp @@ -10,9 +10,9 @@ #include "idefix.hpp" #include "input.hpp" +#include "selfGravity.hpp" +#include "planetarySystem.hpp" -// Forward class hydro declaration -class Hydro; class DataBlock; // Enrolled functions signature @@ -25,8 +25,8 @@ using BodyForceFunc = void (*) (DataBlock &, const real t, IdefixArray4D&) class Gravity { public: - void Init(Input &, DataBlock*); ///< Initialisation - void ComputeGravity(); ///< compute gravitational field at current time t + Gravity(Input&, DataBlock*); + void ComputeGravity(int ); ///< compute gravitational field at current time t void EnrollPotential(GravPotentialFunc); void EnrollBodyForce(BodyForceFunc); @@ -51,11 +51,26 @@ class Gravity { // Bodyforce IdefixArray4D bodyForceVector; + // Self gravity + SelfGravity selfGravity; + + // JM : moved in public class to handle changing centralMass during computation + real centralMass{1.0}; ///< central mass parameter when central mass potential + ///< is enabled + + // Gravitational constant G + real gravCst{1.0}; + + // Whether we should skip gravity computation every n steps + int skipGravity{1}; + + private: + friend class PlanetarySystem; bool haveInitialisedPotential{false}; ///< whether a potential has already been initialised bool haveInitialisedBodyForce{false}; ///< whether a body force has already been initialised - real centralMass; ///< central mass parameter when central mass potential - ///< is enabled + bool haveInitialisedSelfGravity{false}; ///< whether self-gravity has already been initialised + DataBlock *data; // User defined gravitational potential @@ -63,6 +78,11 @@ class Gravity { // Body force BodyForceFunc bodyForceFunc{NULL}; + + #ifdef DEBUG_GRAVITY + // Used to get fields usefull for debugging + std::ofstream potTotFile; + #endif }; #endif // GRAVITY_GRAVITY_HPP_ diff --git a/src/gravity/laplacian.cpp b/src/gravity/laplacian.cpp new file mode 100644 index 00000000..8953eb4e --- /dev/null +++ b/src/gravity/laplacian.cpp @@ -0,0 +1,786 @@ +// *********************************************************************************** +// Idefix MHD astrophysical code +// Copyright(C) Geoffroy R. J. Lesur +// and other code contributors +// Licensed under CeCILL 2.1 License, see COPYING for more information +// *********************************************************************************** + + + +#include +#include +#include "laplacian.hpp" +#include "selfGravity.hpp" +#include "dataBlock.hpp" + + +Laplacian::Laplacian(DataBlock *datain, std::array leftBound, + std::array rightBound, + bool havePreconditionnerIn) { + // Save the parents data objects + this->data = datain; + this->havePreconditioner = havePreconditionnerIn; + + // init the grid elements using the parent datablock + this->np_tot = data->np_tot; + this->np_int = data->np_int; + this->nghost = data->nghost; + this->beg = data->beg; + this->end = data->end; + this->x = data->x; + this->dx = data->dx; + this->sinx2 = data->sinx2; + this->dV = data->dV; + this->A = data->A; + this->loffset = {0,0,0}; + this->roffset = {0,0,0}; + + this->lbound = leftBound; + this->rbound = rightBound; + + isPeriodic = true; + for(int dir = 0 ; dir < 3 ; dir++) { + if(lbound[dir] != LaplacianBoundaryType::periodic && + lbound[dir] != LaplacianBoundaryType::internalgrav) isPeriodic = false; + if(rbound[dir] != LaplacianBoundaryType::periodic && + rbound[dir] != LaplacianBoundaryType::internalgrav) isPeriodic = false; + } + + #ifdef WITH_MPI + if(lbound[IDIR] == origin) { + // create communicator for spherical radius + int remainDims[3] = {false, true, true}; + MPI_SAFE_CALL(MPI_Cart_sub(data->mygrid->CartComm, remainDims, &originComm)); + } + #endif + + + #if GEOMETRY == SPHERICAL + if ((this->rbound[JDIR]==axis) || (this->lbound[JDIR]==axis)) { + // Check wether the x3 spherical axis is full two pi + if(fabs((data->mygrid->xend[KDIR] - data->mygrid->xbeg[KDIR] -2.0*M_PI)) < 1e-10) { + this->isTwoPi = true; + } + + #ifdef WITH_MPI + // Check that there is no domain decomposition in phi + if(data->mygrid->nproc[KDIR]>1) { + IDEFIX_ERROR("Laplacian:: Axis boundaries are not compatible with " + "MPI domain decomposition in X3"); + } + #endif + } + #endif + + // Init MPI stack when needed + #ifdef WITH_MPI + this->arr4D = IdefixArray4D ("WorkingArrayMpi", 1, this->np_tot[KDIR], + this->np_tot[JDIR], + this->np_tot[IDIR]); + + int ntarget = 0; + std::vector mapVars; + mapVars.push_back(ntarget); + + this->mpi.Init(data->mygrid, mapVars, this->nghost.data(), this->np_int.data()); + #endif + + + if(this->lbound[IDIR] == origin) { + InitInternalGrid(); + } + + // Init preconditionner if needed + if(havePreconditioner) { + InitPreconditionner(); + } + // Initialise the Laplacian coefficients + PreComputeLaplacian(); +} + +void Laplacian::InitInternalGrid() { + idfx::pushRegion("Laplacian::InitInternalGrid"); + // Extend the grid so that the inner radius will be 1/10 of the initial inner radius + GridHost gh(*data->mygrid); + gh.SyncFromDevice(); + + real dx0 = gh.dx[IDIR](0); + real rin = gh.xbeg[IDIR]/10.0; + + loffset[IDIR] = (gh.xl[IDIR](0) - rin) / dx0; + + this->np_tot[IDIR] += loffset[IDIR]; + this->np_int[IDIR] += loffset[IDIR]; + this->end[IDIR] += loffset[IDIR]; + + // Allocate required arrays + this->x[IDIR] = IdefixArray1D("SG_x1", this->np_tot[IDIR]); + this->dx[IDIR] = IdefixArray1D("SG_dx1", this->np_tot[IDIR]); + this->dV = IdefixArray3D("SG_dV", this->np_tot[KDIR], + this->np_tot[JDIR], + this->np_tot[IDIR]); + for(int dir = 0 ; dir < 3 ; dir ++) { + this->A[dir] = IdefixArray3D("SG_A", this->np_tot[KDIR]+KOFFSET, + this->np_tot[JDIR]+JOFFSET, + this->np_tot[IDIR]+IOFFSET); + } + // xl and xr are used only temporarily in this subroutine. + auto x1l = IdefixArray1D("SG_x1l", this->np_tot[IDIR]); + auto x1r = IdefixArray1D("SG_x1r", this->np_tot[IDIR]); + // incidentally, sinx2 is not affected by a grid extension in x1, so no need to + // allocate a new array + + + // Fill selfgravity with the existing datablock grid + int ioffset = loffset[IDIR]; + + auto x1in = data->x[IDIR]; + auto x1lin = data->xl[IDIR]; + auto x1rin = data->xr[IDIR]; + auto dx1in = data->dx[IDIR]; + auto x1 = this->x[IDIR]; + auto dx1 = this->dx[IDIR]; + idefix_for("InternalGridCopy",0, data->np_tot[IDIR], + KOKKOS_LAMBDA(int i) { + x1(i+ioffset) = x1in(i); + dx1(i+ioffset) = dx1in(i); + x1l(i+ioffset) = x1lin(i); + x1r(i+ioffset) = x1rin(i); + }); + + // extend with a uniform grid and spacing equal to first of the datablock + idefix_for("InternalGridFill",0, ioffset, + KOKKOS_LAMBDA(int i) { + const real dx0 = dx1(ioffset); + dx1(i) = dx0; + x1(i) = x1(ioffset) + (i-ioffset)*dx0; + x1l(i) = x1l(ioffset) + (i-ioffset)*dx0; + x1r(i) = x1l(ioffset) + (i-ioffset+1)*dx0; + }); + + // Check that all is well + IdefixArray1D::HostMirror xH = Kokkos::create_mirror_view(x1l); + Kokkos::deep_copy(xH, x1l); + + if(xH(0)<0.0) { + IDEFIX_ERROR("SelfGravity: Your grid extension goes beyond the origin!"); + } + + IdefixArray3D dV = this->dV; + //IdefixArray1D dx1 = this->dx[IDIR]; + IdefixArray1D dx2 = this->dx[JDIR]; + IdefixArray1D dx3 = this->dx[KDIR]; + //IdefixArray1D x1 = this->x[IDIR]; + IdefixArray1D x2 = this->x[JDIR]; + IdefixArray1D x3 = this->x[KDIR]; + IdefixArray1D x1p = x1r; + IdefixArray1D x2p = data->xr[JDIR]; + IdefixArray1D x3p = data->xr[KDIR]; + IdefixArray1D x1m = x1l; + IdefixArray1D x2m = data->xl[JDIR]; + IdefixArray1D x3m = data->xl[KDIR]; + + // Fill the volume and area arrays (assume spherical geometry) + idefix_for("Volumes",0,this->np_tot[KDIR],0,this->np_tot[JDIR],0,this->np_tot[IDIR], + KOKKOS_LAMBDA (int k, int j, int i) { + real dVr = FABS(x1p(i)*x1p(i)*x1p(i) - x1m(i)*x1m(i)*x1m(i))/3.0; + real dmu = FABS(cos(x2m(j)) - cos(x2p(j))); + dV(k,j,i) = D_EXPAND( dVr, *dmu, *dx3(k)); + } + ); + + // Compute Areas + IdefixArray3D Ax1 = this->A[IDIR]; + IdefixArray3D Ax2 = this->A[JDIR]; + IdefixArray3D Ax3 = this->A[KDIR]; + + // X1 direction + int end = this->np_tot[IDIR]; + idefix_for("AreaX1",0,this->np_tot[KDIR],0,this->np_tot[JDIR],0,this->np_tot[IDIR]+IOFFSET, + KOKKOS_LAMBDA (int k, int j, int i) { + real dmu = FABS(cos(x2m(j)) - cos(x2p(j))); + if(i == end) { + Ax1(k,j,i) = D_EXPAND(x1p(i-1)*x1p(i-1), *dmu, *dx3(k)); + } else { + Ax1(k,j,i) = D_EXPAND(x1m(i)*x1m(i), *dmu, *dx3(k)); // r^2*dmu*dphi + } + }); + + // X2 direction + end = this->np_tot[JDIR]; + idefix_for("AreaX2",0,this->np_tot[KDIR],0,this->np_tot[JDIR]+JOFFSET,0,this->np_tot[IDIR], + KOKKOS_LAMBDA (int k, int j, int i) { + if (j == end) { + Ax2(k,j,i) = D_EXPAND(x1(i)*dx1(i), *FABS(sin(x2p(j-1))), *dx3(k)); + } else { + Ax2(k,j,i) = D_EXPAND(x1(i)*dx1(i), *FABS(sin(x2m(j))), *dx3(k)); // = r*dr*sin(thp)*dphi + } + }); + + // X3 direction + end = this->np_tot[KDIR]; + idefix_for("AreaX3",0,this->np_tot[KDIR]+KOFFSET,0,this->np_tot[JDIR],0,this->np_tot[IDIR], + KOKKOS_LAMBDA (int k, int j, int i) { + Ax3(k,j,i) = D_EXPAND(x1(i)*dx1(i), *dx2(j), *ONE_F); // = r*dr*dth + } + ); + + idfx::popRegion(); +} + +void Laplacian::InitPreconditionner() { + idfx::pushRegion("Laplacian::InitPreconditioner"); + IdefixArray3D P = IdefixArray3D ("Preconditionner", this->np_tot[KDIR], + this->np_tot[JDIR], + this->np_tot[IDIR]); + + int ibeg, iend, jbeg, jend, kbeg, kend; + ibeg = this->beg[IDIR]; + iend = this->end[IDIR]; + jbeg = this->beg[JDIR]; + jend = this->end[JDIR]; + kbeg = this->beg[KDIR]; + kend = this->end[KDIR]; + + // Set array to zero + idefix_for("ResetPrecond", kbeg, kend, jbeg, jend, ibeg, iend, + KOKKOS_LAMBDA (int k, int j, int i) { + P(k,j,i) = 0; + }); + + IdefixArray3D dV = this->dV; + + #if (GEOMETRY==CYLINDRICAL) || (GEOMETRY==POLAR) + IdefixArray1D r = this->x[IDIR]; + #elif GEOMETRY==SPHERICAL + IdefixArray1D r = this->x[IDIR]; + IdefixArray1D sinth = this->sinx2; + #endif + + // Loop on dimensions to get the diagonal elements + for(int dir = 0 ; dir < DIMENSIONS ; dir++) { + IdefixArray1D dx = this->dx[dir]; + IdefixArray3D Ax = this->A[dir]; + int ioffset = (dir == IDIR) ? 1 : 0; + int joffset = (dir == JDIR) ? 1 : 0; + int koffset = (dir == KDIR) ? 1 : 0; + + idefix_for("InitPrecond", kbeg, kend, jbeg, jend, ibeg, iend, + KOKKOS_LAMBDA (int k, int j, int i) { + int index = i*ioffset + j*joffset + k*koffset; + real dx0 = dx(index); + real dxm = dx(index-1); + real dxp = dx(index+1); + + real Ap = Ax(k+koffset, j+joffset, i+ioffset); + real Am = Ax(k,j,i); + + // Compute the diagonal elements coming from the laplacian + real d = -(2.0*Ap/(dx0+dxp) + 2.0*Am/(dx0+dxm)) / dV(k,j,i); + + // Handling curvature terms + real h1, h2, h3; + #if GEOMETRY==CARTESIAN + h1=1.; + h2=1.; + h3=1.; + #elif GEOMETRY==CYLINDRICAL + h1=1.; + h2=1.; + h3=r(i); // Should never be used (cf. Idefix docu) + #elif GEOMETRY==POLAR + h1=1.; + h2=r(i); + h3=1.; + #else + h1=1.; + h2=r(i); + h3=r(i)*sinth(j); + #endif + + // Full diagonal element (including curvature terms) + d = d/(h1*ioffset+h2*joffset+h3*koffset); + + // We store in P the sum of all these elements + P(k,j,i) += d; + }); + } + + // Store the array for future use + this->precond = P; + + idfx::popRegion(); +} + +void Laplacian::PreComputeLaplacian() { + idfx::pushRegion("Laplacian::PreComputeLaplacian"); + // Precompute Laplacian Factor + + // Allocate Laplacian factors + this->Lx1 = IdefixArray4D("SelfGravity_Lx1",2, + this->np_tot[KDIR], + this->np_tot[JDIR], + this->np_tot[IDIR]); + #if DIMENSIONS > 1 + this->Lx2 = IdefixArray4D("SelfGravity_Lx2",2, + this->np_tot[KDIR], + this->np_tot[JDIR], + this->np_tot[IDIR]); + + #if DIMENSIONS > 2 + this->Lx3 = IdefixArray4D("SelfGravity_Lx3",2, + this->np_tot[KDIR], + this->np_tot[JDIR], + this->np_tot[IDIR]); + #endif + #endif + + + // Local copy of Laplacian arrays + IdefixArray4D Lx1 = this->Lx1; // X stride + IdefixArray4D Lx2 = this->Lx2; // Y stride + IdefixArray4D Lx3 = this->Lx3; // Z stride + + + IdefixArray3D P = this->precond; + bool havePreconditioner = this->havePreconditioner; + IdefixArray1D dx1 = this->dx[IDIR]; + IdefixArray1D dx2 = this->dx[JDIR]; + IdefixArray1D dx3 = this->dx[KDIR]; + IdefixArray3D Ax1 = this->A[IDIR]; + IdefixArray3D Ax2 = this->A[JDIR]; + IdefixArray3D Ax3 = this->A[KDIR]; + IdefixArray3D dV = this->dV; + #if GEOMETRY == POLAR + IdefixArray1D r = this->x[IDIR]; + #elif GEOMETRY == SPHERICAL + IdefixArray1D r = this->x[IDIR]; + IdefixArray1D sinth = this->sinx2; + #endif + + idefix_for("L_Factor",KOFFSET,this->np_tot[KDIR]-KOFFSET, + JOFFSET,this->np_tot[JDIR]-JOFFSET, + IOFFSET,this->np_tot[IDIR]-IOFFSET, + KOKKOS_LAMBDA(int k, int j,int i) { + [[maybe_unused]] real h1, h2, h3; + #if GEOMETRY == CARTESIAN + h1 = 1.; + h2 = 1.; + h3 = 1.; + #elif GEOMETRY == POLAR + h1 = 1.; + h2 = r(i); + h3 = 1.; + #else + h1 = 1.; + h2 = r(i); + h3 = r(i) * sinth(j); + #endif + + // i-1 coefficient + Lx1(0,k,j,i) = 2.0 * Ax1(k, j, i) / h1 / (dx1(i) + dx1(i-1)) / dV(k,j,i); + // i+1 coefficient + Lx1(1,k,j,i) = 2.0 * Ax1(k, j, i+1) / h1 / (dx1(i+1) + dx1(i)) / dV(k,j,i); + if(havePreconditioner) { + Lx1(0,k,j,i) /= P(k,j,i); + Lx1(1,k,j,i) /= P(k,j,i); + } + #if DIMENSIONS > 1 + Lx2(0,k,j,i) = 2.0 * Ax2(k, j, i) / h2 / (dx2(j) + dx2(j-1)) / dV(k,j,i); + Lx2(1,k,j,i) = 2.0 * Ax2(k, j+1, i) / h2 / (dx2(j+1) + dx2(j)) / dV(k,j,i); + if(havePreconditioner) { + Lx2(0,k,j,i) /= P(k,j,i); + Lx2(1,k,j,i) /= P(k,j,i); + } + #if DIMENSIONS > 2 + Lx3(0,k,j,i) = 2.0 * Ax3(k, j, i) / h3 / (dx3(k) + dx3(k-1)) / dV(k,j,i); + Lx3(1,k,j,i) = 2.0 * Ax3(k+1, j, i) / h3 / (dx3(k+1) + dx3(k)) / dV(k,j,i); + if(havePreconditioner) { + Lx3(0,k,j,i) /= P(k,j,i); + Lx3(1,k,j,i) /= P(k,j,i); + } + #endif + #endif + }); + idfx::popRegion(); +} + + +void Laplacian::operator()(IdefixArray3D array, IdefixArray3D laplacian) { + idfx::pushRegion("Laplacian::ComputeLaplacian"); + + int ibeg, iend, jbeg, jend, kbeg, kend; + ibeg = this->beg[IDIR]; + iend = this->end[IDIR]; + jbeg = this->beg[JDIR]; + jend = this->end[JDIR]; + kbeg = this->beg[KDIR]; + kend = this->end[KDIR]; + IdefixArray4D Lx1 = this->Lx1; + #if DIMENSIONS > 1 + IdefixArray4D Lx2 = this->Lx2; + #if DIMENSIONS > 2 + IdefixArray4D Lx3 = this->Lx3; + #endif + #endif + + // Handling boundaries before laplacian calculation + this->SetBoundaries(array); + + idefix_for("FiniteDifference", kbeg, kend, jbeg, jend, ibeg, iend, + KOKKOS_LAMBDA (int k, int j, int i) { + real gc = 0; + real Delta = 0; + real Lm, Lr; + #if DIMENSIONS > 2 + Lm = Lx3(0,k,j,i); + Lr = Lx3(1,k,j,i); + gc += Lm + Lr; + Delta += array(k-1,j,i)*Lm + array(k+1,j,i)*Lr; + #endif + #if DIMENSIONS > 1 + Lm = Lx2(0,k,j,i); + Lr = Lx2(1,k,j,i); + gc += Lm + Lr; + Delta += array(k,j-1,i)*Lm + array(k,j+1,i)*Lr; + #endif + Lm = Lx1(0,k,j,i); + Lr = Lx1(1,k,j,i); + gc += Lm + Lr; + Delta += array(k,j,i-1)*Lm + array(k,j,i+1)*Lr; + + Delta -= gc * array(k,j,i); + + laplacian(k, j, i) = Delta; + }); + + idfx::popRegion(); +} + +void Laplacian::EnforceBoundary(int dir, BoundarySide side, LaplacianBoundaryType type, + IdefixArray3D &arr) { + idfx::pushRegion("Laplacian::EnforceBoundary"); + + IdefixArray3D localVar = arr; + + // Number of active cells + const int nxi = this->np_int[IDIR]; + const int nxj = this->np_int[JDIR]; + const int nxk = this->np_int[KDIR]; + + // Number of ghost cells + const int ighost = this->nghost[IDIR]; + const int jghost = this->nghost[JDIR]; + const int kghost = this->nghost[KDIR]; + + // Boundaries of the loop + const int ibeg = (dir == IDIR) ? side*(ighost+nxi) : 0; + const int iend = (dir == IDIR) ? ighost + side*(ighost+nxi) : this->np_tot[IDIR]; + const int jbeg = (dir == JDIR) ? side*(jghost+nxj) : 0; + const int jend = (dir == JDIR) ? jghost + side*(jghost+nxj) : this->np_tot[JDIR]; + const int kbeg = (dir == KDIR) ? side*(kghost+nxk) : 0; + const int kend = (dir == KDIR) ? kghost + side*(kghost+nxk) : this->np_tot[KDIR]; + + switch(type) { + case internalgrav: + // internal is used for MPI-enforced boundary conditions. Nothing to be done here. + break; + + case periodic: { + if(data->mygrid->nproc[dir] > 1) break; // Periodicity already enforced by MPI calls + + idefix_for("BoundaryPeriodic", kbeg, kend, jbeg, jend, ibeg, iend, + KOKKOS_LAMBDA (int k, int j, int i) { + int iref, jref, kref; + // This hack takes care of cases where we have more ghost zones than active zones + if(dir==IDIR) + iref = ighost + (i+ighost*(nxi-1))%nxi; + else + iref = i; + if(dir==JDIR) + jref = jghost + (j+jghost*(nxj-1))%nxj; + else + jref = j; + if(dir==KDIR) + kref = kghost + (k+kghost*(nxk-1))%nxk; + else + kref = k; + + localVar(k,j,i) = localVar(kref,jref,iref); + }); + break; + } + + case userdef: { + if(this->haveUserDefBoundary) { + // Warning: unlike hydro userdef boundary functions, the selfGravity + // userdef boundary functions take an additional argument arr which + // specifies the array for which boundaries are to be handled + this->userDefBoundaryFunc(*data, dir, side, data->t, arr); + } else { + IDEFIX_ERROR("Laplacian:: No function enrolled to define your own boundary conditions"); + } + break; + } + + case nullpot: { + idefix_for("BoundaryNullPot", kbeg, kend, jbeg, jend, ibeg, iend, + KOKKOS_LAMBDA (int k, int j, int i) { + localVar(k,j,i) = 0.0; + }); + break; + } + + case nullgrad: { + idefix_for("BoundaryNullGrad", kbeg, kend, jbeg, jend, ibeg, iend, + KOKKOS_LAMBDA (int k, int j, int i) { + const int iref = (dir==IDIR) ? ighost + side*(nxi-1) : i; + const int jref = (dir==JDIR) ? jghost + side*(nxj-1) : j; + const int kref = (dir==KDIR) ? kghost + side*(nxk-1) : k; + + localVar(k,j,i) = localVar(kref,jref,iref); + }); + break; + } + + case axis: { + // Handling specific loop boundaries + int jref, offset; + if(side == left) { + jref = this->beg[JDIR]; + offset = -1; + } + if(side == right) { + jref = this->end[JDIR]-1; + offset = 1; + } + + // NB: we assume no domain decomposition along phi here + + int np_int_k = this->np_int[KDIR]; + int nghost_k = this->nghost[KDIR]; + + if(this->isTwoPi) { + idefix_for("BoundaryAxis",kbeg,kend,jbeg,jend,ibeg,iend, + KOKKOS_LAMBDA (int k, int j, int i) { + int kcomp = nghost_k + (( k - nghost_k + np_int_k/2) % np_int_k); + // Assuming sVc=1 for a scalar + localVar(k,j,i) = localVar(kcomp, 2*jref-j+offset,i); + }); + } else { // not 2pi + idefix_for("BoundaryAxis",kbeg,kend,jbeg,jend,ibeg,iend, + KOKKOS_LAMBDA (int k, int j, int i) { + // kcomp = k by construction since we're doing a fraction of twopi + // Assuming sVc=1 for a scalar + localVar(k,j,i) = localVar(k, 2*jref-j+offset,i); + }); + } + break; + } + + case origin: { + // Assume the grid is extended inwards close to the origin. Hence the inner ghost + // all have the same value. + int iref = this->beg[IDIR]; + + real psiIn = 0.0; + + idefix_reduce("meanPsiIn", + beg[KDIR], end[KDIR], + beg[JDIR], end[JDIR], + KOKKOS_LAMBDA(int k, int j, real &psi) { + psi += localVar(k,j,iref); + },Kokkos::Sum (psiIn)); + + #ifdef WITH_MPI + MPI_Allreduce(MPI_IN_PLACE, &psiIn, 1, realMPI, MPI_SUM, originComm); + #endif + // Do a mean by dividing by the number of points + psiIn = psiIn/(data->mygrid->np_int[JDIR]*data->mygrid->np_int[KDIR]); + + // put this in the ghost cells + idefix_for("BoundaryOrigin",kbeg,kend,jbeg,jend,ibeg,iend, + KOKKOS_LAMBDA (int k, int j, int i) { + localVar(k,j,i) = psiIn; + }); + break; + } + + default: { + std::stringstream msg ("Laplacian:: Boundary condition type is not yet implemented"); + IDEFIX_ERROR(msg); + } + } + + idfx::popRegion(); +} + + +void Laplacian::EnrollUserDefBoundary(UserDefBoundaryFunc myFunc) { + this->userDefBoundaryFunc = myFunc; + this->haveUserDefBoundary = true; +} + +void Laplacian::SetBoundaries(IdefixArray3D &arr) { + idfx::pushRegion("Laplacian::SetBoundaries"); + + #ifdef WITH_MPI + this->arr4D = IdefixArray4D (arr.data(), 1, this->np_tot[KDIR], + this->np_tot[JDIR], + this->np_tot[IDIR]); + #endif + + for(int dir = 0 ; dir < DIMENSIONS ; dir++) { + // MPI Exchange data when needed + #ifdef WITH_MPI + if(data->mygrid->nproc[dir]>1) { + switch(dir) { + case 0: + this->mpi.ExchangeX1(this->arr4D); + break; + case 1: + this->mpi.ExchangeX2(this->arr4D); + break; + case 2: + this->mpi.ExchangeX3(this->arr4D); + break; + } + } + #endif + + EnforceBoundary(dir, left, this->lbound[dir], arr); + EnforceBoundary(dir, right, this->rbound[dir], arr); + } + + idfx::popRegion(); +} + +real Laplacian::ComputeCFL() { + idfx::pushRegion("Laplacian::ComputeCFL"); + + int ibeg = this->beg[IDIR]; + int iend = this->end[IDIR]; + int jbeg = this->beg[JDIR]; + int jend = this->end[JDIR]; + int kbeg = this->beg[KDIR]; + int kend = this->end[KDIR]; + + #if DIMENSIONS == 1 + IdefixArray1D dx1 = this->dx[IDIR]; + real dx1min2; + + idefix_reduce("GetMin1", + kbeg, kend, + jbeg, jend, + ibeg, iend, + KOKKOS_LAMBDA (int k, int j, int i, real &localMin) { + localMin = std::fmin(dx1(i) * dx1(i), localMin); + }, + Kokkos::Min(dx1min2)); + + // Reduction on the whole grid + #ifdef WITH_MPI + MPI_Allreduce(MPI_IN_PLACE, &dx1min2, 1, realMPI, MPI_MIN, MPI_COMM_WORLD); + #endif + + real dtmax = 1. / 2. * dx1min2; + + #elif DIMENSIONS == 2 + IdefixArray1D dx1 = this->dx[IDIR]; + IdefixArray1D dx2 = this->dx[JDIR]; + real dx1min2, dx2min2; + #if (GEOMETRY == POLAR || GEOMETRY == SPHERICAL) + IdefixArray1D r = this->x[IDIR]; + #endif + + idefix_reduce("GetMin1", + kbeg, kend, + jbeg, jend, + ibeg, iend, + KOKKOS_LAMBDA (int k, int j, int i, real &localMin) { + localMin = std::fmin(dx1(i) * dx1(i), localMin); + }, + Kokkos::Min(dx1min2)); + + idefix_reduce("GetMin2", + kbeg, kend, + jbeg, jend, + ibeg, iend, + KOKKOS_LAMBDA (int k, int j, int i, real &localMin) { + real dl = dx2(j); + #if (GEOMETRY == POLAR || GEOMETRY == SPHERICAL) + dl = dl * r(i); + #endif + localMin = std::fmin(dl * dl, localMin); + }, + Kokkos::Min(dx2min2)); + + // Reduction on the whole grid + #ifdef WITH_MPI + MPI_Allreduce(MPI_IN_PLACE, &dx1min2, 1, realMPI, MPI_MIN, MPI_COMM_WORLD); + MPI_Allreduce(MPI_IN_PLACE, &dx2min2, 1, realMPI, MPI_MIN, MPI_COMM_WORLD); + #endif + + real dtmax = 1. / 2. * 1. / ( 1. / dx1min2 + 1. / dx2min2); + + #else + IdefixArray1D dx1 = this->dx[IDIR]; + IdefixArray1D dx2 = this->dx[JDIR]; + IdefixArray1D dx3 = this->dx[KDIR]; + real dx1min2, dx2min2, dx3min2; + #if GEOMETRY == POLAR + IdefixArray1D r = x[IDIR]; + #elif GEOMETRY == SPHERICAL + IdefixArray1D r = x[IDIR]; + IdefixArray1D sinth = sinx2; + #endif + + idefix_reduce("GetMin1", + kbeg, kend, + jbeg, jend, + ibeg, iend, + KOKKOS_LAMBDA (int k, int j, int i, real &localMin) { + localMin = std::fmin(dx1(i) * dx1(i), localMin); + }, + Kokkos::Min(dx1min2)); + + idefix_reduce("GetMin2", + kbeg, kend, + jbeg, jend, + ibeg, iend, + KOKKOS_LAMBDA (int k, int j, int i, real &localMin) { + real dl = dx2(j); + #if (GEOMETRY == POLAR || GEOMETRY == SPHERICAL) + dl = dl * r(i); + #endif + localMin = std::fmin(dl * dl, localMin); + }, + Kokkos::Min(dx2min2)); + + idefix_reduce("GetMin3", // Cylindrical not taken into account as it shouldn't be used in 3D + kbeg, kend, + jbeg, jend, + ibeg, iend, + KOKKOS_LAMBDA (int k, int j, int i, real &localMin) { + real dl = dx3(k); + #if GEOMETRY == SPHERICAL + dl = dl * r(i) * sinth(j); + #endif + localMin = std::fmin(dl * dl, localMin); + }, + Kokkos::Min(dx3min2)); + + // Reduction on the whole grid + #ifdef WITH_MPI + MPI_Allreduce(MPI_IN_PLACE, &dx1min2, 1, realMPI, MPI_MIN, MPI_COMM_WORLD); + MPI_Allreduce(MPI_IN_PLACE, &dx2min2, 1, realMPI, MPI_MIN, MPI_COMM_WORLD); + MPI_Allreduce(MPI_IN_PLACE, &dx3min2, 1, realMPI, MPI_MIN, MPI_COMM_WORLD); + #endif + + real dtmax = 1. / 2. * 1. / ( 1. / dx1min2 + 1. / dx2min2 + 1. / dx3min2); + #endif + + idfx::popRegion(); + + return(0.95 * dtmax); // Taking a percentage to avoid dt=dtmax leading to a breakup +} diff --git a/src/gravity/laplacian.hpp b/src/gravity/laplacian.hpp new file mode 100644 index 00000000..4c41842b --- /dev/null +++ b/src/gravity/laplacian.hpp @@ -0,0 +1,99 @@ +// *********************************************************************************** +// Idefix MHD astrophysical code +// Copyright(C) Geoffroy R. J. Lesur +// and other code contributors +// Licensed under CeCILL 2.1 License, see COPYING for more information +// *********************************************************************************** + +#ifndef GRAVITY_LAPLACIAN_HPP_ +#define GRAVITY_LAPLACIAN_HPP_ + +#include +#include "idefix.hpp" +#ifdef WITH_MPI +#include "mpi.hpp" +#endif + +class DataBlock; + +class Laplacian { + public: + // Types of boundary which can be treated + enum LaplacianBoundaryType {internalgrav, periodic, nullgrad, nullpot, userdef, axis, origin}; + + Laplacian() = default; + Laplacian(DataBlock *, std::array, + std::array, bool ); + + void InitPreconditionner(); // For preconditionning versions + void PreComputeLaplacian(); // For faster Laplacian computation + + void InitInternalGrid(); // initialise the extra internal grid (for origin BCs) + + void SetBoundaries(IdefixArray3D &); // Set the proper boundaries for the given array + + real ComputeCFL(); // Compute the CFL associated to the Laplacian operator (for explicit schemes) + + // The main laplacian operator + void operator() (IdefixArray3D in, IdefixArray3D laplacian); + + // Handling userdef boundary. + using UserDefBoundaryFunc = void (*) (DataBlock &, int dir, BoundarySide side, + const real t, IdefixArray3D &arr); + void EnforceBoundary(int dir, BoundarySide side, LaplacianBoundaryType type, + IdefixArray3D &); + bool haveUserDefBoundary{false}; + + void EnrollUserDefBoundary(UserDefBoundaryFunc); // Enroll user-defined boundary conditions + + // User defined Boundary conditions + UserDefBoundaryFunc userDefBoundaryFunc{NULL}; + + // Local potential array size + std::array np_tot; + std::array np_int; + + std::array nghost; + std::array beg; + std::array end; + + // offset in the left and right direction between selfgravity grid and datablock grid + std::array loffset; + std::array roffset; + + // Grid for self-gravity solver (note that this grid may extend the grid of the current datablock) + std::array,3> x; ///< geometrical central points + std::array,3> dx; ///< cell width + + IdefixArray1D sinx2; ///< sinx2 (only in spherical) + + IdefixArray3D dV; ///< cell volume + std::array,3> A; ///< cell left interface area + + bool isPeriodic; // Periodicity status of the density distribution + std::array lbound; // Boundary condition to the left + std::array rbound; // Boundary condition to the right + // Warning : might differ from (M)HD solver ! + + IdefixArray3D precond; //< Diagonal preconditionner + IdefixArray4D Lx1; //< Laplacian operator in x1 + IdefixArray4D Lx2; //< Laplacian operator in x2 + IdefixArray4D Lx3; //< Laplacian operator in x3 + + bool isTwoPi{false}; + bool havePreconditioner{false}; // Use of preconditionner (or not) + + + DataBlock *data; + + #ifdef WITH_MPI + Mpi mpi; // Mpi object when WITH_MPI is set + IdefixArray4D arr4D; // Intermediate array for boundary handling + + MPI_Comm originComm; ///< MPI communicator used by the origin boundary condition + + #endif +}; + + +#endif // GRAVITY_LAPLACIAN_HPP_ diff --git a/src/gravity/selfGravity.cpp b/src/gravity/selfGravity.cpp new file mode 100644 index 00000000..67c0f834 --- /dev/null +++ b/src/gravity/selfGravity.cpp @@ -0,0 +1,517 @@ +// *********************************************************************************** +// Idefix MHD astrophysical code +// Copyright(C) Geoffroy R. J. Lesur +// and other code contributors +// Licensed under CeCILL 2.1 License, see COPYING for more information +// *********************************************************************************** + + + +#include +#include + +#include "selfGravity.hpp" +#include "dataBlock.hpp" +#include "fluid.hpp" +#include "vector.hpp" +#include "bicgstab.hpp" +#include "cg.hpp" +#include "minres.hpp" +#include "jacobi.hpp" + + +void SelfGravity::Init(Input &input, DataBlock *datain) { + idfx::pushRegion("SelfGravity::Init"); + + // Save the parents data objects + this->data = datain; + + // Initialise (default) solver parameters + this->dt = 0.; + this->isPeriodic = true; + + // Update targetError when provided + real targetError = input.GetOrSet("SelfGravity","targetError",0,1e-2); + + // Get maxiter when provided + real maxiter = input.GetOrSet("SelfGravity","maxIter",0,1000); + + // Get the number of skipped cycles when provided and check consistency + this->skipSelfGravity = input.GetOrSet("SelfGravity","skip",0,1); + if(skipSelfGravity<1) { + IDEFIX_ERROR("[SelfGravity]:skip should be a strictly positive integer"); + } + + // Get the gravity-related boundary conditions + for(int dir = 0 ; dir < 3 ; dir++) { + std::string label = std::string("boundary-X")+std::to_string(dir+1)+std::string("-beg"); + std::string boundary = input.Get("SelfGravity",label,0); + + if(boundary.compare("nullpot") == 0) { + this->lbound[dir] = Laplacian::LaplacianBoundaryType::nullpot; + this->isPeriodic = false; + } else if(boundary.compare("periodic") == 0) { + this->lbound[dir] = Laplacian::LaplacianBoundaryType::periodic; + } else if(boundary.compare("nullgrad") == 0) { + this->lbound[dir] = Laplacian::LaplacianBoundaryType::nullgrad; + this->isPeriodic = false; + } else if(boundary.compare("internalgrav") == 0) { + this->lbound[dir] = Laplacian::LaplacianBoundaryType::internalgrav; + this->isPeriodic = false; + } else if(boundary.compare("userdef") == 0) { + this->lbound[dir] = Laplacian::LaplacianBoundaryType::userdef; + this->isPeriodic = false; + } else if(boundary.compare("axis") == 0) { + this->lbound[dir] = Laplacian::LaplacianBoundaryType::axis; + this->isPeriodic = false; + } else if(boundary.compare("origin") == 0) { + this->lbound[dir] = Laplacian::LaplacianBoundaryType::origin; + this->isPeriodic = false; + // origin only compatible with spherical & axis=IDIR + #if GEOMETRY != SPHERICAL + IDEFIX_ERROR("Origin boundary conditions are working in spherical coordinates"); + #endif + if(dir != IDIR) { + IDEFIX_ERROR("Origin boundary conditions are meaningful only on the X1 direction"); + } + } else { + std::stringstream msg; + msg << "SelfGravity:: Unknown boundary type " << boundary; + IDEFIX_ERROR(msg); + } + + label = std::string("boundary-X")+std::to_string(dir+1)+std::string("-end"); + boundary = input.Get("SelfGravity",label,0); + if(boundary.compare("nullpot") == 0) { + this->rbound[dir] = Laplacian::LaplacianBoundaryType::nullpot; + this->isPeriodic = false; + } else if(boundary.compare("periodic") == 0) { + this->rbound[dir] = Laplacian::LaplacianBoundaryType::periodic; + } else if(boundary.compare("nullgrad") == 0) { + this->rbound[dir] = Laplacian::LaplacianBoundaryType::nullgrad; + this->isPeriodic = false; + } else if(boundary.compare("internalgrav") == 0) { + this->rbound[dir] = Laplacian::LaplacianBoundaryType::internalgrav; + this->isPeriodic = false; + } else if(boundary.compare("userdef") == 0) { + this->rbound[dir] = Laplacian::LaplacianBoundaryType::userdef; + this->isPeriodic = false; + } else if(boundary.compare("axis") == 0) { + this->rbound[dir] = Laplacian::LaplacianBoundaryType::axis; + this->isPeriodic = false; + } else { + std::stringstream msg; + msg << "SelfGravity:: Unknown boundary type " << boundary; + IDEFIX_ERROR(msg); + } + } + + // Update internal boundaries in case of domain decomposition + #ifdef WITH_MPI + for(int dir = 0 ; dir < DIMENSIONS ; dir++) { + if(data->mygrid->nproc[dir]>1) { + if(this->data->lbound[dir]==internal ) { + this->lbound[dir] = Laplacian::internalgrav; + } + if(this->data->rbound[dir]==internal) { + this->rbound[dir] = Laplacian::internalgrav; + } + } + } + #endif + + // Update solver when provided + if(input.CheckEntry("SelfGravity","solver") >= 0) { + std::string strSolver = input.Get("SelfGravity","solver",0); + if(strSolver.compare("Jacobi")==0) { + solver = JACOBI; + } else if(strSolver.compare("BICGSTAB")==0) { + solver = BICGSTAB; + } else if(strSolver.compare("PBICGSTAB")==0) { + solver = PBICGSTAB; + } else if(strSolver.compare("CG")==0) { + solver = CG; + } else if(strSolver.compare("PCG")==0) { + solver = PCG; + } else if(strSolver.compare("MINRES")==0) { + solver = MINRES; + } else if(strSolver.compare("PMINRES")==0) { + solver = PMINRES; + } else { + try { + // Try to use the old solver definition with integer (deprecated) + int s = std::stoi(strSolver); + if(s<0 || s > 2) throw std::runtime_error("Unknown solver number (should be 0,1 or 2)"); + this->solver = static_cast (s); + IDEFIX_DEPRECATED("The use of integer to define self-gravity solver is deprecated."); + } catch(const std::exception& e) { + std::stringstream msg; + msg << "SelfGravity: Unknown solver \"" << strSolver << "\"." + << "Use \"Jacobi\", \"BICGSTAB\" or \"PBICGSTAB\"." + << std::endl; + IDEFIX_ERROR(msg); + } + } + } else { + this->solver = BICGSTAB; + } + + // Enable preconditionner + if(this->solver==PBICGSTAB || this->solver == PCG || this->solver == PMINRES) { + this->havePreconditioner = true; + } + + // Make the Laplacian operator + laplacian = Laplacian(data, lbound, rbound, this->havePreconditioner ); + + np_tot = laplacian.np_tot; + + // Instantiate the bicgstab solver + if(solver == BICGSTAB || solver == PBICGSTAB) { + iterativeSolver = new Bicgstab(laplacian, targetError, maxiter, + laplacian.np_tot, laplacian.beg, laplacian.end); + } else if(solver == CG || solver == PCG) { + iterativeSolver = new Cg(laplacian, targetError, maxiter, + laplacian.np_tot, laplacian.beg, laplacian.end); + } else if(solver == MINRES || solver == PMINRES) { + iterativeSolver = new Minres(laplacian, + targetError, maxiter, + laplacian.np_tot, laplacian.beg, laplacian.end); + } else { + real step = laplacian.ComputeCFL(); + iterativeSolver = new Jacobi(laplacian, targetError, maxiter, step, + laplacian.np_tot, laplacian.beg, laplacian.end); + } + + + + // Arrays initialisation + this->density = IdefixArray3D ("Density", this->np_tot[KDIR], + this->np_tot[JDIR], + this->np_tot[IDIR]); + // Fill density array with 0 + { + auto d = this->density; + idefix_for("InitDensity",0,this->np_tot[KDIR],0,this->np_tot[JDIR],0,this->np_tot[IDIR], + KOKKOS_LAMBDA (int k, int j, int i) { + d(k,j,i) = 0.0; + }); + } + + this->potential = IdefixArray3D ("Potential", this->np_tot[KDIR], + this->np_tot[JDIR], + this->np_tot[IDIR]); + + + idfx::popRegion(); +} + + + +void SelfGravity::ShowConfig() { + idfx::cout << "SelfGravity: Using "; + switch(solver) { + case JACOBI: + idfx::cout << "Jacobi"; + break; + case BICGSTAB: + idfx::cout << "unpreconditionned BICGSTAB"; + break; + case PBICGSTAB: + idfx::cout << "preconditionned BICGSTAB"; + break; + case PCG: + idfx::cout << "preconditionned CG"; + break; + case CG: + idfx::cout << "unpreconditionned CG"; + break; + case MINRES: + idfx::cout << "unpreconditionned MinRes"; + break; + case PMINRES: + idfx::cout << "preconditionned MinRes"; + break; + default: + IDEFIX_ERROR("SelfGravity:: Unknown solver"); + } + idfx::cout << " solver." << std::endl; + // idfx::cout << "SelfGravity: target L2 norm error=" << targetError << "." << std::endl; + // idfx::cout << "SelfGravity: 4piG=" << gravCst << "." << std::endl; + + // The setup is periodic if it passes the previous boundary loading + if(this->isPeriodic == true) { + idfx::cout << "SelfGravity: Setup is periodic, using specific mass" + << " re-normalisation." << std::endl; + } + + if(this->lbound[IDIR] == Laplacian::LaplacianBoundaryType::origin) { + idfx::cout << "SelfGravity: using origin boundary with " << laplacian.loffset[IDIR] + << " additional radial points." << std::endl; + } + + if(this->skipSelfGravity>1) { + idfx::cout << "SelfGravity: self-gravity field will be updated every " << skipSelfGravity + << " cycles." << std::endl; + } + iterativeSolver->ShowConfig(); +} + + + +void SelfGravity::InitSolver() { + idfx::pushRegion("SelfGravity::InitSolver"); + + // Loading needed attributes + IdefixArray3D density = this->density; + IdefixArray4D Vc = data->hydro->Vc; + + // Initialise the density field + // todo: check bounds + int ioffset = laplacian.loffset[IDIR]; + int joffset = laplacian.loffset[JDIR]; + int koffset = laplacian.loffset[KDIR]; + + idefix_for("InitDensity", data->beg[KDIR], data->end[KDIR], + data->beg[JDIR], data->end[JDIR], + data->beg[IDIR], data->end[IDIR], + KOKKOS_LAMBDA (int k, int j, int i) { + density(k+koffset, j+joffset, i+ioffset) = Vc(RHO, k, j, i); + }); + + // Make sure that dust mass contributes to the self-gravitating field + if(data->haveDust) { + for(int i = 0 ; i < data->dust.size() ; i++) { + IdefixArray4D VcDust = data->dust[i]->Vc; + idefix_for("InitDustDensity", data->beg[KDIR], data->end[KDIR], + data->beg[JDIR], data->end[JDIR], + data->beg[IDIR], data->end[IDIR], + KOKKOS_LAMBDA (int k, int j, int i) { + density(k+koffset, j+joffset, i+ioffset) += VcDust(RHO, k, j, i); + }); + } + } + + // Deal with the mean issue for periodic density distribution + if(this->isPeriodic == true) { + SubstractMeanDensity(); // Remove density mean + } + + // divide density by preconditionner if we're doing the preconditionned version + if(havePreconditioner) { + int ibeg, iend, jbeg, jend, kbeg, kend; + ibeg = laplacian.beg[IDIR]; + iend = laplacian.end[IDIR]; + jbeg = laplacian.beg[JDIR]; + jend = laplacian.end[JDIR]; + kbeg = laplacian.beg[KDIR]; + kend = laplacian.end[KDIR]; + IdefixArray3D P = laplacian.precond; + idefix_for("Precond density", kbeg, kend, jbeg, jend, ibeg, iend, + KOKKOS_LAMBDA (int k, int j, int i) { + density(k, j, i) = density(k,j,i) / P(k,j,i); + }); + } + + // Look for Nans in the input field + int nanDensity = 0; + idefix_reduce("checkNanVc",0, this->np_tot[KDIR], 0, this->np_tot[JDIR], 0, this->np_tot[IDIR], + KOKKOS_LAMBDA (int k, int j, int i, int &nnan) { + if(std::isnan(density(k,j,i))) nnan++; + }, Kokkos::Sum(nanDensity) // reduction variable + ); + #ifdef WITH_MPI + MPI_Allreduce(MPI_IN_PLACE, &nanDensity,1,MPI_INT, MPI_SUM, MPI_COMM_WORLD); + #endif + + if(nanDensity>0) { + std::stringstream msg; + msg << "Input density in self-gravity contains "<< nanDensity << " NaNs" << std::endl; + throw std::runtime_error(msg.str()); + } + + + #ifdef DEBUG_GRAVITY + WriteField(rhoFile, density); + #endif + + idfx::popRegion(); +} + + +void SelfGravity::SubstractMeanDensity() { + idfx::pushRegion("SelfGravity::SubstractMeanDensity"); + + // Loading needed attributes + IdefixArray3D density = this->density; + IdefixArray3D dV = laplacian.dV; + + int ibeg, iend, jbeg, jend, kbeg, kend; + ibeg = laplacian.beg[IDIR]; + iend = laplacian.end[IDIR]; + jbeg = laplacian.beg[JDIR]; + jend = laplacian.end[JDIR]; + kbeg = laplacian.beg[KDIR]; + kend = laplacian.end[KDIR]; + + // Do the reduction on a vector + MyVector meanDensityVector; + + // Sum the density over the grid, weighted by cell volume + // and compute the total grid volume as a normalisation constant + // both stored in a 2D reduction vector + idefix_reduce("SumWeightedRho", + kbeg, kend, + jbeg, jend, + ibeg, iend, + KOKKOS_LAMBDA (int k, int j, int i, MyVector &localVector) { + localVector.v[0] += density(k,j,i) * dV(k,j,i); + localVector.v[1] += dV(k,j,i); + }, + Kokkos::Sum(meanDensityVector)); + + // Reduction on the whole grid + #ifdef WITH_MPI + MPI_Allreduce(MPI_IN_PLACE, &meanDensityVector.v, 2, realMPI, MPI_SUM, MPI_COMM_WORLD); + #endif + + real mean = meanDensityVector.v[0] / meanDensityVector.v[1]; + + // Remove the mean value of the density field + idefix_for("SubstractMeanDensity", + 0, this->np_tot[KDIR], + 0, this->np_tot[JDIR], + 0, this->np_tot[IDIR], + KOKKOS_LAMBDA (int k, int j, int i) { + density(k, j, i) -= mean; + }); + + #ifdef DEBUG_GRAVITY + idfx::cout << "SelfGravity:: Average value " << mean << " substracted to density" << std::endl; + #endif + + idfx::popRegion(); +} + +void SelfGravity::EnrollUserDefBoundary(Laplacian::UserDefBoundaryFunc myFunc) { + laplacian.EnrollUserDefBoundary(myFunc); +} + + + + +#ifdef DEBUG_GRAVITY +// This routine is for debugging purpose +void SelfGravity::WriteField(std::ofstream &stream, IdefixArray3D &in, int index) { + stream.precision(15); + stream << std::scientific;// << index; + for(int i = 0 ; i < this->np_tot[IDIR] ; i++) { + stream << in(0,0,i) << "\t"; + } + stream << std::endl; +} + +void SelfGravity::WriteField(std::ofstream &stream, IdefixArray1D &in, int index) { + stream.precision(15); + stream << std::scientific;// << index; + for(int i = 0 ; i < this->np_tot[IDIR] ; i++) { + stream << in(i) << "\t"; + } + stream << std::endl; +} +#endif + +void SelfGravity::SolvePoisson() { + idfx::pushRegion("SelfGravity::SolvePoisson"); + + Kokkos::Timer timer; + + elapsedTime -= timer.seconds(); + + #ifdef DEBUG_GRAVITY + rhoFile.open("rho.dat",std::ios::trunc); + potentialFile.open("potential.dat",std::ios::trunc); + geometryFile.open("geometry.dat",std::ios::trunc); + WriteField(geometryFile,this->x[IDIR]); + WriteField(geometryFile,this->dx[IDIR]); + WriteField(geometryFile,this->A[IDIR]); + WriteField(geometryFile,this->dV); + geometryFile.close(); + #endif + + InitSolver(); // (Re)initialise the solver + + this->nsteps = iterativeSolver->Solve(potential, density); + if (this->nsteps<0) { + idfx::cout << "SelfGravity:: BICGSTAB failed, resetting potential" << std::endl; + + // Look for Nans to explain the repetitive failing + if(data->CheckNan()>0) { + std::stringstream msg; + msg << "Nan found after BICGSTAB failed at time " << data->t << std::endl; + throw std::runtime_error(msg.str()); + } + + // Re-initialise potential + IdefixArray3D potential = this->potential; + + idefix_for("ResetPotential", + 0, this->np_tot[KDIR], + 0, this->np_tot[JDIR], + 0, this->np_tot[IDIR], + KOKKOS_LAMBDA (int k, int j, int i) { + potential(k, j, i) = ZERO_F; + }); + + // Try again ! + this->nsteps = iterativeSolver->Solve(this->potential, density); + if (this->nsteps<0) { + IDEFIX_ERROR("SelfGravity:: BICGSTAB failed despite restart"); + } + } + + currentError = iterativeSolver->GetError(); + + #ifdef DEBUG_GRAVITY + WriteField(potentialFile,potential,n); + if(this->convStatus == true) { + idfx::cout << "SelfGravity:: Reached convergence after " << n << " iterations" << std::endl; + rhoFile.close(); + potentialFile.close(); + } else if(n == maxiter) { + idfx::cout << "SelfGravity:: Reached max iter" << std::endl; + rhoFile.close(); + potentialFile.close(); + IDEFIX_WARNING("SelfGravity:: Failed to converge before reaching max iter"); + } + #endif + + elapsedTime += timer.seconds(); + idfx::popRegion(); +} + +void SelfGravity::AddSelfGravityPotential(IdefixArray3D &phiP) { + idfx::pushRegion("SelfGravity::AddSelfGravityPotential"); + + // Loading needed data + IdefixArray3D localPot = phiP; + IdefixArray3D potential = this->potential; + real gravCst = this->data->gravity->gravCst; + + // Updating ghost cells before to return potential + laplacian.SetBoundaries(potential); + + // Adding self-gravity contribution + int ioffset = laplacian.loffset[IDIR]; + int joffset = laplacian.loffset[JDIR]; + int koffset = laplacian.loffset[KDIR]; + idefix_for("AddSelfGravityPotential", 0, data->np_tot[KDIR], + 0, data->np_tot[JDIR], + 0, data->np_tot[IDIR], + KOKKOS_LAMBDA (int k, int j, int i) { + // Takes into account the unit conversion, scaled by the choice of gravCst + localPot(k, j, i) += 4.*M_PI*gravCst * potential(k+koffset, j+joffset, i+ioffset); + }); + + idfx::popRegion(); +} diff --git a/src/gravity/selfGravity.hpp b/src/gravity/selfGravity.hpp new file mode 100644 index 00000000..f38b2f5f --- /dev/null +++ b/src/gravity/selfGravity.hpp @@ -0,0 +1,80 @@ +// *********************************************************************************** +// Idefix MHD astrophysical code +// Copyright(C) Geoffroy R. J. Lesur +// and other code contributors +// Licensed under CeCILL 2.1 License, see COPYING for more information +// *********************************************************************************** + +#ifndef GRAVITY_SELFGRAVITY_HPP_ +#define GRAVITY_SELFGRAVITY_HPP_ + +#include + +#include "idefix.hpp" +#include "input.hpp" +#include "grid.hpp" +#include "fluid_defs.hpp" +#include "iterativesolver.hpp" +#include "laplacian.hpp" + +#ifdef WITH_MPI +#include "mpi.hpp" +#endif + +// Forward class hydro declaration +class DataBlock; + +class SelfGravity { + public: + enum GravitySolver {JACOBI, BICGSTAB, PBICGSTAB, PCG, CG, PMINRES, MINRES}; + + void Init(Input &, DataBlock *); // Initialisation of the class attributes + void ShowConfig(); // display current configuration + void InitSolver(); // (Re)initialisation of the solver for a given density distribution + + void SubstractMeanDensity(); // Compute and substract the average input density + + void SolvePoisson(); // Solve Poisson equation + void AddSelfGravityPotential(IdefixArray3D &); + + void EnrollUserDefBoundary(Laplacian::UserDefBoundaryFunc myFunc); // User-defined boundary + + IterativeSolver *iterativeSolver; + + // The linear operator involved in Poisson equation + Laplacian laplacian; + + #ifdef DEBUG_GRAVITY + // Used to get fields usefull for debugging + void WriteField(std::ofstream &, IdefixArray3D &, int = 0); + void WriteField(std::ofstream &, IdefixArray1D &, int = 0); + std::ofstream rhoFile; + std::ofstream potentialFile; + std::ofstream geometryFile; + #endif + + real currentError{0}; // last error of the iterative solver + int nsteps{0}; // # of steps of the latest iteration + double elapsedTime; // time spent solving self gravity + + // Whether we should skip self-gravity computation every n steps + int skipSelfGravity{1}; + + private: + DataBlock *data; // My parent data object + IdefixArray3D potential; // Gravitational potential + IdefixArray3D density; // Density + real dt; // CFL timestep + + // Local potential array size + std::array np_tot; + + std::array lbound; // Boundary condition to the left + std::array rbound; // Boundary condition to the right + + bool isPeriodic; + bool havePreconditioner{false}; + GravitySolver solver; // The solver used to solve Poisson +}; + +#endif // GRAVITY_SELFGRAVITY_HPP_ diff --git a/src/grid.cpp b/src/grid.cpp index 07c97062..14a1e174 100644 --- a/src/grid.cpp +++ b/src/grid.cpp @@ -11,14 +11,44 @@ #include "gridHost.hpp" #include "grid.hpp" +Grid::Grid(SubGrid * subgrid) { + idfx::pushRegion("Grid::Grid(SubGrid)"); + // Copy data from parent + x = subgrid->parentGrid->x; + xr = subgrid->parentGrid->xr; + xl = subgrid->parentGrid->xl; + dx = subgrid->parentGrid->dx; + + xbeg = subgrid->parentGrid->xbeg; + xend = subgrid->parentGrid->xend; + + np_tot = subgrid->parentGrid->np_tot; + np_int = subgrid->parentGrid->np_int; + + nghost = subgrid->parentGrid->nghost; + + lbound = subgrid->parentGrid->lbound; + rbound = subgrid->parentGrid->rbound; + + haveGridCoarsening = subgrid->parentGrid->haveGridCoarsening; + coarseningDirection = subgrid->parentGrid->coarseningDirection; + + nproc = subgrid->parentGrid->nproc; + xproc = subgrid->parentGrid->xproc; + + // Now slice if along the chosen direction + SliceMe(subgrid); + + idfx::popRegion(); +} + Grid::Grid(Input &input) { idfx::pushRegion("Grid::Grid(Input)"); - xbeg = std::vector(3); - xend = std::vector(3); - - nghost = std::vector(3); + #if GEOMETRY != CARTESIAN + isRegularCartesian = false; + #endif // Get grid size from input file, block [Grid] int npoints[3]; for(int dir = 0 ; dir < 3 ; dir++) { @@ -40,11 +70,6 @@ Grid::Grid(Input &input) { } } - np_tot = std::vector(3); - np_int = std::vector(3); - lbound = std::vector(3); - rbound = std::vector(3); - for(int dir = 0 ; dir < 3 ; dir++) { np_tot[dir] = npoints[dir] + 2*nghost[dir]; np_int[dir] = npoints[dir]; @@ -104,10 +129,6 @@ Grid::Grid(Input &input) { } // Allocate the grid structure on device. Initialisation will come from GridHost - x = std::vector>(3); - xr = std::vector>(3); - xl = std::vector>(3); - dx = std::vector>(3); for(int dir = 0 ; dir < 3 ; dir++) { x[dir] = IdefixArray1D("Grid_x",np_tot[dir]); xr[dir] = IdefixArray1D("Grid_xr",np_tot[dir]); @@ -116,8 +137,6 @@ Grid::Grid(Input &input) { } // Allocate proc structure (by default: one proc in each direction, size one) - nproc = std::vector (3); - xproc = std::vector (3); for(int i=0 ; i < 3; i++) { nproc[i] = 1; xproc[i] = 0; @@ -210,7 +229,8 @@ Grid::Grid(Input &input) { msg << "Grid coarsening can only be static or dynamic. I got: " << coarsenType; IDEFIX_ERROR(msg); } - this->coarseningDirection = std::vector(3, false); + this->coarseningDirection = {false, false, false}; + int directions = input.CheckEntry("Grid","coarsening"); for(int i = 1 ; i < directions ; i++) { std::string dirname = input.Get("Grid","coarsening",i); @@ -380,3 +400,47 @@ void Grid::ShowConfig() { idfx::cout << std::endl; } } + +void Grid::SliceMe(SubGrid *subgrid) { + // Slice this grid along the direction in subgrid + int dir = subgrid->direction; + + auto x = IdefixArray1D("Grid_x",1); + auto xr = IdefixArray1D("Grid_xr",1); + auto xl = IdefixArray1D("Grid_xl",1); + auto dx = IdefixArray1D("Grid_dx",1); + + auto xorigin = subgrid->parentGrid->x[dir]; + auto xrorigin = subgrid->parentGrid->xr[dir]; + auto xlorigin = subgrid->parentGrid->xl[dir]; + auto dxorigin = subgrid->parentGrid->dx[dir]; + + idefix_for("Slice_grid", subgrid->index, subgrid->index+1, + KOKKOS_LAMBDA(int i) { + x(0) = xorigin(i); + xr(0) = xrorigin(i); + xl(0) = xlorigin(i); + dx(0) = dxorigin(i); + }); + + // Replace the arrays in the current subgrid + this->x[dir] = x; + this->xr[dir] = xr; + this->xl[dir] = xl; + this->dx[dir] = dx; + + this->nghost[dir] = 0; + this->np_tot[dir] = 1; + this->np_int[dir] = 1; + this->xbeg[dir] = subgrid->x0; + this->xend[dir] = subgrid->x0; + + // Slice the MPI communicator + #ifdef WITH_MPI + int remainDims[3] = {true, true, true}; + remainDims[dir] = false; + MPI_Cart_sub(subgrid->parentGrid->CartComm, remainDims, &this->CartComm); + nproc[dir] = 1; + xproc[dir] = 0; + #endif +} diff --git a/src/grid.hpp b/src/grid.hpp index 25f7ca62..0ffdb3a5 100644 --- a/src/grid.hpp +++ b/src/grid.hpp @@ -8,6 +8,7 @@ #ifndef GRID_HPP_ #define GRID_HPP_ #include +#include #include "idefix.hpp" #include "input.hpp" @@ -19,33 +20,34 @@ /// the arrays of a Grid are on the device. If a Host access is needed, /// it is recommended to use a GridHost instance from this Grid, and sync it. ///////////////////////////////////////////////////////////////////////////////////////////////// +class SubGrid; // Forward class declaration class Grid { public: - std::vector> x; ///< geometrical central points - std::vector> xr; ///< cell right interface - std::vector> xl; ///< cell left interface - std::vector> dx; ///< cell width + std::array,3> x; ///< geometrical central points + std::array,3> xr; ///< cell right interface + std::array,3> xl; ///< cell left interface + std::array,3> dx; ///< cell width - std::vector xbeg; ///< Beginning of grid - std::vector xend; ///< End of grid + std::array xbeg; ///< Beginning of grid + std::array xend; ///< End of grid - std::vector np_tot; ///< total number of grid points (including ghosts) - std::vector np_int; ///< internal number of grid points (excluding ghosts) + std::array np_tot; ///< total number of grid points (including ghosts) + std::array np_int; ///< internal number of grid points (excluding ghosts) - std::vector nghost; ///< number of ghost cells - std::vector lbound; ///< Boundary condition to the left - std::vector rbound; ///< Boundary condition to the right + std::array nghost; ///< number of ghost cells + std::array lbound; ///< Boundary condition to the left + std::array rbound; ///< Boundary condition to the right bool haveAxis{false}; ///< Do we require a special treatment of the axis in spherical coords? - + bool isRegularCartesian{true}; ///< whether the grid is regular and cartesian (true) or not GridCoarsening haveGridCoarsening{GridCoarsening::disabled}; ///< Is grid coarsening enabled? - std::vector coarseningDirection; ///< whether a coarsening is used in each direction + std::array coarseningDirection; ///< whether a coarsening is used in each direction // MPI data - std::vector nproc; /// xproc; /// nproc; /// xproc; ///x[direction]); + Kokkos::deep_copy(x,parentGrid->x[direction]); + int iref = -1; + for(int i = 0 ; i < x.extent(0) - 1 ; i++) { + if(x(i+1) >= x0) { + if(x(i+1) - x0 > x0 - x(i) ) { + // we're closer to x(i) + iref = i; + } else { + iref = i+1; + } + break; + } + } + if(iref < 0) { + std::stringstream msg; + msg << "Cannot generate a subgrid in direction " << d << "with x0=" << x0 + << "." << std::endl + << "Bounds are " << x(0) << "..." << x(x.extent(0)-1) << std::endl; + + IDEFIX_ERROR(msg); + } + this->index = iref; + this->x0 = x(iref); + + this->grid = std::make_unique(this); + idfx::popRegion(); + } + + Grid *parentGrid; + std::unique_ptr grid; + + const SliceType type; + const int direction; + real x0; ///< Cell center coordinate of the slice/average + int index; ///< index in parent grid of the slice/average +}; + #endif // GRID_HPP_ diff --git a/src/gridHost.cpp b/src/gridHost.cpp index fcded9a6..1c48b53c 100644 --- a/src/gridHost.cpp +++ b/src/gridHost.cpp @@ -28,11 +28,9 @@ GridHost::GridHost(Grid &grid) { haveAxis = grid.haveAxis; + isRegularCartesian = grid.isRegularCartesian; + // Create mirrors on host - x = std::vector::HostMirror> (3); - xr = std::vector::HostMirror> (3); - xl = std::vector::HostMirror> (3); - dx = std::vector::HostMirror> (3); for(int dir = 0 ; dir < 3 ; dir++) { x[dir] = Kokkos::create_mirror_view(grid.x[dir]); xr[dir] = Kokkos::create_mirror_view(grid.xr[dir]); @@ -88,7 +86,7 @@ void GridHost::MakeGrid(Input &input) { } } else if(patchType.compare("l")==0) { // log-increasing patch - + isRegularCartesian = false; double alpha = (patchEnd + fabs(patchStart) - patchStart)/fabs(patchStart); for(int i = 0 - ghostStart ; i < patchSize + ghostEnd ; i++) { @@ -125,6 +123,7 @@ void GridHost::MakeGrid(Input &input) { if(patch == numPatch-1) ghostEnd = nghost[dir]; // Define the grid depending on patch type if((patchType.compare("s+")==0)||(patchType.compare("s-")==0)) { + isRegularCartesian = false; // Stretched grid // - means we take the initial dx on the left side, + on the right side // refPatch corresponds to the patch from which we compute the initial dx @@ -217,10 +216,15 @@ if(haveAxis) { #if DIMENSIONS < 2 IDEFIX_ERROR("Axis Boundaries requires at least two dimenions"); #endif - if((fabs(xbeg[JDIR])>1e-10) && (lbound[JDIR] == axis)) { + #ifdef SINGLE_PRECISION + const real smallNumber = 1e-5; + #else + const real smallNumber = 1e-10; + #endif + if((fabs(xbeg[JDIR])>smallNumber) && (lbound[JDIR] == axis)) { IDEFIX_ERROR("Axis Boundaries requires your X2 domain to start at exactly x2=0.0"); } - if((fabs(xend[JDIR]-M_PI)>1e-10) && (rbound[JDIR] == axis )) { + if((fabs(xend[JDIR]-M_PI)>smallNumber) && (rbound[JDIR] == axis )) { IDEFIX_ERROR("Axis Boundaries requires your X2 domain to end at exactly x2=Pi"); } // Enforce symmetry of theta grid spacing when we cross the axis @@ -257,6 +261,7 @@ void GridHost::SyncFromDevice() { xbeg = grid->xbeg; xend = grid->xend; + isRegularCartesian = grid->isRegularCartesian; idfx::popRegion(); } @@ -274,6 +279,7 @@ void GridHost::SyncToDevice() { grid->xbeg = xbeg; grid->xend = xend; + grid->isRegularCartesian = isRegularCartesian; idfx::popRegion(); } diff --git a/src/gridHost.hpp b/src/gridHost.hpp index bf48febd..065c65af 100644 --- a/src/gridHost.hpp +++ b/src/gridHost.hpp @@ -22,22 +22,23 @@ class GridHost { public: - std::vector::HostMirror> x; ///< geometrical central points - std::vector::HostMirror> xr; ///< cell right interface - std::vector::HostMirror> xl; ///< cell left interface - std::vector::HostMirror> dx; ///< cell width + std::array::HostMirror,3> x; ///< geometrical central points + std::array::HostMirror,3> xr; ///< cell right interface + std::array::HostMirror,3> xl; ///< cell left interface + std::array::HostMirror,3> dx; ///< cell width - std::vector xbeg; ///< Beginning of grid - std::vector xend; ///< End of grid + std::array xbeg; ///< Beginning of grid + std::array xend; ///< End of grid - std::vector np_tot; ///< total number of grid points - std::vector np_int; ///< internal number of grid points + std::array np_tot; ///< total number of grid points + std::array np_int; ///< internal number of grid points - std::vector nghost; ///< number of ghost cells - std::vector lbound; ///< Boundary condition to the left - std::vector rbound; ///< Boundary condition to the right + std::array nghost; ///< number of ghost cells + std::array lbound; ///< Boundary condition to the left + std::array rbound; ///< Boundary condition to the right bool haveAxis=false; ///< Do we require a special treatment of the axis in spherical coords? + bool isRegularCartesian; ///< whether the grid is regular and cartesian or not explicit GridHost(Grid&); ///< Constructor from a corresponding Grid on the Device. ///< (NB: this constructor does not sync any data) diff --git a/src/hydro/CMakeLists.txt b/src/hydro/CMakeLists.txt deleted file mode 100644 index 7e832c22..00000000 --- a/src/hydro/CMakeLists.txt +++ /dev/null @@ -1,37 +0,0 @@ -add_subdirectory(boundary) -add_subdirectory(electromotiveforce) - -if(Idefix_MHD) - add_subdirectory(MHDsolvers) -else() - add_subdirectory(HDsolvers) -endif() - - -target_sources(idefix - PUBLIC ${CMAKE_CURRENT_LIST_DIR}/addNonIdealMHDFlux.hpp - PUBLIC ${CMAKE_CURRENT_LIST_DIR}/addSourceTerms.cpp - PUBLIC ${CMAKE_CURRENT_LIST_DIR}/axis.cpp - PUBLIC ${CMAKE_CURRENT_LIST_DIR}/axis.hpp - PUBLIC ${CMAKE_CURRENT_LIST_DIR}/calcCurrent.cpp - PUBLIC ${CMAKE_CURRENT_LIST_DIR}/calcParabolicFlux.hpp - PUBLIC ${CMAKE_CURRENT_LIST_DIR}/calcRiemannFlux.hpp - PUBLIC ${CMAKE_CURRENT_LIST_DIR}/calcRightHandSide.hpp - PUBLIC ${CMAKE_CURRENT_LIST_DIR}/checkDivB.cpp - PUBLIC ${CMAKE_CURRENT_LIST_DIR}/coarsenFlow.cpp - PUBLIC ${CMAKE_CURRENT_LIST_DIR}/convertConsToPrim.cpp - PUBLIC ${CMAKE_CURRENT_LIST_DIR}/convertConsToPrimHD.hpp - PUBLIC ${CMAKE_CURRENT_LIST_DIR}/convertConsToPrimMHD.hpp - PUBLIC ${CMAKE_CURRENT_LIST_DIR}/slopeLimiter.hpp - PUBLIC ${CMAKE_CURRENT_LIST_DIR}/shockFlattening.hpp - PUBLIC ${CMAKE_CURRENT_LIST_DIR}/shockFlattening.cpp - PUBLIC ${CMAKE_CURRENT_LIST_DIR}/fluxHD.hpp - PUBLIC ${CMAKE_CURRENT_LIST_DIR}/fluxMHD.hpp - PUBLIC ${CMAKE_CURRENT_LIST_DIR}/hydro_defs.hpp - PUBLIC ${CMAKE_CURRENT_LIST_DIR}/hydro.cpp - PUBLIC ${CMAKE_CURRENT_LIST_DIR}/hydro.hpp - PUBLIC ${CMAKE_CURRENT_LIST_DIR}/viscosity.hpp - PUBLIC ${CMAKE_CURRENT_LIST_DIR}/viscosity.cpp - PUBLIC ${CMAKE_CURRENT_LIST_DIR}/thermalDiffusion.hpp - PUBLIC ${CMAKE_CURRENT_LIST_DIR}/thermalDiffusion.cpp - ) diff --git a/src/hydro/addNonIdealMHDFlux.hpp b/src/hydro/addNonIdealMHDFlux.hpp deleted file mode 100644 index fe27e84e..00000000 --- a/src/hydro/addNonIdealMHDFlux.hpp +++ /dev/null @@ -1,296 +0,0 @@ -// *********************************************************************************** -// Idefix MHD astrophysical code -// Copyright(C) Geoffroy R. J. Lesur -// and other code contributors -// Licensed under CeCILL 2.1 License, see COPYING for more information -// *********************************************************************************** - -#ifndef HYDRO_ADDNONIDEALMHDFLUX_HPP_ -#define HYDRO_ADDNONIDEALMHDFLUX_HPP_ - -#include "hydro.hpp" -#include "dataBlock.hpp" - -// Compute parabolic fluxes -template -void Hydro::AddNonIdealMHDFlux(const real t) { - idfx::pushRegion("Hydro::addNonIdealMHDFlux"); - -#if MHD == YES - - int ioffset,joffset,koffset; - - IdefixArray4D Flux = this->FluxRiemann; - IdefixArray4D Vc = this->Vc; - IdefixArray4D Vs = this->Vs; - IdefixArray3D dMax = this->dMax; - IdefixArray4D J = this->J; - IdefixArray3D etaArr = this->etaOhmic; - IdefixArray3D xAmbiArr = this->xAmbipolar; - - // these two are required to ensure that the type is captured by KOKKOS_LAMBDA - HydroModuleStatus resistivity = resistivityStatus.status; - HydroModuleStatus ambipolar = ambipolarStatus.status; - - bool haveResistivity{false}; - bool haveAmbipolar{false}; - - if(data->rklCycle) { - haveResistivity = resistivityStatus.isRKL; - haveAmbipolar = ambipolarStatus.isRKL; - } else { - haveResistivity = resistivityStatus.isExplicit; - haveAmbipolar = ambipolarStatus.isExplicit; - } - - real etaConstant = this->etaO; - real xAConstant = this->xA; - - ioffset=joffset=koffset=0; - - switch(dir) { - case(IDIR): - ioffset = 1; - break; - case(JDIR): - joffset = 1; - break; - case(KDIR): - koffset=1; - break; - default: - IDEFIX_ERROR("Wrong direction"); - } - - // Load the diffusivity array when required - if(resistivity == UserDefFunction && dir == IDIR) { - if(ohmicDiffusivityFunc) - ohmicDiffusivityFunc(*data, t, etaArr); - else - IDEFIX_ERROR("No user-defined Ohmic diffusivity function has been enrolled"); - } - - if(ambipolar == UserDefFunction && dir == IDIR) { - if(ambipolarDiffusivityFunc) - ambipolarDiffusivityFunc(*data, t, xAmbiArr); - else - IDEFIX_ERROR("No user-defined ambipolar diffusivity function has been enrolled"); - } - - // Note the flux follows the same sign convention as the hyperbolic flux - // HEnce signs are reversed compared to the parabolic fluxes found in Pluto 4.3 - idefix_for("CalcParabolicFlux", - data->beg[KDIR],data->end[KDIR]+koffset, - data->beg[JDIR],data->end[JDIR]+joffset, - data->beg[IDIR],data->end[IDIR]+ioffset, - KOKKOS_LAMBDA (int k, int j, int i) { - real Jx1, Jx2, Jx3; - real Bx1, Bx2, Bx3; - real eta,xA; - real locdmax = 0.0; - - int ip1; // Offset indieces - [[maybe_unused]] int jp1, kp1; - ip1=i+1; -#if DIMENSIONS >=2 - jp1 = j+1; -#else - jp1=j; -#endif -#if DIMENSIONS == 3 - kp1 = k+1; -#else - kp1 = k; -#endif - - - Jx1=Jx2=Jx3=ZERO_F; - - if(resistivity == Constant) - eta = etaConstant; - - if(ambipolar == Constant) - xA = xAConstant; - - if(dir==IDIR) { - EXPAND( , - Jx3 = AVERAGE_4D_Y(J, KDIR, k, jp1, i); , - Jx2 = AVERAGE_4D_Z(J, JDIR, kp1, j, i); - Jx1 = AVERAGE_4D_XYZ(J, IDIR, kp1, jp1, i); ) - - EXPAND( Bx1 = Vs(BX1s,k,j,i); , - Bx2 = HALF_F*( Vc(BX2,k,j,i-1) + Vc(BX2,k,j,i)); , - Bx3 = HALF_F*( Vc(BX3,k,j,i-1) + Vc(BX3,k,j,i)); ) - if(haveResistivity) { - if(resistivity == UserDefFunction) - eta = AVERAGE_3D_X(etaArr,k,j,i); - - // Do not update BX2 if BX2s is defined - #if (DIMENSIONS < 2 && COMPONENTS >= 2) - Flux(BX2,k,j,i) += - eta * Jx3; - #endif - #if (DIMENSIONS < 3 && COMPONENTS == 3) - Flux(BX3,k,j,i) += eta * Jx2; - #endif - - #if HAVE_ENERGY - Flux(ENG,k,j,i) += EXPAND( ZERO_F , - - Bx2 * eta * Jx3 , - + Bx3 * eta * Jx2 ); - #endif - - locdmax += eta; - } - - if(haveAmbipolar) { - if(ambipolar == UserDefFunction) - xA = AVERAGE_3D_X(xAmbiArr,k,j,i); - - [[maybe_unused]] real BdotB = EXPAND( Bx1*Bx1, +Bx2*Bx2, +Bx3*Bx3); - - [[maybe_unused]] real Fx2 = -xA * BdotB * Jx3; - [[maybe_unused]] real Fx3 = xA * BdotB * Jx2; - #if COMPONENTS == 3 - real JdotB = Jx1 * Bx1 + Jx2 * Bx2 + Jx3 * Bx3; - Fx2 += xA * JdotB * Bx3; - Fx3 += -xA * JdotB * Bx2; - #endif - #if (DIMENSIONS < 2 && COMPONENTS >= 2) - Flux(BX2,k,j,i) += Fx2; - #endif - #if (DIMENSIONS < 3 && COMPONENTS == 3) - Flux(BX3,k,j,i) += Fx3; - #endif - - #if HAVE_ENERGY - Flux(ENG,k,j,i) += EXPAND( ZERO_F , - + Bx2 * Fx2 , - + Bx3 * Fx3 ); - #endif - - locdmax += xA*BdotB; - } - } - - if(dir==JDIR) { - EXPAND( Jx3 = AVERAGE_4D_X(J, KDIR, k, j, ip1); , - , - Jx1 = AVERAGE_4D_Z(J, IDIR, kp1, j, i); - Jx2 = AVERAGE_4D_XYZ(J, JDIR, kp1, j, ip1); ) - - EXPAND( Bx1 = HALF_F*( Vc(BX1,k,j-1,i) + Vc(BX1,k,j,i)); , - Bx2 = Vs(BX2s,k,j,i); , - Bx3 = HALF_F*( Vc(BX3,k,j-1,i) + Vc(BX3,k,j,i)); ) - - if(haveResistivity) { - if(resistivity == UserDefFunction) - eta = AVERAGE_3D_Y(etaArr,k,j,i); - - // This term is always overwritten by CT, since this sweep is performed whenver - // DIMENSIONS>=2 - //#if DIMENSIONS == 1 - // Flux(BX1,k,j,i) += eta * Jx3; - //#endif - #if (DIMENSIONS < 3 && COMPONENTS == 3) - Flux(BX3,k,j,i) += - eta * Jx1; - #endif - - #if HAVE_ENERGY - Flux(ENG,k,j,i) += EXPAND( Bx1 * eta * Jx3 , - , - - Bx3 * eta * Jx1 ); - #endif - locdmax += eta; - } - - - if(haveAmbipolar) { - if(ambipolar == UserDefFunction) - xA = AVERAGE_3D_Y(xAmbiArr,k,j,i); - - [[maybe_unused]] real BdotB = EXPAND( Bx1*Bx1, +Bx2*Bx2, +Bx3*Bx3); - - [[maybe_unused]] real Fx1 = xA * BdotB * Jx3; - [[maybe_unused]] real Fx3 = -xA * BdotB * Jx1; - #if COMPONENTS == 3 - real JdotB = Jx1 * Bx1 + Jx2 * Bx2 + Jx3 * Bx3; - Fx1 += -xA * JdotB * Bx3; - Fx3 += xA * JdotB * Bx1; - #endif - - // This term is always overwritten by CT, since this sweep is performed whenver - // DIMENSIONS>=2 - //#if DIMENSIONS == 1 - // Flux(BX1,k,j,i) += Fx1; - //#endif - #if (DIMENSIONS < 3 && COMPONENTS == 3) - Flux(BX3,k,j,i) += Fx3; - #endif - - #if HAVE_ENERGY - Flux(ENG,k,j,i) += EXPAND( + Bx1 * Fx1 , - + ZERO_F , - + Bx3 * Fx3 ); - #endif - - locdmax += xA*BdotB; - } - } - - if(dir==KDIR) { - Jx1 = AVERAGE_4D_Y(J, IDIR, k, jp1, i); - Jx2 = AVERAGE_4D_X(J, JDIR, k, j, ip1); - Jx3 = AVERAGE_4D_XYZ(J, KDIR, k, jp1, ip1); - - Bx1 = HALF_F*( Vc(BX1,k-1,j,i) + Vc(BX1,k,j,i)); - Bx2 = HALF_F*( Vc(BX2,k-1,j,i) + Vc(BX2,k,j,i)); - Bx3 = Vs(BX3s,k,j,i); - - - if(haveResistivity) { - if(resistivity == UserDefFunction) - eta = AVERAGE_3D_Z(etaArr,k,j,i); - - // This ie never needed since this is overwritten by CT - //Flux(BX1,k,j,i) += -eta * Jx2; - //Flux(BX2,k,j,i) += eta * Jx1; - - #if HAVE_ENERGY - Flux(ENG,k,j,i) += - Bx1 * eta * Jx2 + Bx2 * eta * Jx1; - #endif - dMax(k,j,i) += eta; - } - - if(haveAmbipolar) { - if(ambipolar == UserDefFunction) - xA = AVERAGE_3D_Z(xAmbiArr,k,j,i); - - [[maybe_unused]] real BdotB = Bx1*Bx1 + Bx2*Bx2 + Bx3*Bx3; - - [[maybe_unused]] real Fx1 = -xA * BdotB * Jx2; - [[maybe_unused]] real Fx2 = xA * BdotB * Jx1; - #if COMPONENTS == 3 - real JdotB = Jx1 * Bx1 + Jx2 * Bx2 + Jx3 * Bx3; - Fx1 += xA * JdotB * Bx2; - Fx2 += -xA * JdotB * Bx1; - #endif - // This is never needed since this is overwritten by CT - //Flux(BX1,k,j,i) += Fx1; - //Flux(BX2,k,j,i) += Fx2; - - #if HAVE_ENERGY - Flux(ENG,k,j,i) += Bx1 * Fx1 + Bx2 * Fx2; - #endif - - locdmax += xA*BdotB; - } - } - dMax(k,j,i) = FMAX(dMax(k,j,i),locdmax); - } - ); - -#endif - idfx::popRegion(); -} - -#endif // HYDRO_ADDNONIDEALMHDFLUX_HPP_ diff --git a/src/hydro/addSourceTerms.cpp b/src/hydro/addSourceTerms.cpp deleted file mode 100644 index 555b4a52..00000000 --- a/src/hydro/addSourceTerms.cpp +++ /dev/null @@ -1,187 +0,0 @@ -// *********************************************************************************** -// Idefix MHD astrophysical code -// Copyright(C) Geoffroy R. J. Lesur -// and other code contributors -// Licensed under CeCILL 2.1 License, see COPYING for more information -// *********************************************************************************** - -#include "hydro.hpp" -#include "dataBlock.hpp" -#include "fargo.hpp" - -// Add source terms -void Hydro::AddSourceTerms(real t, real dt) { - idfx::pushRegion("Hydro::AddSourceTerms"); - - IdefixArray4D Uc = this->Uc; - IdefixArray4D Vc = this->Vc; - IdefixArray1D x1 = data->x[IDIR]; - IdefixArray1D x2 = data->x[JDIR]; - IdefixArray2D fargoVelocity = data->fargo.meanVelocity; -#ifdef ISOTHERMAL - IdefixArray3D csIsoArr = this->isoSoundSpeedArray; -#endif -#if GEOMETRY == SPHERICAL - IdefixArray1D sinx2 = data->sinx2; - IdefixArray1D tanx2 = data->tanx2; - IdefixArray1D rt = data->rt; -#endif - - -#ifdef ISOTHERMAL - [[maybe_unused]] real csIso = this->isoSoundSpeed; - [[maybe_unused]] HydroModuleStatus haveIsoCs = this->haveIsoSoundSpeed; -#endif - - bool haveRotation = this->haveRotation; - real OmegaZ = this->OmegaZ; - - // Fargo - bool haveFargo = data->haveFargo; - - // shearing box (only with fargo&cartesian) - [[maybe_unused]] real sbS = this->sbS; - - if(haveUserSourceTerm) userSourceTerm(*data, t, dt); - - idefix_for("AddSourceTerms", - data->beg[KDIR],data->end[KDIR], - data->beg[JDIR],data->end[JDIR], - data->beg[IDIR],data->end[IDIR], - KOKKOS_LAMBDA (int k, int j, int i) { - #if GEOMETRY == CARTESIAN - // Manually add Coriolis force in cartesian geometry. Otherwise - // Coriolis is treated as a modification to the fluxes - if(haveRotation) { - Uc(MX1,k,j,i) += TWO_F * dt * Vc(RHO,k,j,i) * OmegaZ * Vc(VX2,k,j,i); - Uc(MX2,k,j,i) += - TWO_F * dt * Vc(RHO,k,j,i) * OmegaZ * Vc(VX1,k,j,i); - } - if(haveFargo) { - Uc(MX1,k,j,i) += TWO_F * dt * Vc(RHO,k,j,i) * OmegaZ * sbS * x1(i); - } - #endif - // fetch fargo velocity when required - [[maybe_unused]] real fargoV = ZERO_F; - if(haveFargo) { - // No source term when CARTESIAN+Fargo - #if GEOMETRY == POLAR - fargoV = fargoVelocity(k,i); - #elif GEOMETRY == SPHERICAL - fargoV = fargoVelocity(j,i); - #endif - } -#if GEOMETRY == CYLINDRICAL - #if COMPONENTS == 3 - real vphi,Sm; - vphi = Vc(iVPHI,k,j,i); - if(haveRotation) vphi += OmegaZ*x1(i); - Sm = Vc(RHO,k,j,i) * vphi*vphi; // Centrifugal - // Presure (because pressure is included in the flux, additional source terms arise) - #ifdef ISOTHERMAL - real c2Iso; - if(haveIsoCs == UserDefFunction) { - c2Iso = csIsoArr(k,j,i); - c2Iso = c2Iso*c2Iso; - } else { - c2Iso = csIso*csIso; - } - Sm += Vc(RHO,k,j,i)*c2Iso; - #else - Sm += Vc(PRS,k,j,i); - #endif // ISOTHERMAL - #if MHD==YES - Sm -= Vc(iBPHI,k,j,i)*Vc(iBPHI,k,j,i); // Hoop stress - // Magnetic pressure - Sm += HALF_F*(EXPAND( Vc(BX1,k,j,i)*Vc(BX1,k,j,i) , - +Vc(BX2,k,j,i)*Vc(BX2,k,j,i) , - +Vc(BX3,k,j,i)*Vc(BX3,k,j,i) )); - #endif // MHD - Uc(MX1,k,j,i) += dt * Sm / x1(i); - #endif // COMPONENTS - -#elif GEOMETRY == POLAR - real vphi,Sm; - vphi = Vc(iVPHI,k,j,i) + fargoV; - if(haveRotation) vphi += OmegaZ*x1(i); - Sm = Vc(RHO,k,j,i) * vphi*vphi; // Centrifugal - // Pressure (because we're including pressure in the flux, - // we need that to get the radial pressure gradient) - #ifdef ISOTHERMAL - real c2Iso; - if(haveIsoCs == UserDefFunction) { - c2Iso = csIsoArr(k,j,i); - c2Iso = c2Iso*c2Iso; - } else { - c2Iso = csIso*csIso; - } - Sm += Vc(RHO,k,j,i)*c2Iso; - #else - Sm += Vc(PRS,k,j,i); - #endif // ISOTHERMAL - #if MHD==YES - Sm -= Vc(iBPHI,k,j,i)*Vc(iBPHI,k,j,i); // Hoop stress - // Magnetic pressus - Sm += HALF_F*(EXPAND( Vc(BX1,k,j,i)*Vc(BX1,k,j,i) , - +Vc(BX2,k,j,i)*Vc(BX2,k,j,i) , - +Vc(BX3,k,j,i)*Vc(BX3,k,j,i) )); - #endif // MHD - Uc(MX1,k,j,i) += dt * Sm / x1(i); - -#elif GEOMETRY == SPHERICAL - real vphi,Sm,ct; - vphi = SELECT(ZERO_F, ZERO_F, Vc(iVPHI,k,j,i))+fargoV; - if(haveRotation) vphi += OmegaZ*x1(i)*FABS(sinx2(j)); - // Centrifugal - Sm = Vc(RHO,k,j,i) * (EXPAND( ZERO_F, + Vc(VX2,k,j,i)*Vc(VX2,k,j,i), + vphi*vphi)); - // Pressure curvature - #ifdef ISOTHERMAL - real c2Iso; - if(haveIsoCs == UserDefFunction) { - c2Iso = csIsoArr(k,j,i); - c2Iso = c2Iso*c2Iso; - } else { - c2Iso = csIso*csIso; - } - Sm += 2.0*Vc(RHO,k,j,i)*c2Iso; - #else - Sm += 2.0*Vc(PRS,k,j,i); - #endif // ISOTHERMAL - #if MHD == YES - // Hoop stress - Sm -= EXPAND( ZERO_F , - + Vc(iBTH,k,j,i)*Vc(iBTH,k,j,i) , - + Vc(iBPHI,k,j,i)*Vc(iBPHI,k,j,i) ); - // 2* mag pressure curvature - Sm += EXPAND( Vc(BX1,k,j,i)*Vc(BX1,k,j,i) , - +Vc(BX2,k,j,i)*Vc(BX2,k,j,i) , - +Vc(BX3,k,j,i)*Vc(BX3,k,j,i) ); - #endif - Uc(MX1,k,j,i) += dt*Sm/x1(i); - #if COMPONENTS >= 2 - ct = 1.0/tanx2(j); - // Centrifugal - Sm = Vc(RHO,k,j,i) * (EXPAND( ZERO_F, - Vc(iVTH,k,j,i)*Vc(iVR,k,j,i), + ct*vphi*vphi)); - // Pressure curvature - #ifdef ISOTHERMAL - Sm += ct * c2Iso * Vc(RHO,k,j,i); - #else - Sm += ct * Vc(PRS,k,j,i); - #endif - #if MHD == YES - // Hoop stress - Sm += EXPAND( ZERO_F , - + Vc(iBTH,k,j,i)*Vc(iBR,k,j,i) , - - ct*Vc(iBPHI,k,j,i)*Vc(iBPHI,k,j,i) ); - // Magnetic pressure - Sm += HALF_F*ct*(EXPAND( Vc(BX1,k,j,i)*Vc(BX1,k,j,i) , - +Vc(BX2,k,j,i)*Vc(BX2,k,j,i) , - +Vc(BX3,k,j,i)*Vc(BX3,k,j,i)) ); - #endif - Uc(MX2,k,j,i) += dt*Sm / rt(i); - #endif // COMPONENTS -#endif - } - ); - - idfx::popRegion(); -} diff --git a/src/hydro/axis.hpp b/src/hydro/axis.hpp deleted file mode 100644 index ca80eba8..00000000 --- a/src/hydro/axis.hpp +++ /dev/null @@ -1,77 +0,0 @@ -// *********************************************************************************** -// Idefix MHD astrophysical code -// Copyright(C) Geoffroy R. J. Lesur -// and other code contributors -// Licensed under CeCILL 2.1 License, see COPYING for more information -// *********************************************************************************** - -#ifndef HYDRO_AXIS_HPP_ -#define HYDRO_AXIS_HPP_ - -#include -#include "idefix.hpp" -#include "grid.hpp" -#include "electroMotiveForce.hpp" - -// Forward class hydro declaration -class Hydro; -class DataBlock; - -// Whether we use athena++ procedure to regularise BX2s -//#define AXIS_BX2S_USE_ATHENA_REGULARISATION - -class Axis { - public: - void Init(Grid &, Hydro *); // Initialisation - void RegularizeEMFs(); // Regularize the EMF sitting on the axis - void RegularizeCurrent(); // Regularize the currents along the axis - void EnforceAxisBoundary(int side); // Enforce the boundary conditions (along X2) - void ReconstructBx2s(); // Reconstruct BX2s in the ghost zone using divB=0 - void ShowConfig(); - - - void SymmetrizeEx1Side(int); // Symmetrize on a specific side (internal method) - void RegularizeEx3side(int); // Regularize Ex3 along the axis (internal method) - void RegularizeCurrentSide(int); // Regularize J along the axis (internal method) - void FixBx2sAxis(int side); // Fix BX2s on the axis using the field around it (internal) - void FixBx2sAxisGhostAverage(int side); //Fix BX2s on the axis using the average of neighbouring - // cell in theta direction (like Athena) - void ExchangeMPI(int side); // Function has to be public for GPU, but its technically - // a private function - - - private: - bool isTwoPi = false; - bool axisRight = false; - bool axisLeft = false; - bool needMPIExchange = false; - - enum {faceTop, faceBot}; -#ifdef WITH_MPI - MPI_Request sendRequest; - MPI_Request recvRequest; - - IdefixArray1D bufferSend; - IdefixArray1D bufferRecv; - - int bufferSize; - - IdefixArray1D mapVars; - int mapNVars{0}; - -#endif - void InitMPI(); - - IdefixArray1D Ex1Avg; - IdefixArray2D BAvg; - IdefixArray2D JAvg; - IdefixArray1D symmetryVc; - IdefixArray1D symmetryVs; - - - Hydro *hydro; - DataBlock *data; - ElectroMotiveForce *emf; -}; - -#endif // HYDRO_AXIS_HPP_ diff --git a/src/hydro/boundary/CMakeLists.txt b/src/hydro/boundary/CMakeLists.txt deleted file mode 100644 index c457f865..00000000 --- a/src/hydro/boundary/CMakeLists.txt +++ /dev/null @@ -1,5 +0,0 @@ -target_sources(idefix - PUBLIC ${CMAKE_CURRENT_LIST_DIR}/boundaryloop.hpp - PUBLIC ${CMAKE_CURRENT_LIST_DIR}/hydroboundary.cpp - PUBLIC ${CMAKE_CURRENT_LIST_DIR}/hydroboundary.hpp - ) diff --git a/src/hydro/boundary/boundaryloop.hpp b/src/hydro/boundary/boundaryloop.hpp deleted file mode 100644 index 10a11a9c..00000000 --- a/src/hydro/boundary/boundaryloop.hpp +++ /dev/null @@ -1,145 +0,0 @@ -// *********************************************************************************** -// Idefix MHD astrophysical code -// Copyright(C) Geoffroy R. J. Lesur -// and other code contributors -// Licensed under CeCILL 2.1 License, see COPYING for more information -// *********************************************************************************** - - -#ifndef HYDRO_BOUNDARY_BOUNDARYLOOP_HPP_ -#define HYDRO_BOUNDARY_BOUNDARYLOOP_HPP_ - -#include -#include "idefix.hpp" - -template -inline void HydroBoundary::BoundaryForAll( - const std::string & name, - const int &dir, - const BoundarySide &side, - Function function) { - const int nxi = data->np_int[IDIR]; - const int nxj = data->np_int[JDIR]; - const int nxk = data->np_int[KDIR]; - - const int ighost = data->nghost[IDIR]; - const int jghost = data->nghost[JDIR]; - const int kghost = data->nghost[KDIR]; - - // Boundaries of the loop - const int ibeg = (dir == IDIR) ? side*(ighost+nxi) : 0; - const int iend = (dir == IDIR) ? ighost + side*(ighost+nxi) : data->np_tot[IDIR]; - const int jbeg = (dir == JDIR) ? side*(jghost+nxj) : 0; - const int jend = (dir == JDIR) ? jghost + side*(jghost+nxj) : data->np_tot[JDIR]; - const int kbeg = (dir == KDIR) ? side*(kghost+nxk) : 0; - const int kend = (dir == KDIR) ? kghost + side*(kghost+nxk) : data->np_tot[KDIR]; - - idefix_for(name, 0, NVAR, kbeg, kend, jbeg, jend, ibeg, iend, function); -} - -template -inline void HydroBoundary::BoundaryFor( - const std::string & name, - const int &dir, - const BoundarySide &side, - Function function) { - const int nxi = data->np_int[IDIR]; - const int nxj = data->np_int[JDIR]; - const int nxk = data->np_int[KDIR]; - - const int ighost = data->nghost[IDIR]; - const int jghost = data->nghost[JDIR]; - const int kghost = data->nghost[KDIR]; - - // Boundaries of the loop - const int ibeg = (dir == IDIR) ? side*(ighost+nxi) : 0; - const int iend = (dir == IDIR) ? ighost + side*(ighost+nxi) : data->np_tot[IDIR]; - const int jbeg = (dir == JDIR) ? side*(jghost+nxj) : 0; - const int jend = (dir == JDIR) ? jghost + side*(jghost+nxj) : data->np_tot[JDIR]; - const int kbeg = (dir == KDIR) ? side*(kghost+nxk) : 0; - const int kend = (dir == KDIR) ? kghost + side*(kghost+nxk) : data->np_tot[KDIR]; - - - - idefix_for(name, kbeg, kend, jbeg, jend, ibeg, iend, function); -} - -template -inline void HydroBoundary::BoundaryForX1s( - const std::string & name, - const int &dir, - const BoundarySide &side, - Function function) { - const int nxi = data->np_int[IDIR]+1; - const int nxj = data->np_int[JDIR]; - const int nxk = data->np_int[KDIR]; - - const int ighost = data->nghost[IDIR]; - const int jghost = data->nghost[JDIR]; - const int kghost = data->nghost[KDIR]; - - // Boundaries of the loop - const int ibeg = (dir == IDIR) ? side*(ighost+nxi) : 0; - const int iend = (dir == IDIR) ? ighost + side*(ighost+nxi) : data->np_tot[IDIR]+1; - const int jbeg = (dir == JDIR) ? side*(jghost+nxj) : 0; - const int jend = (dir == JDIR) ? jghost + side*(jghost+nxj) : data->np_tot[JDIR]; - const int kbeg = (dir == KDIR) ? side*(kghost+nxk) : 0; - const int kend = (dir == KDIR) ? kghost + side*(kghost+nxk) : data->np_tot[KDIR]; - - idefix_for(name, kbeg, kend, jbeg, jend, ibeg, iend, function); -} - -template -inline void HydroBoundary::BoundaryForX2s( - const std::string & name, - const int &dir, - const BoundarySide &side, - Function function) { - const int nxi = data->np_int[IDIR]; - const int nxj = data->np_int[JDIR]+1; - const int nxk = data->np_int[KDIR]; - - const int ighost = data->nghost[IDIR]; - const int jghost = data->nghost[JDIR]; - const int kghost = data->nghost[KDIR]; - - // Boundaries of the loop - const int ibeg = (dir == IDIR) ? side*(ighost+nxi) : 0; - const int iend = (dir == IDIR) ? ighost + side*(ighost+nxi) : data->np_tot[IDIR]; - const int jbeg = (dir == JDIR) ? side*(jghost+nxj) : 0; - const int jend = (dir == JDIR) ? jghost + side*(jghost+nxj) : data->np_tot[JDIR]+1; - const int kbeg = (dir == KDIR) ? side*(kghost+nxk) : 0; - const int kend = (dir == KDIR) ? kghost + side*(kghost+nxk) : data->np_tot[KDIR]; - - idefix_for(name, kbeg, kend, jbeg, jend, ibeg, iend, function); -} - -template -inline void HydroBoundary::BoundaryForX3s( - const std::string & name, - const int &dir, - const BoundarySide &side, - Function function) { - const int nxi = data->np_int[IDIR]; - const int nxj = data->np_int[JDIR]; - const int nxk = data->np_int[KDIR]+1; - - const int ighost = data->nghost[IDIR]; - const int jghost = data->nghost[JDIR]; - const int kghost = data->nghost[KDIR]; - - // Boundaries of the loop - const int ibeg = (dir == IDIR) ? side*(ighost+nxi) : 0; - const int iend = (dir == IDIR) ? ighost + side*(ighost+nxi) : data->np_tot[IDIR]; - const int jbeg = (dir == JDIR) ? side*(jghost+nxj) : 0; - const int jend = (dir == JDIR) ? jghost + side*(jghost+nxj) : data->np_tot[JDIR]; - const int kbeg = (dir == KDIR) ? side*(kghost+nxk) : 0; - const int kend = (dir == KDIR) ? kghost + side*(kghost+nxk) : data->np_tot[KDIR]+1; - - idefix_for(name, kbeg, kend, jbeg, jend, ibeg, iend, function); -} - - - - -#endif // HYDRO_BOUNDARY_BOUNDARYLOOP_HPP_ diff --git a/src/hydro/boundary/hydroboundary.cpp b/src/hydro/boundary/hydroboundary.cpp deleted file mode 100644 index 2d7c0cd1..00000000 --- a/src/hydro/boundary/hydroboundary.cpp +++ /dev/null @@ -1,735 +0,0 @@ -// *********************************************************************************** -// Idefix MHD astrophysical code -// Copyright(C) Geoffroy R. J. Lesur -// and other code contributors -// Licensed under CeCILL 2.1 License, see COPYING for more information -// *********************************************************************************** - -#include "hydroboundary.hpp" -#include "hydro.hpp" -#include "dataBlock.hpp" -#include "boundaryloop.hpp" - - -void HydroBoundary::Init(Input & input, Grid &grid, Hydro* hydro) { - idfx::pushRegion("HydroBoundary::Init"); - this->hydro = hydro; - this->data = hydro->data; - - if(data->lbound[IDIR] == shearingbox || data->rbound[IDIR] == shearingbox) { - sBArray = IdefixArray4D("ShearingBoxArray", - NVAR, - data->np_tot[KDIR]+1, - data->np_tot[JDIR]+1, - data->nghost[IDIR]); - } - - // Init MPI stack when needed -#ifdef WITH_MPI - //////////////////////////////////////////////////////////////////////////// - // Init variable mappers - // The variable mapper list all of the variable which are exchanged in MPI boundary calls - // This is required since we skip some of the variables in Vc to limit the amount of data - // being exchanged - #if MHD == YES - int mapNVars = NVAR - DIMENSIONS; // We will not send magnetic field components which are in Vs - #else - int mapNVars = NVAR; - #endif - - std::vector mapVars; - - // Init the list of variables we will exchange in MPI routines - int ntarget = 0; - for(int n = 0 ; n < mapNVars ; n++) { - mapVars.push_back(ntarget); - ntarget++; - #if MHD == YES - // Skip centered field components if they are also defined in Vs - #if DIMENSIONS >= 1 - if(ntarget==BX1) ntarget++; - #endif - #if DIMENSIONS >= 2 - if(ntarget==BX2) ntarget++; - #endif - #if DIMENSIONS == 3 - if(ntarget==BX3) ntarget++; - #endif - #endif - } - #if MHD == YES - mpi.Init(data->mygrid, mapVars, data->nghost.data(), data->np_int.data(), true); - #else - mpi.Init(data->mygrid, mapVars, data->nghost.data(), data->np_int.data()); - #endif -#endif // MPI - idfx::popRegion(); -} - -void HydroBoundary::EnrollFluxBoundary(UserDefBoundaryFunc myFunc) { - this->haveFluxBoundary = true; - this->fluxBoundaryFunc = myFunc; -} - -void HydroBoundary::EnforceFluxBoundaries(int dir) { - idfx::pushRegion("HydroBoundary::EnforceFluxBoundaries"); - if(haveFluxBoundary) { - if(data->lbound[dir] != internal) { - fluxBoundaryFunc(*data, dir, left, data->t); - } - if(data->rbound[dir] != internal) { - fluxBoundaryFunc(*data, dir, right, data->t); - } - } else { - IDEFIX_ERROR("Cannot enforce flux boundary conditions without enrolling a specific function"); - } - idfx::popRegion(); -} - -void HydroBoundary::SetBoundaries(real t) { - idfx::pushRegion("HydroBoundary::SetBoundaries"); - // set internal boundary conditions - if(haveInternalBoundary) internalBoundaryFunc(*data, t); - for(int dir=0 ; dir < DIMENSIONS ; dir++ ) { - // MPI Exchange data when needed - #ifdef WITH_MPI - if(data->mygrid->nproc[dir]>1) { - switch(dir) { - case 0: - mpi.ExchangeX1(hydro->Vc, hydro->Vs); - break; - case 1: - mpi.ExchangeX2(hydro->Vc, hydro->Vs); - break; - case 2: - mpi.ExchangeX3(hydro->Vc, hydro->Vs); - break; - } - } - #endif - EnforceBoundaryDir(t, dir); - #if MHD == YES - // Reconstruct the normal field component when using CT - ReconstructNormalField(dir); - #endif - } // Loop on dimension ends - -#if MHD == YES - // Remake the cell-centered field. - ReconstructVcField(hydro->Vc); -#endif - idfx::popRegion(); -} - - -// Enforce boundary conditions by writing into ghost zones -void HydroBoundary::EnforceBoundaryDir(real t, int dir) { - idfx::pushRegion("Hydro::EnforceBoundaryDir"); - - // left boundary - - switch(data->lbound[dir]) { - case internal: - // internal is used for MPI-enforced boundary conditions. Nothing to be done here. - break; - - case periodic: - if(data->mygrid->nproc[dir] > 1) break; // Periodicity already enforced by MPI calls - EnforcePeriodic(dir,left); - break; - - case reflective: - EnforceReflective(dir,left); - break; - - case outflow: - EnforceOutflow(dir,left); - break; - - case shearingbox: - EnforceShearingBox(t,dir,left); - break; - case axis: - hydro->myAxis.EnforceAxisBoundary(left); - break; - case userdef: - if(this->haveUserDefBoundary) - this->userDefBoundaryFunc(*data, dir, left, t); - else - IDEFIX_ERROR("No function has been enrolled to define your own boundary conditions"); - break; - - default: - std::stringstream msg ("Boundary condition type is not yet implemented"); - IDEFIX_ERROR(msg); - } - - // right boundary - - switch(data->rbound[dir]) { - case internal: - // internal is used for MPI-enforced boundary conditions. Nothing to be done here. - break; - - case periodic: - if(data->mygrid->nproc[dir] > 1) break; // Periodicity already enforced by MPI calls - EnforcePeriodic(dir,right); - break; - case reflective: - EnforceReflective(dir,right); - break; - case outflow: - EnforceOutflow(dir,right); - break; - case shearingbox: - EnforceShearingBox(t,dir,right); - break; - case axis: - hydro->myAxis.EnforceAxisBoundary(right); - break; - case userdef: - if(this->haveUserDefBoundary) - this->userDefBoundaryFunc(*data, dir, right, t); - else - IDEFIX_ERROR("No function has been enrolled to define your own boundary conditions"); - break; - default: - std::stringstream msg("Boundary condition type is not yet implemented"); - IDEFIX_ERROR(msg); - } - - idfx::popRegion(); -} - - -void HydroBoundary::ReconstructVcField(IdefixArray4D &Vc) { - idfx::pushRegion("Hydro::ReconstructVcField"); - - IdefixArray4D Vs=hydro->Vs; - - // Reconstruct cell average field when using CT - idefix_for("ReconstructVcMagField",0,data->np_tot[KDIR],0,data->np_tot[JDIR],0,data->np_tot[IDIR], - KOKKOS_LAMBDA (int k, int j, int i) { - D_EXPAND( Vc(BX1,k,j,i) = HALF_F * (Vs(BX1s,k,j,i) + Vs(BX1s,k,j,i+1)) ; , - Vc(BX2,k,j,i) = HALF_F * (Vs(BX2s,k,j,i) + Vs(BX2s,k,j+1,i)) ; , - Vc(BX3,k,j,i) = HALF_F * (Vs(BX3s,k,j,i) + Vs(BX3s,k+1,j,i)) ; ) - } - ); - - idfx::popRegion(); -} - - -void HydroBoundary::ReconstructNormalField(int dir) { - idfx::pushRegion("Hydro::ReconstructNormalField"); - - // Reconstruct the field - IdefixArray4D Vs = hydro->Vs; - // Coordinates - IdefixArray1D x1=data->x[IDIR]; - IdefixArray1D x2=data->x[JDIR]; - IdefixArray1D x3=data->x[KDIR]; - IdefixArray1D dx1=data->dx[IDIR]; - IdefixArray1D dx2=data->dx[JDIR]; - IdefixArray1D dx3=data->dx[KDIR]; - - IdefixArray3D Ax1=data->A[IDIR]; - IdefixArray3D Ax2=data->A[JDIR]; - IdefixArray3D Ax3=data->A[KDIR]; - - int nstart, nend; - int nx1,nx2,nx3; - - // reconstruct BX1s - nstart = data->beg[IDIR]-1; - nend = data->end[IDIR]; - - nx1=data->np_tot[IDIR]; - nx2=data->np_tot[JDIR]; - nx3=data->np_tot[KDIR]; - - if(dir==IDIR) { - idefix_for("ReconstructBX1s",0,nx3,0,nx2, - KOKKOS_LAMBDA (int k, int j) { - for(int i = nstart ; i>=0 ; i-- ) { - Vs(BX1s,k,j,i) = 1.0 / Ax1(k,j,i) * ( Ax1(k,j,i+1)*Vs(BX1s,k,j,i+1) - +(D_EXPAND( ZERO_F , - + Ax2(k,j+1,i) * Vs(BX2s,k,j+1,i) - Ax2(k,j,i) * Vs(BX2s,k,j,i) , - + Ax3(k+1,j,i) * Vs(BX3s,k+1,j,i) - Ax3(k,j,i) * Vs(BX3s,k,j,i) ))); - } - - for(int i = nend ; i=2 - if(dir==JDIR) { - nstart = data->beg[JDIR]-1; - nend = data->end[JDIR]; - if(!hydro->haveAxis) { - idefix_for("ReconstructBX2s",0,data->np_tot[KDIR],0,data->np_tot[IDIR], - KOKKOS_LAMBDA (int k, int i) { - for(int j = nstart ; j>=0 ; j-- ) { - Vs(BX2s,k,j,i) = 1.0 / Ax2(k,j,i) * ( Ax2(k,j+1,i)*Vs(BX2s,k,j+1,i) - +(D_EXPAND( Ax1(k,j,i+1) * Vs(BX1s,k,j,i+1) - Ax1(k,j,i) * Vs(BX1s,k,j,i) , - , - + Ax3(k+1,j,i) * Vs(BX3s,k+1,j,i) - Ax3(k,j,i) * Vs(BX3s,k,j,i) ))); - } - for(int j = nend ; jmyAxis.ReconstructBx2s(); - } - } -#endif - -#if DIMENSIONS == 3 - if(dir==KDIR) { - nstart = data->beg[KDIR]-1; - nend = data->end[KDIR]; - - idefix_for("ReconstructBX3s",0,data->np_tot[JDIR],0,data->np_tot[IDIR], - KOKKOS_LAMBDA (int j, int i) { - for(int k = nstart ; k>=0 ; k-- ) { - Vs(BX3s,k,j,i) = 1.0 / Ax3(k,j,i) * ( Ax3(k+1,j,i)*Vs(BX3s,k+1,j,i) - + ( Ax1(k,j,i+1) * Vs(BX1s,k,j,i+1) - Ax1(k,j,i) * Vs(BX1s,k,j,i) - + Ax2(k,j+1,i) * Vs(BX2s,k,j+1,i) - Ax2(k,j,i) * Vs(BX2s,k,j,i) )); - } - for(int k = nend ; kuserDefBoundaryFunc = myFunc; - this->haveUserDefBoundary = true; -} - -void HydroBoundary::EnrollInternalBoundary(InternalBoundaryFunc myFunc) { - this->internalBoundaryFunc = myFunc; - this->haveInternalBoundary = true; -} - -void HydroBoundary::EnforcePeriodic(int dir, BoundarySide side ) { - IdefixArray4D Vc = hydro->Vc; - int nxi = data->np_int[IDIR]; - int nxj = data->np_int[JDIR]; - int nxk = data->np_int[KDIR]; - - const int ighost = data->nghost[IDIR]; - const int jghost = data->nghost[JDIR]; - const int kghost = data->nghost[KDIR]; - - BoundaryForAll("BoundaryPeriodic", dir, side, - KOKKOS_LAMBDA (int n, int k, int j, int i) { - int iref, jref, kref; - // This hack takes care of cases where we have more ghost zones than active zones - if(dir==IDIR) - iref = ighost + (i+ighost*(nxi-1))%nxi; - else - iref = i; - if(dir==JDIR) - jref = jghost + (j+jghost*(nxj-1))%nxj; - else - jref = j; - if(dir==KDIR) - kref = kghost + (k+kghost*(nxk-1))%nxk; - else - kref = k; - - Vc(n,k,j,i) = Vc(n,kref,jref,iref); - }); - - #if MHD==YES - IdefixArray4D Vs = hydro->Vs; - if(dir==JDIR || dir==KDIR) { - nxi = data->np_int[IDIR]+1; - nxj = data->np_int[JDIR]; - nxk = data->np_int[KDIR]; - BoundaryForX1s("BoundaryPeriodicX1s",dir,side, - KOKKOS_LAMBDA (int k, int j, int i) { - int iref, jref, kref; - // This hack takes care of cases where we have more ghost zones than active zones - if(dir==IDIR) - iref = ighost + (i+ighost*(nxi-1))%nxi; - else - iref = i; - if(dir==JDIR) - jref = jghost + (j+jghost*(nxj-1))%nxj; - else - jref = j; - if(dir==KDIR) - kref = kghost + (k+kghost*(nxk-1))%nxk; - else - kref = k; - - Vs(BX1s,k,j,i) = Vs(BX1s,kref,jref,iref); - }); - } - #if DIMENSIONS >=2 - if(dir==IDIR || dir==KDIR) { - nxi = data->np_int[IDIR]; - nxj = data->np_int[JDIR]+1; - nxk = data->np_int[KDIR]; - BoundaryForX2s("BoundaryPeriodicX2s",dir,side, - KOKKOS_LAMBDA (int k, int j, int i) { - int iref, jref, kref; - // This hack takes care of cases where we have more ghost zones than active zones - if(dir==IDIR) - iref = ighost + (i+ighost*(nxi-1))%nxi; - else - iref = i; - if(dir==JDIR) - jref = jghost + (j+jghost*(nxj-1))%nxj; - else - jref = j; - if(dir==KDIR) - kref = kghost + (k+kghost*(nxk-1))%nxk; - else - kref = k; - - Vs(BX2s,k,j,i) = Vs(BX2s,kref,jref,iref); - }); - } - #endif - #if DIMENSIONS == 3 - nxi = data->np_int[IDIR]; - nxj = data->np_int[JDIR]; - nxk = data->np_int[KDIR]+1; - if(dir==IDIR || dir==JDIR) { - BoundaryForX3s("BoundaryPeriodicX3s",dir,side, - KOKKOS_LAMBDA (int k, int j, int i) { - int iref, jref, kref; - // This hack takes care of cases where we have more ghost zones than active zones - if(dir==IDIR) - iref = ighost + (i+ighost*(nxi-1))%nxi; - else - iref = i; - if(dir==JDIR) - jref = jghost + (j+jghost*(nxj-1))%nxj; - else - jref = j; - if(dir==KDIR) - kref = kghost + (k+kghost*(nxk-1))%nxk; - else - kref = k; - - Vs(BX3s,k,j,i) = Vs(BX3s,kref,jref,iref); - }); - } - #endif - #endif// MHD -} - - -void HydroBoundary::EnforceReflective(int dir, BoundarySide side ) { - IdefixArray4D Vc = hydro->Vc; - const int nxi = data->np_int[IDIR]; - const int nxj = data->np_int[JDIR]; - const int nxk = data->np_int[KDIR]; - - const int ighost = data->nghost[IDIR]; - const int jghost = data->nghost[JDIR]; - const int kghost = data->nghost[KDIR]; - - BoundaryForAll("BoundaryReflective", dir, side, - KOKKOS_LAMBDA (int n, int k, int j, int i) { - // ref= 2*ibound -i -1 - // with ibound = nghost on the left and ibount = nghost + nx -1 on the right - const int iref = (dir==IDIR) ? 2*(ighost + side*(nxi-1)) - i - 1 : i; - const int jref = (dir==JDIR) ? 2*(jghost + side*(nxj-1)) - j - 1 : j; - const int kref = (dir==KDIR) ? 2*(kghost + side*(nxk-1)) - k - 1 : k; - - const int sign = (n == VX1+dir) ? -1.0 : 1.0; - - Vc(n,k,j,i) = sign * Vc(n,kref,jref,iref); - }); - - #if MHD==YES - IdefixArray4D Vs = hydro->Vs; - if(dir==JDIR || dir==KDIR) { - BoundaryForX1s("BoundaryReflectiveX1s",dir,side, - KOKKOS_LAMBDA (int k, int j, int i) { - // ref= 2*ibound -i -1 - // with ibound = nghost on the left and ibount = nghost + nx -1 on the right - //const int iref = (dir==IDIR) ? 2*(ighost + side*(nxi-1)) - i - 1 : i; - const int jref = (dir==JDIR) ? 2*(jghost + side*(nxj-1)) - j - 1 : j; - const int kref = (dir==KDIR) ? 2*(kghost + side*(nxk-1)) - k - 1 : k; - - Vs(BX1s,k,j,i) = -Vs(BX1s,kref,jref,i); - }); - } - #if DIMENSIONS >=2 - if(dir==IDIR || dir==KDIR) { - BoundaryForX2s("BoundaryReflectiveX2s",dir,side, - KOKKOS_LAMBDA (int k, int j, int i) { - const int iref = (dir==IDIR) ? 2*(ighost + side*(nxi-1)) - i - 1 : i; - //const int jref = (dir==JDIR) ? 2*(jghost + side*(nxj-1)) - j - 1 : j; - const int kref = (dir==KDIR) ? 2*(kghost + side*(nxk-1)) - k - 1 : k; - - Vs(BX2s,k,j,i) = -Vs(BX2s,kref,j,iref); - }); - } - #endif - #if DIMENSIONS == 3 - if(dir==IDIR || dir==JDIR) { - BoundaryForX3s("BoundaryReflectiveX3s",dir,side, - KOKKOS_LAMBDA (int k, int j, int i) { - const int iref = (dir==IDIR) ? 2*(ighost + side*(nxi-1)) - i - 1 : i; - const int jref = (dir==JDIR) ? 2*(jghost + side*(nxj-1)) - j - 1 : j; - //const int kref = (dir==KDIR) ? 2*(kghost + side*(nxk-1)) - k - 1 : k; - - Vs(BX3s,k,j,i) = -Vs(BX3s,k,jref,iref); - }); - } - #endif - #endif// MHD -} - -void HydroBoundary::EnforceOutflow(int dir, BoundarySide side ) { - IdefixArray4D Vc = hydro->Vc; - const int nxi = data->np_int[IDIR]; - const int nxj = data->np_int[JDIR]; - const int nxk = data->np_int[KDIR]; - - const int ighost = data->nghost[IDIR]; - const int jghost = data->nghost[JDIR]; - const int kghost = data->nghost[KDIR]; - - BoundaryForAll("BoundaryOutflow", dir, side, - KOKKOS_LAMBDA (int n, int k, int j, int i) { - // ref= ibound - // with ibound = nghost on the left and ibound = nghost + nx -1 on the right - const int iref = (dir==IDIR) ? ighost + side*(nxi-1) : i; - const int jref = (dir==JDIR) ? jghost + side*(nxj-1) : j; - const int kref = (dir==KDIR) ? kghost + side*(nxk-1) : k; - - // should it go inwards or outwards? - // side = 1 on the left and =-1 on the right - const int sign = 1-2*side; - - if( (n== VX1+dir) && (sign*Vc(n,kref,jref,iref) >= ZERO_F) ) { - Vc(n,k,j,i) = ZERO_F; - } else { - Vc(n,k,j,i) = Vc(n,kref,jref,iref); - } - }); - - #if MHD==YES - IdefixArray4D Vs = hydro->Vs; - if(dir==JDIR || dir==KDIR) { - BoundaryForX1s("BoundaryOutflowX1s",dir,side, - KOKKOS_LAMBDA (int k, int j, int i) { - // with ibound = nghost on the left and ibount = nghost + nx -1 on the right - //const int iref = (dir==IDIR) ? ighost + side*(nxi-1) : i; - const int jref = (dir==JDIR) ? jghost + side*(nxj-1) : j; - const int kref = (dir==KDIR) ? kghost + side*(nxk-1) : k; - - Vs(BX1s,k,j,i) = Vs(BX1s,kref,jref,i); - }); - } - #if DIMENSIONS >=2 - if(dir==IDIR || dir==KDIR) { - BoundaryForX2s("BoundaryOutflowX2s",dir,side, - KOKKOS_LAMBDA (int k, int j, int i) { - const int iref = (dir==IDIR) ? ighost + side*(nxi-1) : i; - //const int jref = (dir==JDIR) ? jghost + side*(nxj-1) : j; - const int kref = (dir==KDIR) ? kghost + side*(nxk-1) : k; - - Vs(BX2s,k,j,i) = Vs(BX2s,kref,j,iref); - }); - } - #endif - #if DIMENSIONS == 3 - if(dir==IDIR || dir==JDIR) { - BoundaryForX3s("BoundaryOutflowX3s",dir,side, - KOKKOS_LAMBDA (int k, int j, int i) { - const int iref = (dir==IDIR) ? ighost + side*(nxi-1) : i; - const int jref = (dir==JDIR) ? jghost + side*(nxj-1) : j; - //const int kref = (dir==KDIR) ? kghost + side*(nxk-1) : k; - - Vs(BX3s,k,j,i) = Vs(BX3s,k,jref,iref); - }); - } - #endif - #endif// MHD -} - -void HydroBoundary::EnforceShearingBox(real t, int dir, BoundarySide side) { - if(dir != IDIR) - IDEFIX_ERROR("Shearing box boundaries can only be applied along the X1 direction"); - if(data->mygrid->nproc[JDIR]>1) - IDEFIX_ERROR("Shearing box is not yet compatible with domain decomposition in X2"); - - // First thing is to enforce periodicity (already performed by MPI) - if(data->mygrid->nproc[dir] == 1) EnforcePeriodic(dir, side); - - IdefixArray4D scrh = sBArray; - IdefixArray4D Vc = hydro->Vc; - - const int nxi = data->np_int[IDIR]; - const int nxj = data->np_int[JDIR]; - const int nxk = data->np_int[KDIR]; - - const int ighost = data->nghost[IDIR]; - const int jghost = data->nghost[JDIR]; - const int kghost = data->nghost[KDIR]; - - // Where does the boundary starts along x1? - const int istart = side*(ighost+nxi); - - // Shear rate - const real S = hydro->sbS; - - // Box size - const real Lx = data->mygrid->xend[IDIR] - data->mygrid->xbeg[IDIR]; - const real Ly = data->mygrid->xend[JDIR] - data->mygrid->xbeg[JDIR]; - - // total number of cells in y (active domain) - const int ny = data->mygrid->np_int[JDIR]; - const real dy = Ly/ny; - - // Compute offset in y modulo the box size - const int sign=2*side-1; - const real sbVelocity = sign*S*Lx; - real dL = std::fmod(sbVelocity*t,Ly); - - // translate this into # of cells - const int m = static_cast (std::floor(dL/dy+HALF_F)); - - // remainding shift - const real eps = dL / dy - m; - - - // New we need to perform the shift - BoundaryForAll("BoundaryShearingBox", dir, side, - KOKKOS_LAMBDA ( int n, int k, int j, int i) { - // jorigin - const int jo = jghost + ((j-m-jghost)%nxj+nxj)%nxj; - const int jop2 = jghost + ((jo+2-jghost)%nxj+nxj)%nxj; - const int jop1 = jghost + ((jo+1-jghost)%nxj+nxj)%nxj; - const int jom1 = jghost + ((jo-1-jghost)%nxj+nxj)%nxj; - const int jom2 = jghost + ((jo-2-jghost)%nxj+nxj)%nxj; - - // Define Left and right fluxes - // Fluxes are defined from slop-limited interpolation - // Using Van-leer slope limiter (consistently with the main advection scheme) - real Fl,Fr; - real dqm, dqp, dq; - - if(eps>=ZERO_F) { - // Compute Fl - dqm = Vc(n,k,jom1,i) - Vc(n,k,jom2,i); - dqp = Vc(n,k,jo,i) - Vc(n,k,jom1,i); - dq = (dqp*dqm > ZERO_F ? TWO_F*dqp*dqm/(dqp + dqm) : ZERO_F); - - Fl = Vc(n,k,jom1,i) + 0.5*dq*(1.0-eps); - //Compute Fr - dqm=dqp; - dqp = Vc(n,k,jop1,i) - Vc(n,k,jo,i); - dq = (dqp*dqm > ZERO_F ? TWO_F*dqp*dqm/(dqp + dqm) : ZERO_F); - - Fr = Vc(n,k,jo,i) + 0.5*dq*(1.0-eps); - } else { - //Compute Fl - dqm = Vc(n,k,jo,i) - Vc(n,k,jom1,i); - dqp = Vc(n,k,jop1,i) - Vc(n,k,jo,i); - dq = (dqp*dqm > ZERO_F ? TWO_F*dqp*dqm/(dqp + dqm) : ZERO_F); - - Fl = Vc(n,k,jo,i) - 0.5*dq*(1.0+eps); - // Compute Fr - dqm=dqp; - dqp = Vc(n,k,jop2,i) - Vc(n,k,jop1,i); - dq = (dqp*dqm > ZERO_F ? TWO_F*dqp*dqm/(dqp + dqm) : ZERO_F); - - Fr = Vc(n,k,jop1,i) - 0.5*dq*(1.0+eps); - } - scrh(n,k,j,i-istart) = Vc(n,k,jo,i) - eps*(Fr - Fl); - }); - // Copy scrach back to our boundary - BoundaryForAll("BoundaryShearingBoxCopy", dir, side, - KOKKOS_LAMBDA ( int n, int k, int j, int i) { - Vc(n,k,j,i) = scrh(n,k,j,i-istart); - if(n==VX2) Vc(n,k,j,i) += sbVelocity; - }); - - // Magnetised version of the same thing - #if MHD==YES - IdefixArray4D Vs = hydro->Vs; - #if DIMENSIONS >= 2 - for(int component = BX2s ; component < DIMENSIONS ; component++) { - BoundaryFor("BoundaryShearingBoxBXs", dir, side, - KOKKOS_LAMBDA (int k, int j, int i) { - // jorigin - const int jo = jghost + ((j-m-jghost)%nxj+nxj)%nxj; - const int jop2 = jghost + ((jo+2-jghost)%nxj+nxj)%nxj; - const int jop1 = jghost + ((jo+1-jghost)%nxj+nxj)%nxj; - const int jom1 = jghost + ((jo-1-jghost)%nxj+nxj)%nxj; - const int jom2 = jghost + ((jo-2-jghost)%nxj+nxj)%nxj; - - // Define Left and right fluxes - // Fluxes are defined from slop-limited interpolation - // Using Van-leer slope limiter (consistently with the main advection scheme) - real Fl,Fr; - real dqm, dqp, dq; - - if(eps>=ZERO_F) { - // Compute Fl - dqm = Vs(component,k,jom1,i) - Vs(component,k,jom2,i); - dqp = Vs(component,k,jo,i) - Vs(component,k,jom1,i); - dq = (dqp*dqm > ZERO_F ? TWO_F*dqp*dqm/(dqp + dqm) : ZERO_F); - - Fl = Vs(component,k,jom1,i) + 0.5*dq*(1.0-eps); - //Compute Fr - dqm=dqp; - dqp = Vs(component,k,jop1,i) - Vs(component,k,jo,i); - dq = (dqp*dqm > ZERO_F ? TWO_F*dqp*dqm/(dqp + dqm) : ZERO_F); - - Fr = Vs(component,k,jo,i) + 0.5*dq*(1.0-eps); - } else { - //Compute Fl - dqm = Vs(component,k,jo,i) - Vs(component,k,jom1,i); - dqp = Vs(component,k,jop1,i) - Vs(component,k,jo,i); - dq = (dqp*dqm > ZERO_F ? TWO_F*dqp*dqm/(dqp + dqm) : ZERO_F); - - Fl = Vs(component,k,jo,i) - 0.5*dq*(1.0+eps); - // Compute Fr - dqm=dqp; - dqp = Vs(component,k,jop2,i) - Vs(component,k,jop1,i); - dq = (dqp*dqm > ZERO_F ? TWO_F*dqp*dqm/(dqp + dqm) : ZERO_F); - - Fr = Vs(component,k,jop1,i) - 0.5*dq*(1.0+eps); - } - scrh(0,k,j,i-istart) = Vs(component,k,jo,i) - eps*(Fr - Fl); - }); - // Copy scratch back to our boundary - BoundaryFor("BoundaryShearingBoxCopyBXs", dir, side, - KOKKOS_LAMBDA ( int k, int j, int i) { - Vs(component,k,j,i) = scrh(0,k,j,i-istart); - }); - }// loop on components - #endif// DIMENSIONS - #endif // MHD -} diff --git a/src/hydro/boundary/hydroboundary.hpp b/src/hydro/boundary/hydroboundary.hpp deleted file mode 100644 index 960759d6..00000000 --- a/src/hydro/boundary/hydroboundary.hpp +++ /dev/null @@ -1,96 +0,0 @@ -// *********************************************************************************** -// Idefix MHD astrophysical code -// Copyright(C) Geoffroy R. J. Lesur -// and other code contributors -// Licensed under CeCILL 2.1 License, see COPYING for more information -// *********************************************************************************** - -#ifndef HYDRO_BOUNDARY_HYDROBOUNDARY_HPP_ -#define HYDRO_BOUNDARY_HYDROBOUNDARY_HPP_ -#include -#include -#include "idefix.hpp" -#include "hydro_defs.hpp" -#include "grid.hpp" - -#ifdef WITH_MPI -#include "mpi.hpp" -#endif - -// forward class declaration -class DataBlock; -class Hydro; - -class HydroBoundary { - public: - void Init(Input &, Grid &, Hydro* ); - void SetBoundaries(real); ///< Set the ghost zones in all directions - void EnforceBoundaryDir(real, int); ///< write in the ghost zone in specific direction - void ReconstructVcField(IdefixArray4D &); ///< reconstruct cell-centered magnetic field - void ReconstructNormalField(int dir); ///< reconstruct normal field using divB=0 - - void EnforceFluxBoundaries(int dir); ///< Apply boundary condition conditions to the fluxes - - void EnrollUserDefBoundary(UserDefBoundaryFunc); - void EnrollInternalBoundary(InternalBoundaryFunc); - void EnrollFluxBoundary(UserDefBoundaryFunc); - - void EnforcePeriodic(int, BoundarySide ); ///< Enforce periodic BC in direction and side - void EnforceReflective(int, BoundarySide ); ///< Enforce reflective BC in direction and side - void EnforceOutflow(int, BoundarySide ); ///< Enforce outflow BC in direction and side - void EnforceShearingBox(real, int, BoundarySide ); ///< Enforce Shearing box BCs - - #ifdef WITH_MPI - Mpi mpi; ///< Mpi object when WITH_MPI is set - #endif - - // User defined Boundary conditions - UserDefBoundaryFunc userDefBoundaryFunc{NULL}; - bool haveUserDefBoundary{false}; - - // Internal boundary function - bool haveInternalBoundary{false}; - InternalBoundaryFunc internalBoundaryFunc{NULL}; - - // Flux boundary function - bool haveFluxBoundary{false}; - UserDefBoundaryFunc fluxBoundaryFunc{NULL}; - - // specific for loops on ghost cells - template - void BoundaryFor(const std::string &, - const int &, - const BoundarySide &, - Function ); - - template - void BoundaryForAll(const std::string &, - const int &, - const BoundarySide &, - Function ); - - template - void BoundaryForX1s(const std::string &, - const int &, - const BoundarySide &, - Function ); - - template - void BoundaryForX2s(const std::string &, - const int &, - const BoundarySide &, - Function ); - - template - void BoundaryForX3s(const std::string &, - const int &, - const BoundarySide &, - Function ); - IdefixArray4D sBArray; ///< Array use by shearingbox boundary conditions - - private: - Hydro *hydro; // pointer to parent hydro object - DataBlock *data; // pointer to parent datablock -}; - -#endif // HYDRO_BOUNDARY_HYDROBOUNDARY_HPP_ diff --git a/src/hydro/calcRiemannFlux.hpp b/src/hydro/calcRiemannFlux.hpp deleted file mode 100644 index 8136baeb..00000000 --- a/src/hydro/calcRiemannFlux.hpp +++ /dev/null @@ -1,79 +0,0 @@ -// *********************************************************************************** -// Idefix MHD astrophysical code -// Copyright(C) Geoffroy R. J. Lesur -// and other code contributors -// Licensed under CeCILL 2.1 License, see COPYING for more information -// *********************************************************************************** - -#ifndef HYDRO_CALCRIEMANNFLUX_HPP_ -#define HYDRO_CALCRIEMANNFLUX_HPP_ - -#include "hydro.hpp" -#include "dataBlock.hpp" - -#if MHD == YES -#include "hlldMHD.hpp" -#include "hllMHD.hpp" -#include "roeMHD.hpp" -#include "tvdlfMHD.hpp" -#else -#include "hllcHD.hpp" -#include "hllHD.hpp" -#include "tvdlfHD.hpp" -#include "roeHD.hpp" -#endif - -// Compute Riemann fluxes from states -template -void Hydro::CalcRiemannFlux(const real t) { - idfx::pushRegion("Hydro::CalcRiemannFlux"); - - if(hallStatus.status == UserDefFunction && dir == IDIR) { - if(hallDiffusivityFunc) - hallDiffusivityFunc(*data, t, xHall); - else - IDEFIX_ERROR("No user-defined Hall diffusivity function has been enrolled"); - } - - if(haveIsoSoundSpeed == UserDefFunction && dir == IDIR) { - if(isoSoundSpeedFunc) - isoSoundSpeedFunc(*data, t, isoSoundSpeedArray); - else - IDEFIX_ERROR("No user-defined isothermal sound speed function has been enrolled"); - } - - switch (mySolver) { -#if MHD == YES - case TVDLF: - TvdlfMHD(); - break; - case HLL: - HllMHD(); - break; - case HLLD: - HlldMHD(); - break; - case ROE: - RoeMHD(); - break; -#else - case TVDLF: - TvdlfHD(); - break; - case HLL: - HllHD(); - break; - case HLLC: - HllcHD(); - break; - case ROE: - RoeHD(); - break; -#endif - default: // do nothing - break; - } - - idfx::popRegion(); -} -#endif // HYDRO_CALCRIEMANNFLUX_HPP_ diff --git a/src/hydro/calcRightHandSide.hpp b/src/hydro/calcRightHandSide.hpp deleted file mode 100644 index 3dd593da..00000000 --- a/src/hydro/calcRightHandSide.hpp +++ /dev/null @@ -1,428 +0,0 @@ -// *********************************************************************************** -// Idefix MHD astrophysical code -// Copyright(C) Geoffroy R. J. Lesur -// and other code contributors -// Licensed under CeCILL 2.1 License, see COPYING for more information -// *********************************************************************************** - -#ifndef HYDRO_CALCRIGHTHANDSIDE_HPP_ -#define HYDRO_CALCRIGHTHANDSIDE_HPP_ - -#include "hydro.hpp" -#include "dataBlock.hpp" - - -// Compute the right handside in direction dir from conservative equation, with timestep dt -template -void Hydro::CalcRightHandSide(real t, real dt) { - idfx::pushRegion("Hydro::CalcRightHandSide"); - - IdefixArray4D Uc = this->Uc; - IdefixArray4D Vc = this->Vc; - IdefixArray4D Flux = this->FluxRiemann; - IdefixArray3D A = data->A[dir]; - IdefixArray3D dV = data->dV; - IdefixArray1D x1m = data->xl[IDIR]; - IdefixArray1D x1 = data->x[IDIR]; - - IdefixArray1D rt = data->rt; - IdefixArray1D dmu = data->dmu; - IdefixArray1D sinx2m = data->sinx2m; - IdefixArray1D sinx2 = data->sinx2; - IdefixArray1D dx = data->dx[dir]; - IdefixArray1D dx2 = data->dx[JDIR]; - IdefixArray3D invDt = this->InvDt; - IdefixArray3D cMax = this->cMax; - IdefixArray3D dMax = this->dMax; - IdefixArray4D viscSrc = this->viscosity.viscSrc; - IdefixArray2D fargoVelocity = data->fargo.meanVelocity; - - // Grid coarsening - bool haveGridCoarsening = false; - IdefixArray2D coarseningLevel; - - if(data->haveGridCoarsening) { - haveGridCoarsening = data->coarseningDirection[dir]; - coarseningLevel = data->coarseningLevel[dir]; - } - - // Gravitational potential - IdefixArray3D phiP = data->gravity.phiP; - bool needPotential = data->gravity.havePotential; - - // BodyForce - IdefixArray4D bodyForce = data->gravity.bodyForceVector; - bool needBodyForce = data->gravity.haveBodyForce; - - // parabolic terms - bool haveParabolicTerms = this->haveExplicitParabolicTerms; - - // Viscosity (source term only when non-cartesian geometry) - [[maybe_unused]] bool haveViscosity = this->viscosityStatus.isExplicit; - - // Fargo - bool haveFargo = data->haveFargo; - Fargo::FargoType fargoType = data->fargo.type; - - //Rotation - bool haveRotation = this->haveRotation; - [[maybe_unused]] real Omega = this->OmegaZ; - // disable rotation in cartesian geometry, as Coriolis is then treated as a source term - #if GEOMETRY == CARTESIAN - haveRotation = false; - #endif - - // shearingBox - [[maybe_unused]] bool haveShearingBox = this->haveShearingBox; - [[maybe_unused]] real sbS = this->sbS; - - if(haveFargo && fargoType == Fargo::userdef) { - data->fargo.GetFargoVelocity(t); - } - - constexpr const int ioffset = (dir==IDIR) ? 1 : 0; - constexpr const int joffset = (dir==JDIR) ? 1 : 0; - constexpr const int koffset = (dir==KDIR) ? 1 : 0; - -///////////////////////////////////////////////////////////////////////////// -// Flux correction (for fargo/non-cartesian geometry) -///////////////////////////////////////////////////////////////////////////// - idefix_for("CalcTotalFlux", - data->beg[KDIR],data->end[KDIR]+koffset, - data->beg[JDIR],data->end[JDIR]+joffset, - data->beg[IDIR],data->end[IDIR]+ioffset, - KOKKOS_LAMBDA (int k, int j, int i) { - // Add Fargo velocity to the fluxes - if(haveFargo || haveRotation) { - // Set mean advection direction - #if (GEOMETRY == CARTESIAN || GEOMETRY == POLAR) && DIMENSIONS >=2 - const int meanDir = JDIR; - #elif GEOMETRY == SPHERICAL && DIMENSIONS == 3 - const int meanDir = KDIR; - #else - const int meanDir = 0; - #endif - - // set mean advection velocity - real meanV = ZERO_F; - if(dir == IDIR) { - #if (GEOMETRY == CARTESIAN || GEOMETRY == POLAR) && DIMENSIONS >=2 - if(haveFargo) { - if(fargoType==Fargo::userdef) { - meanV = HALF_F*(fargoVelocity(k,i-1)+fargoVelocity(k,i)); - } else if(fargoType==Fargo::shearingbox) { - meanV = sbS * x1m(i); - } - } - #if GEOMETRY != CARTESIAN - if(haveRotation) { - meanV += x1m(i)*Omega; - } - #endif - #elif GEOMETRY == SPHERICAL && DIMENSIONS == 3 - if(haveFargo) { - meanV = HALF_F*(fargoVelocity(j,i-1)+fargoVelocity(j,i)); - } - if(haveRotation) { - meanV += x1m(i)*sinx2(j)*Omega; - } - #endif// GEOMETRY - } - #if GEOMETRY == SPHERICAL && DIMENSIONS == 3 - if((dir == JDIR)) { - if(haveFargo) { - meanV = HALF_F*(fargoVelocity(j-1,i)+fargoVelocity(j,i)); - } - if(haveRotation) { - meanV += x1(i)*sinx2m(j)*Omega; - } - } - #elif (GEOMETRY == CARTESIAN || GEOMETRY == POLAR) && DIMENSIONS >=2 - if((dir == KDIR) && haveFargo) { - if(fargoType==Fargo::userdef) { - meanV = HALF_F*(fargoVelocity(k-1,i)+fargoVelocity(k,i)); - } else if (fargoType==Fargo::shearingbox) { - meanV = sbS*x1(i); - } - } - #endif // GEOMETRY - - // Should do nothing along meanV direction, automatically satisfied - // since in that case meanV=0 - - #if HAVE_ENERGY - // Mignone (2012): second and third term of rhs of (25) - Flux(ENG,k,j,i) += meanV * (HALF_F*meanV*Flux(RHO,k,j,i) + Flux(MX1+meanDir,k,j,i)); - #endif - // Mignone+2012: second term of rhs of (24) - Flux(MX1+meanDir,k,j,i) += meanV * Flux(RHO,k,j,i); - } // have Fargo - - real Ax = A(k,j,i); - -#if GEOMETRY != CARTESIAN - if(Ax= 2) \ - || (GEOMETRY == CYLINDRICAL && COMPONENTS == 3) - if(dir==IDIR) { - // Conserve angular momentum, hence flux is R*Bphi - Flux(iMPHI,k,j,i) = Flux(iMPHI,k,j,i) * FABS(x1m(i)); - #if MHD == YES - // No area for this one - Flux(iBPHI,k,j,i) = Flux(iBPHI,k,j,i) / Ax; - #endif // MHD - } -#endif // GEOMETRY==POLAR OR CYLINDRICAL - -#if GEOMETRY == SPHERICAL - if(dir==IDIR) { - #if COMPONENTS == 3 - Flux(iMPHI,k,j,i) = Flux(iMPHI,k,j,i) * FABS(x1m(i)); - #endif // COMPONENTS == 3 - #if MHD == YES - EXPAND( , - Flux(iBTH,k,j,i) = Flux(iBTH,k,j,i) * x1m(i) / Ax; , - Flux(iBPHI,k,j,i) = Flux(iBPHI,k,j,i) * x1m(i) / Ax; ) - #endif // MHD - } else if(dir==JDIR) { - #if COMPONENTS == 3 - Flux(iMPHI,k,j,i) = Flux(iMPHI,k,j,i) * FABS(sinx2m(j)); - #if MHD == YES - Flux(iBPHI,k,j,i) = Flux(iBPHI,k,j,i) / Ax; - #endif // MHD - #endif // COMPONENTS = 3 - } -#endif // GEOMETRY == SPHERICAL - } - ); - - // If user has requested specific flux functions for the boundaries, here they come - if(boundary.haveFluxBoundary) boundary.EnforceFluxBoundaries(dir); - -///////////////////////////////////////////////////////////////////////////// -// Final conserved quantity budget from fluxes divergence -///////////////////////////////////////////////////////////////////////////// - idefix_for("CalcRightHandSide", - data->beg[KDIR],data->end[KDIR], - data->beg[JDIR],data->end[JDIR], - data->beg[IDIR],data->end[IDIR], - KOKKOS_LAMBDA (int k, int j, int i) { - real dtdV=dt / dV(k,j,i); - real rhs[NVAR]; - -#pragma unroll - for(int nv = 0 ; nv < NVAR ; nv++) { - rhs[nv] = - dtdV*(Flux(nv, k+koffset, j+joffset, i+ioffset) - Flux(nv, k, j, i)); - } - -#if GEOMETRY != CARTESIAN - if(dir==IDIR) { - #ifdef iMPHI - rhs[iMPHI] = rhs[iMPHI] / x1(i); - #endif - - #if (GEOMETRY == POLAR || GEOMETRY == CYLINDRICAL) && (defined iBPHI) && (MHD == YES) - rhs[iBPHI] = - dt / dx(i) * (Flux(iBPHI, k, j, i+1) - Flux(iBPHI, k, j, i) ); - - #elif (GEOMETRY == SPHERICAL) && (MHD == YES) - real q = dt / (x1(i)*dx(i)); - EXPAND( , - rhs[iBTH] = -q * ((Flux(iBTH, k, j, i+1) - Flux(iBTH, k, j, i) )); , - rhs[iBPHI] = -q * ((Flux(iBPHI, k, j, i+1) - Flux(iBPHI, k, j, i) )); ) - #endif - } else if(dir==JDIR) { - #if (GEOMETRY == SPHERICAL) && (COMPONENTS == 3) - rhs[iMPHI] /= FABS(sinx2(j)); - #if MHD == YES - rhs[iBPHI] = -dt / (rt(i)*dx(j)) * (Flux(iBPHI, k, j+1, i) - Flux(iBPHI, k, j, i)); - #endif // MHD - #endif // GEOMETRY - } - // Nothing for KDIR - -#endif // GEOMETRY != CARTESIAN - - - // Fargo terms to enfore conservation (actually substract back what was added in - // Totalflux loop) - if(haveFargo || haveRotation) { - // fetch fargo velocity when required - real meanV = ZERO_F; - #if (GEOMETRY == POLAR || GEOMETRY == CARTESIAN) && DIMENSIONS >=2 - if((dir==IDIR || dir == KDIR) && haveFargo) { - if(fargoType==Fargo::userdef) { - meanV = fargoVelocity(k,i); - } else if(fargoType==Fargo::shearingbox) { - meanV = sbS * x1(i); - } - } - #if GEOMETRY != CARTESIAN - if((dir==IDIR) && haveRotation) { - meanV += Omega*x1(i); - } - #endif - const int meanDir = JDIR; - #elif GEOMETRY == SPHERICAL && DIMENSIONS ==3 - if((dir==IDIR || dir == JDIR) && haveFargo) meanV = fargoVelocity(j,i); - if((dir==IDIR || dir == JDIR) && haveRotation) { - meanV += Omega*x1(i)*sinx2(j); - } - const int meanDir = KDIR; - #else - const int meanDir = 0; - #endif - // NB: MX1+meanDir = iMPHI - // Mignone+2012, rhs of eq. 24, last term - rhs[MX1+meanDir] -= meanV*rhs[RHO]; - #if HAVE_ENERGY - // Mignone+2012, rhs of eq.25, 4th and 5th term. NB: rhs[MX1+meanDir] is alreay the - // divergence of the *total* Momentum Flux F_my+w F_rho - rhs[ENG] -= meanV * ( HALF_F*meanV*rhs[RHO] + rhs[MX1+meanDir]); - #endif - } - -#if GEOMETRY != CARTESIAN - // Viscosity source terms - if(haveViscosity) { -#pragma unroll - for(int nv = 0 ; nv < COMPONENTS ; nv++) { - rhs[nv + VX1] += dt*viscSrc(nv,k,j,i); - } - } - -#endif // GEOMETRY != CARTESIAN - - // elmentary length for gradient computations - const int ig = ioffset*i + joffset*j + koffset*k; - real dl = dx(ig); - - -#if GEOMETRY == POLAR - if (dir==JDIR) - dl = dl*x1(i); - -#elif GEOMETRY == SPHERICAL - if (dir==JDIR) - dl = dl*rt(i); - else if (dir==KDIR) - dl = dl*rt(i)*dmu(j)/dx2(j); -#endif - - // Potential terms - if(needPotential) { - real dphi; - if (dir==IDIR) { - // Gravitational force in direction i - dphi = - 1.0/12.0 * ( - - phiP(k,j,i+2) + 8.0 * phiP(k,j,i+1) - - 8.0*phiP(k,j,i-1) + phiP(k,j,i-2)); - } - if (dir==JDIR) { - // Gravitational force in direction j - dphi = - 1.0/12.0 * ( - - phiP(k,j+2,i) + 8.0 * phiP(k,j+1,i) - - 8.0*phiP(k,j-1,i) + phiP(k,j-2,i)); - } - if (dir==KDIR) { - // Gravitational force in direction k - dphi = - 1.0/12.0 * ( - - phiP(k+2,j,i) + 8.0 * phiP(k+1,j,i) - - 8.0*phiP(k-1,j,i) + phiP(k-2,j,i)); - } - rhs[MX1+dir] += dt * Vc(RHO,k,j,i) * dphi /dl; - - #if HAVE_ENERGY - // Add gravitational force work as a source term - // This is equivalent to rho * v . nabla(phi) - // (note that Flux has already been multiplied by A) - rhs[ENG] += HALF_F * dtdV * - (Flux(RHO,k,j,i) + Flux(RHO, k+koffset, j+joffset, i+ioffset)) * dphi; - #endif - } - - // Body force - if(needBodyForce) { - rhs[MX1+dir] += dt * Vc(RHO,k,j,i) * bodyForce(dir,k,j,i); - #if HAVE_ENERGY - // rho * v . f, where rhov is taken as a volume average of Flux(RHO) - rhs[ENG] += HALF_F * dtdV * dl * - (Flux(RHO,k,j,i) + Flux(RHO, k+koffset, j+joffset, i+ioffset)) * - bodyForce(dir,k,j,i); - #endif // HAVE_ENERGY - - // Particular cases if we do not sweep all of the components - #if DIMENSIONS == 1 && COMPONENTS > 1 - EXPAND( , - rhs[MX2] += dt * Vc(RHO,k,j,i) * bodyForce(JDIR,k,j,i); , - rhs[MX3] += dt * Vc(RHO,k,j,i) * bodyForce(KDIR,k,j,i); ) - #if HAVE_ENERGY - rhs[ENG] += dt * (EXPAND( ZERO_F , - + Vc(RHO,k,j,i) * Vc(VX2,k,j,i) * bodyForce(JDIR,k,j,i) , - + Vc(RHO,k,j,i) * Vc(VX3,k,j,i) * bodyForce(KDIR,k,j,i) )); - #endif - #endif - #if DIMENSIONS == 2 && COMPONENTS == 3 - // Only add this term once! - if(dir==JDIR) { - rhs[MX3] += dt * Vc(RHO,k,j,i) * bodyForce(KDIR,k,j,i); - #if HAVE_ENERGY - rhs[ENG] += dt * Vc(RHO,k,j,i) * Vc(VX3,k,j,i) * bodyForce(KDIR,k,j,i); - #endif - } - #endif - } - - - // Timestep computation - // Change elementary grid spacing according to local coarsening level. - if(haveGridCoarsening) { - int factor; - //factor = 2^(coarsening-1) - if (dir==IDIR) { - factor = 1 << (coarseningLevel(k,j) - 1); - } - if (dir==JDIR) { - factor = 1 << (coarseningLevel(k,i) - 1); - } - if (dir==KDIR) { - factor = 1 << (coarseningLevel(j,i) - 1); - } - dl = dl * factor; - } - - // Compute dt from max signal speed - invDt(k,j,i) = invDt(k,j,i) + HALF_F*(cMax(k+koffset,j+joffset,i+ioffset) - + cMax(k,j,i)) / (dl); - - if(haveParabolicTerms) { - invDt(k,j,i) = invDt(k,j,i) + TWO_F* FMAX(dMax(k+koffset,j+joffset,i+ioffset), - dMax(k,j,i)) / (dl*dl); - } - - - // Evolve the field components -#pragma unroll - for(int nv = 0 ; nv < NVAR ; nv++) { - // Do not evolve the field components if they are computed by CT (i.e. if they are in Vs) - D_EXPAND( if(nv == BX1) { continue; } , - if(nv == BX2) { continue; } , - if(nv == BX3) { continue; } ) - - - Uc(nv,k,j,i) = Uc(nv,k,j,i) + rhs[nv]; - } - } - ); - - idfx::popRegion(); -} -#endif // HYDRO_CALCRIGHTHANDSIDE_HPP_ diff --git a/src/hydro/coarsenFlow.cpp b/src/hydro/coarsenFlow.cpp deleted file mode 100644 index 5b80e302..00000000 --- a/src/hydro/coarsenFlow.cpp +++ /dev/null @@ -1,403 +0,0 @@ -// *********************************************************************************** -// Idefix MHD astrophysical code -// Copyright(C) Geoffroy R. J. Lesur -// and other code contributors -// Licensed under CeCILL 2.1 License, see COPYING for more information -// *********************************************************************************** - -#include "hydro.hpp" -#include "dataBlock.hpp" - -KOKKOS_INLINE_FUNCTION int Kmax(int n, int m) { - return( n>m ? n : m); -} -// This function coarsen the flow according to the grid coarsening array - -void Hydro::CoarsenFlow(IdefixArray4D &Vi) { - idfx::pushRegion("Hydro::CoarsenFlow"); - - IdefixArray3D dV = data->dV; - for(int dir = 0 ; dir < DIMENSIONS ; dir++) { - if(!data->coarseningDirection[dir]) continue; - int begDir = data->beg[dir]; - int endDir = data->end[dir]; - - IdefixArray2D coarseningLevel = data->coarseningLevel[dir]; - - idefix_for("Hydro_CoarsenFlow", - 0, NVAR, - data->beg[KDIR],data->end[KDIR], - data->beg[JDIR],data->end[JDIR], - data->beg[IDIR],data->end[IDIR], - KOKKOS_LAMBDA (int n, int k, int j, int i) { - int factor, index; - int ioffset = 0; - int joffset = 0; - int koffset = 0; - //factor = 2^(coarsening-1) - if(dir==IDIR) { - factor = 1 << (coarseningLevel(k,j) - 1); - index = i; - ioffset = 1; - } - if(dir==JDIR) { - factor = 1 << (coarseningLevel(k,i) - 1); - index = j; - joffset = 1; - } - if(dir==KDIR) { - factor = 1 << (coarseningLevel(j,i) - 1); - index = k; - koffset = 1; - } - // check if coarsening is required in this region - if(factor>1) { - // We average the cells by groups of "factor" in direction "dir", - // so only the first element of each group will do the job. - if( (index-begDir)%factor == 0) { - real q = 0.0; - real V = 0.0; - for(int shift = 0 ; shift < factor ; shift++) { - q = q + Vi(n, k + shift*koffset, j + shift*joffset, i+shift*ioffset) - * dV(k + shift*koffset, j + shift*joffset, i+shift*ioffset); - V = V + dV(k + shift*koffset, j + shift*joffset, i+shift*ioffset); - } - // Average - q = q/V; - // Write back the cell elements - for(int shift = 0 ; shift < factor ; shift++) { - Vi(n, k + shift*koffset, j + shift*joffset, i+shift*ioffset) = q; - } - } - } - }); - } - idfx::popRegion(); -} - -void Hydro::CoarsenMagField(IdefixArray4D &Vsin) { -#if MHD == YES - idfx::pushRegion("Hydro::CoarsenMagField"); - #if DIMENSIONS >= 2 - /********************************** - * MHD Part * - * ********************************/ - for(int dir = 0 ; dir < DIMENSIONS ; dir++) { - if(!data->coarseningDirection[dir]) continue; - int begDir = data->beg[dir]; - int endDir = data->end[dir]; - - IdefixArray2D coarseningLevel = data->coarseningLevel[dir]; - - const int BXn = dir; - const int BXt = (dir == IDIR ? BX2s : BX1s); - const int BXb = (dir == KDIR ? BX2s : BX3s); - - IdefixArray3D An = data->A[dir]; - IdefixArray3D At = data->A[BXt]; - IdefixArray3D Ab = data->A[BXb]; - - int it = 0, ib = 0; - int jt = 0, jb = 0; - int kt = 0, kb = 0; - int endk = data->end[KDIR]; - int endj = data->end[JDIR]; - int endi = data->end[IDIR]; - - // find the index over which the faces are off-centered - // (trick so that we can write a single loop) - if(dir==IDIR) { - jt=1; - kb=1; - endj++; - #if DIMENSIONS == 3 - endk++; - #endif - } - if(dir==JDIR) { - it=1; - kb=1; - endi++; - #if DIMENSIONS == 3 - endk++; - #endif - } - if(dir==KDIR) { - it=1; - jb=1; - endi++; - endj++; - } - #if DIMENSIONS < 3 - // force kb to 0 - kb = 0; - #endif - - // Coarsen components normal to the coarsening direction first (BXt and BXb) - idefix_for("Hydro_CoarsenFlow_BXsn", - data->beg[KDIR],endk, - data->beg[JDIR],endj, - data->beg[IDIR],endi, - KOKKOS_LAMBDA (int k, int j, int i) { - int coarsening_t =1; - [[maybe_unused]] int coarsening_b = 1; - int index; - int ioffset = 0; - int joffset = 0; - int koffset = 0; - if(dir==IDIR) { - coarsening_t = Kmax(coarseningLevel(k,j-1), coarseningLevel(k,j)); - #if DIMENSIONS == 3 - coarsening_b = Kmax(coarseningLevel(k-1,j), coarseningLevel(k,j)); - #endif - ioffset = 1; - index=i; - } - if(dir==JDIR) { - coarsening_t = Kmax(coarseningLevel(k,i-1), coarseningLevel(k,i)); - #if DIMENSIONS == 3 - coarsening_b = Kmax(coarseningLevel(k-1,i), coarseningLevel(k,i)); - #endif - joffset = 1; - index=j; - } - if(dir==KDIR) { - coarsening_t = Kmax(coarseningLevel(j,i-1), coarseningLevel(j,i)); - coarsening_b = Kmax(coarseningLevel(j-1,i), coarseningLevel(j,i)); - koffset = 1; - index=k; - } - - //coarsening_t=0; - //coarsening_b=0; - // Treat t component - if(coarsening_t>1) { - int factor_t = 1 << (coarsening_t-1); - // We average the cells by groups of "factor" in direction "dir", - // so only the first element of each group will do the job. - if( (index-begDir)%factor_t == 0) { - real q = 0.0; - real A = 0.0; - for(int shift = 0 ; shift < factor_t ; shift++) { - q = q + Vsin(BXt, k + shift*koffset, j + shift*joffset, i+shift*ioffset) - * At(k + shift*koffset, j + shift*joffset, i+shift*ioffset); - A = A + At(k + shift*koffset, j + shift*joffset, i+shift*ioffset); - } - - // If the Area is zero, do a point average instead (this happens on the axis instance) - if(FABS(A) < 1e-10) { - // do a point average instead of a surface average - q=0.0; - A=0.0; - for(int shift = 0 ; shift < factor_t ; shift++) { - q = q + Vsin(BXt, k + shift*koffset, j + shift*joffset, i+shift*ioffset); - A = A + 1.0; - } - } - q = q/A; - // Write back the cell elements - for(int shift = 0 ; shift < factor_t ; shift++) { - Vsin(BXt, k + shift*koffset, j + shift*joffset, i+shift*ioffset) = q; - } - } - } - // Treat b component - #if DIMENSIONS == 3 - if(coarsening_b>1) { - int factor_b = 1 << (coarsening_b-1); - // We average the cells by groups of "factor" in direction "dir", - // so only the first element of each group will do the job. - if( (index-begDir)%factor_b == 0) { - real q = 0.0; - real A = 0.0; - for(int shift = 0 ; shift < factor_b ; shift++) { - q = q + Vsin(BXb, k + shift*koffset, j + shift*joffset, i+shift*ioffset) - * Ab(k + shift*koffset, j + shift*joffset, i+shift*ioffset); - A = A + Ab(k + shift*koffset, j + shift*joffset, i+shift*ioffset); - } - // If the Area is zero, do a point average instead (this happens on the axis instance) - if(FABS(A) < 1e-10) { - // do a point average instead of a surface average - q=0.0; - A=0.0; - for(int shift = 0 ; shift < factor_b ; shift++) { - q = q + Vsin(BXb, k + shift*koffset, j + shift*joffset, i+shift*ioffset); - A = A + 1.0; - } - } - // Average - q = q/A; - // Write back the cell elements - for(int shift = 0 ; shift < factor_b ; shift++) { - Vsin(BXb, k + shift*koffset, j + shift*joffset, i+shift*ioffset) = q; - } - } - } - #endif - }); - - // Coarsen components parralel the coarsening direction (BXn) - idefix_for("Hydro_CoarsenFlow_BXsn", - data->beg[KDIR],data->end[KDIR], - data->beg[JDIR],data->end[JDIR], - data->beg[IDIR],data->end[IDIR], - KOKKOS_LAMBDA (int k, int j, int i) { - int coarsening; - int index; - int ioffset = 0; - int joffset = 0; - int koffset = 0; - // Pick the highest coarsening level of the current coordinates and its immediate neighbours - if(dir==IDIR) { - coarsening = coarseningLevel(k,j); - coarsening = Kmax(coarsening, coarseningLevel(k,j-1)); - coarsening = Kmax(coarsening, coarseningLevel(k,j+1)); - #if DIMENSIONS == 3 - coarsening = Kmax(coarsening, coarseningLevel(k-1,j)); - coarsening = Kmax(coarsening, coarseningLevel(k+1,j)); - #endif - ioffset = 1; - index=i; - } - if(dir==JDIR) { - coarsening = coarseningLevel(k,i); - coarsening = Kmax(coarsening, coarseningLevel(k,i-1)); - coarsening = Kmax(coarsening, coarseningLevel(k,i+1)); - #if DIMENSIONS == 3 - coarsening = Kmax(coarsening, coarseningLevel(k-1,i)); - coarsening = Kmax(coarsening, coarseningLevel(k+1,i)); - #endif - joffset = 1; - index=j; - } - if(dir==KDIR) { - coarsening = coarseningLevel(j,i); - coarsening = Kmax(coarsening, coarseningLevel(j,i-1)); - coarsening = Kmax(coarsening, coarseningLevel(j,i+1)); - coarsening = Kmax(coarsening, coarseningLevel(j-1,i)); - coarsening = Kmax(coarsening, coarseningLevel(j+1,i)); - koffset = 1; - index=k; - } - - if(coarsening > 1) { - // the current cell had tangential field components which have been coarsened. Hence, - // We reconstruct the parrallel field component with divB=0 - - int factor = 1 << (coarsening-1); - if( (index-begDir)%factor == 0) { - for(int shift = 0 ; shift < factor-1 ; shift++) { - real qt = Vsin(BXt,k+kt+shift*koffset, j+jt+shift*joffset, i+it+shift*ioffset) - * At(k+kt+shift*koffset, j+jt+shift*joffset, i+it+shift*ioffset) - - Vsin(BXt,k+shift*koffset, j+shift*joffset, i+shift*ioffset) - * At(k+shift*koffset, j+shift*joffset, i+shift*ioffset); - real qb = Vsin(BXb,k+kb+shift*koffset, j+jb+shift*joffset, i+ib+shift*ioffset) - * Ab(k+kb+shift*koffset, j+jb+shift*joffset, i+ib+shift*ioffset) - - Vsin(BXb,k+shift*koffset, j+shift*joffset, i+shift*ioffset) - * Ab(k+shift*koffset, j+shift*joffset, i+shift*ioffset); -/* - if(i==10 && j == 3 && k == 10) { - idfx::cout << "shift=" << shift << " ; oldBxn=" << Vsin(BXn, k+(shift+1)*koffset, j+(shift+1)*joffset, i+(shift+1)*ioffset) << std::endl; - idfx::cout << "qt=" << qt << " ; qb=" << qb << std::endl; - }*/ - - Vsin(BXn, k+(shift+1)*koffset, j+(shift+1)*joffset, i+(shift+1)*ioffset) = - 1.0/An(k+(shift+1)*koffset, j+(shift+1)*joffset, i+(shift+1)*ioffset) * - ( - Vsin(BXn, k+shift*koffset, j+shift*joffset, i+shift*ioffset) * - An(k+shift*koffset, j+shift*joffset, i+shift*ioffset) - - qt - qb - ); - /* - if(i==10 && j == 3 && k == 10) { - idfx::cout << "shift=" << shift << " ; newsBxn=" << Vsin(BXn, k+(shift+1)*koffset, j+(shift+1)*joffset, i+(shift+1)*ioffset) << std::endl; - }*/ - } - } - } - }); - } - idfx::popRegion(); -#endif -} - -void Hydro::CoarsenVectorPotential() { -#if MHD == YES && defined(EVOLVE_VECTOR_POTENTIAL) - idfx::pushRegion("Hydro::VectorPotential"); - - IdefixArray4D Ve = this->Ve; - - #if DIMENSIONS < 3 - IDEFIX_ERROR("Grid Coarsening with vector potential is not implemented for DIMENSIONS < 3"); - #endif - - for(int dir = 0 ; dir < DIMENSIONS ; dir++) { - if(!data->coarseningDirection[dir]) continue; - int begDir = data->beg[dir]; - int endDir = data->end[dir]; - - const int AXn = dir; - const int AXt = (dir == IDIR ? AX2e : AX1e); - const int AXb = (dir == KDIR ? AX2e : AX3e); - - IdefixArray2D coarseningLevel = data->coarseningLevel[dir]; - - idefix_for("Hydro_CoarsenFlow", - data->beg[KDIR],data->end[KDIR], - data->beg[JDIR],data->end[JDIR], - data->beg[IDIR],data->end[IDIR], - KOKKOS_LAMBDA (int k, int j, int i) { - int factor, index; - int ioffset = 0; - int joffset = 0; - int koffset = 0; - //factor = 2^(coarsening-1) - if(dir==IDIR) { - factor = 1 << (coarseningLevel(k,j) - 1); - index = i; - ioffset = 1; - } - if(dir==JDIR) { - factor = 1 << (coarseningLevel(k,i) - 1); - index = j; - joffset = 1; - } - if(dir==KDIR) { - factor = 1 << (coarseningLevel(j,i) - 1); - index = k; - koffset = 1; - } - // check if coarsening is required in this region - if(factor>1) { - // We average the cells by groups of "factor" in direction "dir", - // so only the first element of each group will do the job. - if( (index-begDir)%factor == 0) { - real q = 0.0; - for(int shift = 0 ; shift < factor ; shift++) { - q = q + Ve(AXn, k + shift*koffset, j + shift*joffset, i+shift*ioffset); - } - // Average - q = q/factor; - // Write back the cell elements - for(int shift = 0 ; shift < factor ; shift++) { - Ve(AXn, k + shift*koffset, j + shift*joffset, i+shift*ioffset) = q; - real a = ((real) shift) / ((real) factor); - - Ve(AXt, k + shift*koffset, j + shift*joffset, i+shift*ioffset) = - a * Ve(AXt, k + factor*koffset, j + factor*joffset, i+factor*ioffset) - + (1-a) * Ve(AXt, k, j, i); - - Ve(AXb, k + shift*koffset, j + shift*joffset, i+shift*ioffset) = - a * Ve(AXb, k + factor*koffset, j + factor*joffset, i+factor*ioffset) - + (1-a) * Ve(AXb, k, j, i); - } - } - } - }); - } - - #endif // DIMENSIONS >= 2 - idfx::popRegion(); - - #endif // MHD -} diff --git a/src/hydro/convertConsToPrim.cpp b/src/hydro/convertConsToPrim.cpp deleted file mode 100644 index 47028a03..00000000 --- a/src/hydro/convertConsToPrim.cpp +++ /dev/null @@ -1,86 +0,0 @@ -// *********************************************************************************** -// Idefix MHD astrophysical code -// Copyright(C) Geoffroy R. J. Lesur -// and other code contributors -// Licensed under CeCILL 2.1 License, see COPYING for more information -// *********************************************************************************** - -#include "hydro.hpp" -#include "dataBlock.hpp" - -#if MHD == YES -#include "convertConsToPrimMHD.hpp" -#else -#include "convertConsToPrimHD.hpp" -#endif - -// Convect Conservative to Primitive variable -void Hydro::ConvertConsToPrim() { - idfx::pushRegion("Hydro::ConvertConsToPrim"); - - IdefixArray4D Vc = this->Vc; - IdefixArray4D Uc = this->Uc; - real gamma_m1=this->gamma-ONE_F; - - #if MHD == YES - #ifdef EVOLVE_VECTOR_POTENTIAL - emf.ComputeMagFieldFromA(Ve,Vs); - #endif - boundary.ReconstructVcField(Uc); - #endif - - idefix_for("ConsToPrim", - 0,data->np_tot[KDIR], - 0,data->np_tot[JDIR], - 0,data->np_tot[IDIR], - KOKKOS_LAMBDA (int k, int j, int i) { - real U[NVAR]; - real V[NVAR]; - -#pragma unroll - for(int nv = 0 ; nv < NVAR; nv++) { - U[nv] = Uc(nv,k,j,i); - } - - K_ConsToPrim(V,U,gamma_m1); - -#pragma unroll - for(int nv = 0 ; nv Vc = this->Vc; - IdefixArray4D Uc = this->Uc; - real gamma_m1=this->gamma-ONE_F; - - idefix_for("ConvertPrimToCons", - 0,data->np_tot[KDIR], - 0,data->np_tot[JDIR], - 0,data->np_tot[IDIR], - KOKKOS_LAMBDA (int k, int j, int i) { - real U[NVAR]; - real V[NVAR]; - -#pragma unroll - for(int nv = 0 ; nv < NVAR; nv++) { - V[nv] = Vc(nv,k,j,i); - } - - K_PrimToCons(U,V,gamma_m1); - -#pragma unroll - for(int nv = 0 ; nv -// and other code contributors -// Licensed under CeCILL 2.1 License, see COPYING for more information -// *********************************************************************************** - -#ifndef HYDRO_CONVERTCONSTOPRIMHD_HPP_ -#define HYDRO_CONVERTCONSTOPRIMHD_HPP_ - -#include "idefix.hpp" -#include "hydro.hpp" - -KOKKOS_INLINE_FUNCTION void K_ConsToPrim( - real Vc[], - real Uc[], - real gamma_m1) { - Vc[RHO] = Uc[RHO]; - - EXPAND( Vc[VX1] = Uc[MX1]/Uc[RHO]; , - Vc[VX2] = Uc[MX2]/Uc[RHO]; , - Vc[VX3] = Uc[MX3]/Uc[RHO]; ) - -#if HAVE_ENERGY - real kin; - kin = HALF_F / Uc[RHO] * (EXPAND( Uc[MX1]*Uc[MX1] , - + Uc[MX2]*Uc[MX2] , - + Uc[MX3]*Uc[MX3] )); - - Vc[PRS] = gamma_m1 * (Uc[ENG] - kin); - - // Check pressure positivity - if(Vc[PRS]<= ZERO_F) { - #ifdef SMALL_PRESSURE_TEMPERATURE - Vc[PRS] = SMALL_PRESSURE_TEMPERATURE*Vc[RHO]; - #else - Vc[PRS] = SMALL_PRESSURE_FIX; - #endif - Uc[ENG] = Vc[PRS]/gamma_m1+kin; - } -#endif // Have_energy -} - -KOKKOS_INLINE_FUNCTION void K_PrimToCons(real *KOKKOS_RESTRICT Uc, - const real *KOKKOS_RESTRICT Vc, - real gamma_m1) { - Uc[RHO] = Vc[RHO]; - - EXPAND( Uc[MX1] = Vc[VX1]*Vc[RHO]; , - Uc[MX2] = Vc[VX2]*Vc[RHO]; , - Uc[MX3] = Vc[VX3]*Vc[RHO]; ) - -#if HAVE_ENERGY - - Uc[ENG] = Vc[PRS] / gamma_m1 - + HALF_F * Vc[RHO] * (EXPAND( Vc[VX1]*Vc[VX1] , - + Vc[VX2]*Vc[VX2] , - + Vc[VX3]*Vc[VX3] )); -#endif // Have_energy -} - -#endif // HYDRO_CONVERTCONSTOPRIMHD_HPP_ diff --git a/src/hydro/convertConsToPrimMHD.hpp b/src/hydro/convertConsToPrimMHD.hpp deleted file mode 100644 index 634e1de5..00000000 --- a/src/hydro/convertConsToPrimMHD.hpp +++ /dev/null @@ -1,80 +0,0 @@ -// *********************************************************************************** -// Idefix MHD astrophysical code -// Copyright(C) Geoffroy R. J. Lesur -// and other code contributors -// Licensed under CeCILL 2.1 License, see COPYING for more information -// *********************************************************************************** - -#ifndef HYDRO_CONVERTCONSTOPRIMMHD_HPP_ -#define HYDRO_CONVERTCONSTOPRIMMHD_HPP_ - -#include "idefix.hpp" -#include "hydro.hpp" - - -KOKKOS_INLINE_FUNCTION void K_ConsToPrim(real Vc[], real Uc[], real gamma_m1) { - Vc[RHO] = Uc[RHO]; - - EXPAND( Vc[VX1] = Uc[MX1]/Uc[RHO]; , - Vc[VX2] = Uc[MX2]/Uc[RHO]; , - Vc[VX3] = Uc[MX3]/Uc[RHO]; ) - - EXPAND( Vc[BX1] = Uc[BX1]; , - Vc[BX2] = Uc[BX2]; , - Vc[BX3] = Uc[BX3]; ) - - -#if HAVE_ENERGY - real kin,mag; - kin = HALF_F / Uc[RHO] * (EXPAND( Uc[MX1]*Uc[MX1] , - + Uc[MX2]*Uc[MX2] , - + Uc[MX3]*Uc[MX3] )); - - mag = HALF_F * (EXPAND( Uc[BX1]*Uc[BX1] , - + Uc[BX2]*Uc[BX2] , - + Uc[BX3]*Uc[BX3] )); - - - Vc[PRS] = gamma_m1 * (Uc[ENG] - kin - mag); - - // Check pressure positivity - if(Vc[PRS]<= ZERO_F) { - #ifdef SMALL_PRESSURE_TEMPERATURE - Vc[PRS] = SMALL_PRESSURE_TEMPERATURE*Vc[RHO]; - #else - Vc[PRS] = SMALL_PRESSURE_FIX; - #endif - - Uc[ENG] = Vc[PRS]/gamma_m1+kin+mag; - } -#endif // Have_energy -} - -KOKKOS_INLINE_FUNCTION void K_PrimToCons(real Uc[], real Vc[], real gamma_m1) { - Uc[RHO] = Vc[RHO]; - - EXPAND( Uc[MX1] = Vc[VX1]*Vc[RHO]; , - Uc[MX2] = Vc[VX2]*Vc[RHO]; , - Uc[MX3] = Vc[VX3]*Vc[RHO]; ) - - - EXPAND( Uc[BX1] = Vc[BX1]; , - Uc[BX2] = Vc[BX2]; , - Uc[BX3] = Vc[BX3]; ) - -#if HAVE_ENERGY - - Uc[ENG] = Vc[PRS] / gamma_m1 - + HALF_F * Vc[RHO] * (EXPAND( Vc[VX1]*Vc[VX1] , - + Vc[VX2]*Vc[VX2] , - + Vc[VX3]*Vc[VX3] )) - + HALF_F * (EXPAND( Uc[BX1]*Uc[BX1] , - + Uc[BX2]*Uc[BX2] , - + Uc[BX3]*Uc[BX3] )); -#endif // Have_energy -} - - - - -#endif // HYDRO_CONVERTCONSTOPRIMMHD_HPP_ diff --git a/src/hydro/electromotiveforce/CMakeLists.txt b/src/hydro/electromotiveforce/CMakeLists.txt deleted file mode 100644 index 2edc4c62..00000000 --- a/src/hydro/electromotiveforce/CMakeLists.txt +++ /dev/null @@ -1,11 +0,0 @@ -target_sources(idefix - PUBLIC ${CMAKE_CURRENT_LIST_DIR}/calcCornerEmf.cpp - PUBLIC ${CMAKE_CURRENT_LIST_DIR}/calcNonidealEMF.cpp - PUBLIC ${CMAKE_CURRENT_LIST_DIR}/calcRiemannEmf.cpp - PUBLIC ${CMAKE_CURRENT_LIST_DIR}/electroMotiveForce.cpp - PUBLIC ${CMAKE_CURRENT_LIST_DIR}/electroMotiveForce.hpp - PUBLIC ${CMAKE_CURRENT_LIST_DIR}/EMFexchange.cpp - PUBLIC ${CMAKE_CURRENT_LIST_DIR}/enforceEMFBoundary.cpp - PUBLIC ${CMAKE_CURRENT_LIST_DIR}/evolveMagField.cpp - PUBLIC ${CMAKE_CURRENT_LIST_DIR}/evolveVectorPotential.cpp - ) diff --git a/src/hydro/electromotiveforce/calcRiemannEmf.cpp b/src/hydro/electromotiveforce/calcRiemannEmf.cpp deleted file mode 100644 index 787df098..00000000 --- a/src/hydro/electromotiveforce/calcRiemannEmf.cpp +++ /dev/null @@ -1,243 +0,0 @@ -// *********************************************************************************** -// Idefix MHD astrophysical code -// Copyright(C) Geoffroy R. J. Lesur -// and other code contributors -// Licensed under CeCILL 2.1 License, see COPYING for more information -// *********************************************************************************** - -#include "hydro.hpp" -#include "dataBlock.hpp" - - -KOKKOS_INLINE_FUNCTION real MC_LIM2 (const real dp, const real dm) { - real dc, scrh; - - if (dp*dm < ZERO_F) return(ZERO_F); - - dc = HALF_F*(dp + dm); - scrh = TWO_F*(std::fabs(dp) < std::fabs(dm) ? dp:dm); - return (std::fabs(dc) < std::fabs(scrh) ? dc:scrh); -} - - - -void ElectroMotiveForce::CalcRiemannAverage() { - idfx::pushRegion("ElectroMotiveForce::calcRiemannAverage"); - -#if EMF_AVERAGE == UCT_HLLD || EMF_AVERAGE == UCT_HLL - - // Corned EMFs - IdefixArray3D ez = this->ez; -#if DIMENSIONS == 3 - IdefixArray3D ex = this->ex; - IdefixArray3D ey = this->ey; - - // Face-centered EMFs - IdefixArray3D exj = this->exj; - IdefixArray3D exk = this->exk; - IdefixArray3D eyi = this->eyi; - IdefixArray3D eyk = this->eyk; -#endif - - // Face-centered EMFs - IdefixArray3D ezi = this->ezi; - IdefixArray3D ezj = this->ezj; - - IdefixArray3D axL = this->axL; - IdefixArray3D axR = this->axR; - IdefixArray3D ayL = this->ayL; - IdefixArray3D ayR = this->ayR; -#if DIMENSIONS == 3 - IdefixArray3D azL = this->azL; - IdefixArray3D azR = this->azR; -#endif - - IdefixArray3D dxL = this->dxL; - IdefixArray3D dxR = this->dxR; - IdefixArray3D dyL = this->dyL; - IdefixArray3D dyR = this->dyR; -#if DIMENSIONS == 3 - IdefixArray3D dzL = this->dzL; - IdefixArray3D dzR = this->dzR; -#endif - - IdefixArray4D Vs = hydro->Vs; - - idefix_for("CalcCenterEMF", - data->beg[KDIR],data->end[KDIR]+KOFFSET, - data->beg[JDIR],data->end[JDIR]+JOFFSET, - data->beg[IDIR],data->end[IDIR]+IOFFSET, - KOKKOS_LAMBDA (int k, int j, int i) { - // IDIR -#if DIMENSIONS >= 2 - [[maybe_unused]] real phi, vL, vR, dv, bL, bR, db; - [[maybe_unused]] real aL, aR, dL, dR; - - [[maybe_unused]] int im = i-1, jm = j-1, km = k-1; - - // EMF: Z component at (i-1/2, j-1/2, k) - aL = HALF_F*(axL(k,jm,i) + axL(k,jm+1,i)); - aR = HALF_F*(axR(k,jm,i) + axR(k,jm+1,i)); - dL = HALF_F*(dxL(k,jm,i) + dxL(k,jm+1,i)); - dR = HALF_F*(dxR(k,jm,i) + dxR(k,jm+1,i)); - - db = MC_LIM2(Vs(BX2s,k,j,im+1) - Vs(BX2s,k,j,im), - Vs(BX2s,k,j,im) - Vs(BX2s,k,j,im-1)); - bL = Vs(BX2s,k,j,im) + HALF_F*db; - - db = MC_LIM2(Vs(BX2s,k,j,i+1) - Vs(BX2s,k,j,i), - Vs(BX2s,k,j,i) - Vs(BX2s,k,j,i-1)); - bR = Vs(BX2s,k,j,i) - HALF_F*db; - - dv = MC_LIM2(ezj(k,j,im+1) - ezj(k,j,im), - ezj(k,j,im) - ezj(k,j,im-1)); - vL = ezj(k,j,im) + HALF_F*dv; - - dv = MC_LIM2(ezj(k,j,i+1) - ezj(k,j,i), - ezj(k,j,i) - ezj(k,j,i-1)); - vR = ezj(k,j,i) - HALF_F*dv; - - phi = dR*bR - dL*bL; - ez(k,j,i) = (aL*vL*bL + aR*vR*bR) + phi; -#endif - -#if DIMENSIONS == 3 - // EMF: Y component at (i-1/2, j, k-1/2) - aL = HALF_F*(axL(km,j,i) + axL(km+1,j,i)); - aR = HALF_F*(axR(km,j,i) + axR(km+1,j,i)); - dL = HALF_F*(dxL(km,j,i) + dxL(km+1,j,i)); - dR = HALF_F*(dxR(km,j,i) + dxR(km+1,j,i)); - - db = MC_LIM2(Vs(BX3s,k,j,im+1) - Vs(BX3s,k,j,im), - Vs(BX3s,k,j,im) - Vs(BX3s,k,j,im-1)); - bL = Vs(BX3s,k,j,im) + HALF_F*db; - - db = MC_LIM2(Vs(BX3s,k,j,i+1) - Vs(BX3s,k,j,i), - Vs(BX3s,k,j,i) - Vs(BX3s,k,j,i-1)); - bR = Vs(BX3s,k,j,i) - HALF_F*db; - - dv = MC_LIM2(eyk(k,j,im+1) - eyk(k,j,im), - eyk(k,j,im) - eyk(k,j,im-1)); - vL = eyk(k,j,im) + HALF_F*dv; - - dv = MC_LIM2(eyk(k,j,i+1) - eyk(k,j,i), - eyk(k,j,i) - eyk(k,j,i-1)); - vR = eyk(k,j,i) - HALF_F*dv; - - phi = dR*bR - dL*bL; - ey(k,j,i) = (aL*vL*bL + aR*vR*bR) - phi; -#endif - - // JDIR -#if DIMENSIONS >= 2 - #if DIMENSIONS == 3 - // EMF: X component at (i, j-1/2, k-1/2) - aL = HALF_F*(ayL(km,j,i) + ayL(km+1,j,i)); - aR = HALF_F*(ayR(km,j,i) + ayR(km+1,j,i)); - dL = HALF_F*(dyL(km,j,i) + dyL(km+1,j,i)); - dR = HALF_F*(dyR(km,j,i) + dyR(km+1,j,i)); - - db = MC_LIM2(Vs(BX3s,k,jm+1,i) - Vs(BX3s,k,jm,i), - Vs(BX3s,k,jm,i) - Vs(BX3s,k,jm-1,i)); - bL = Vs(BX3s,k,jm,i) + HALF_F*db; - - db = MC_LIM2(Vs(BX3s,k,j+1,i) - Vs(BX3s,k,j,i), - Vs(BX3s,k,j,i) - Vs(BX3s,k,j-1,i)); - bR = Vs(BX3s,k,j,i) - HALF_F*db; - - dv = MC_LIM2(exk(k,jm+1,i) - exk(k,jm,i), - exk(k,jm,i) - exk(k,jm-1,i)); - vL = exk(k,jm,i) + HALF_F*dv; - - dv = MC_LIM2(exk(k,j+1,i) - exk(k,j,i), - exk(k,j,i) - exk(k,j-1,i)); - vR = exk(k,j,i) - HALF_F*dv; - - phi = dR*bR - dL*bL; - ex(k,j,i) = (aL*vL*bL + aR*vR*bR) + phi; - #endif - - // EMF: Z component at (i-1/2, j-1/2, k) - aL = HALF_F*(ayL(k,j,im) + ayL(k,j,im+1)); - aR = HALF_F*(ayR(k,j,im) + ayR(k,j,im+1)); - dL = HALF_F*(dyL(k,j,im) + dyL(k,j,im+1)); - dR = HALF_F*(dyR(k,j,im) + dyR(k,j,im+1)); - - db = MC_LIM2(Vs(BX1s,k,jm+1,i) - Vs(BX1s,k,jm,i), - Vs(BX1s,k,jm,i) - Vs(BX1s,k,jm-1,i)); - bL = Vs(BX1s,k,jm,i) + HALF_F*db; - - db = MC_LIM2(Vs(BX1s,k,j+1,i) - Vs(BX1s,k,j,i), - Vs(BX1s,k,j,i) - Vs(BX1s,k,j-1,i)); - bR = Vs(BX1s,k,j,i) - HALF_F*db; - - dv = MC_LIM2(ezi(k,jm+1,i) - ezi(k,jm,i), - ezi(k,jm,i) - ezi(k,jm-1,i)); - vL = ezi(k,jm,i) + HALF_F*dv; - - dv = MC_LIM2(ezi(k,j+1,i) - ezi(k,j,i), - ezi(k,j,i) - ezi(k,j-1,i)); - vR = ezi(k,j,i) - HALF_F*dv; - - phi = dR*bR - dL*bL; - ez(k,j,i) += (aL*vL*bL + aR*vR*bR) - phi; -#endif - - // KDIR -#if DIMENSIONS == 3 - // EMF: Y component at (i-1/2, j, k-1/2) - aL = HALF_F*(azL(k,j,im) + azL(k,j,im+1)); - aR = HALF_F*(azR(k,j,im) + azR(k,j,im+1)); - dL = HALF_F*(dzL(k,j,im) + dzL(k,j,im+1)); - dR = HALF_F*(dzR(k,j,im) + dzR(k,j,im+1)); - - db = MC_LIM2(Vs(BX1s,km+1,j,i) - Vs(BX1s,km,j,i), - Vs(BX1s,km,j,i) - Vs(BX1s,km-1,j,i)); - bL = Vs(BX1s,km,j,i) + HALF_F*db; - - db = MC_LIM2(Vs(BX1s,k+1,j,i) - Vs(BX1s,k,j,i), - Vs(BX1s,k,j,i) - Vs(BX1s,k-1,j,i)); - bR = Vs(BX1s,k,j,i) - HALF_F*db; - - dv = MC_LIM2(eyi(km+1,j,i) - eyi(km,j,i), - eyi(km,j,i) - eyi(km-1,j,i)); - vL = eyi(km,j,i) + HALF_F*dv; - - dv = MC_LIM2(eyi(k+1,j,i) - eyi(k,j,i), - eyi(k,j,i) - eyi(k-1,j,i)); - vR = eyi(k,j,i) - HALF_F*dv; - - phi = dR*bR - dL*bL; - ey(k,j,i) += (aL*vL*bL + aR*vR*bR) + phi; - - // EMF: X component at (i, j-1/2, k-1/2) - aL = HALF_F*(azL(k,jm,i) + azL(k,jm+1,i)); - aR = HALF_F*(azR(k,jm,i) + azR(k,jm+1,i)); - dL = HALF_F*(dzL(k,jm,i) + dzL(k,jm+1,i)); - dR = HALF_F*(dzR(k,jm,i) + dzR(k,jm+1,i)); - - db = MC_LIM2(Vs(BX2s,km+1,j,i) - Vs(BX2s,km,j,i), - Vs(BX2s,km,j,i) - Vs(BX2s,km-1,j,i)); - bL = Vs(BX2s,km,j,i) + HALF_F*db; - - db = MC_LIM2(Vs(BX2s,k+1,j,i) - Vs(BX2s,k,j,i), - Vs(BX2s,k,j,i) - Vs(BX2s,k-1,j,i)); - bR = Vs(BX2s,k,j,i) - HALF_F*db; - - dv = MC_LIM2(exj(km+1,j,i) - exj(km,j,i), - exj(km,j,i) - exj(km-1,j,i)); - vL = exj(km,j,i) + HALF_F*dv; - - dv = MC_LIM2(exj(k+1,j,i) - exj(k,j,i), - exj(k,j,i) - exj(k-1,j,i)); - vR = exj(k,j,i) - HALF_F*dv; - - phi = dR*bR - dL*bL; - ex(k,j,i) += (aL*vL*bL + aR*vR*bR) - phi; -#endif - } - ); -#endif // EMF_AVERAGE - - idfx::popRegion(); -} diff --git a/src/hydro/electromotiveforce/electroMotiveForce.hpp b/src/hydro/electromotiveforce/electroMotiveForce.hpp deleted file mode 100644 index e627335a..00000000 --- a/src/hydro/electromotiveforce/electroMotiveForce.hpp +++ /dev/null @@ -1,147 +0,0 @@ -// *********************************************************************************** -// Idefix MHD astrophysical code -// Copyright(C) Geoffroy R. J. Lesur -// and other code contributors -// Licensed under CeCILL 2.1 License, see COPYING for more information -// *********************************************************************************** - -#ifndef HYDRO_ELECTROMOTIVEFORCE_ELECTROMOTIVEFORCE_HPP_ -#define HYDRO_ELECTROMOTIVEFORCE_ELECTROMOTIVEFORCE_HPP_ - -#include "idefix.hpp" -#include "input.hpp" - -// Forward declarations -class Hydro; -class DataBlock; - -class ElectroMotiveForce { - public: - enum AveragingType {none, arithmetic, uct0, uct_contact, uct_hll, uct_hlld}; - - // Type of averaging - AveragingType averaging{none}; - - // Face centered emf components - IdefixArray3D exj; - IdefixArray3D exk; - IdefixArray3D eyi; - IdefixArray3D eyk; - IdefixArray3D ezi; - IdefixArray3D ezj; - - // Edge centered emf components - IdefixArray3D ex; - IdefixArray3D ey; - IdefixArray3D ez; - -// Required by uct_contact averaging - IdefixArray3D svx; - IdefixArray3D svy; - IdefixArray3D svz; - -// required by uct_hll averaging - IdefixArray3D axL; - IdefixArray3D axR; - IdefixArray3D ayL; - IdefixArray3D ayR; - #if DIMENSIONS == 3 - IdefixArray3D azL; - IdefixArray3D azR; - - IdefixArray3D dzL; - IdefixArray3D dzR; - #endif - IdefixArray3D dxL; - IdefixArray3D dxR; - IdefixArray3D dyL; - IdefixArray3D dyR; - - - IdefixArray3D Ex1; - IdefixArray3D Ex2; - IdefixArray3D Ex3; - - // Helper arrays for shearing box - IdefixArray2D sbEyL; - IdefixArray2D sbEyR; - IdefixArray2D sbEyRL; - - // Range of existence - - // Init from Hydro class - void Init(Input &, Hydro *); - - void EvolveMagField(real, real, IdefixArray4D&); - void CalcCornerEMF(real ); - void ShowConfig(); - - // Different flavors of EMF average schemes - void CalcRiemannAverage(); - void CalcArithmeticAverage(); - void CalcCellCenteredEMF(); - void CalcUCT0Average(); - void CalcContactAverage(); - - // Enforce boundary conditions on the EMFs. - void EnforceEMFBoundary(); - void CalcNonidealEMF(real ); - - // Specific routines to symmetrize EMFs with shearing box BCs - void SymmetrizeEMFShearingBox(); - void ExtrapolateEMFShearingBox(BoundarySide, - IdefixArray2D, - IdefixArray2D); - - // Routines for evolving the magnetic potential (only available when EVOLVE_VECTOR_POTENTIAL) - void EvolveVectorPotential(real, IdefixArray4D &); - void ComputeMagFieldFromA(IdefixArray4D &Vein, IdefixArray4D &Vsout); - -#ifdef WITH_MPI - // Exchange surface EMFs to remove interprocess round off errors - void ExchangeAll(); - void ExchangeX1(); - void ExchangeX2(); - void ExchangeX3(); -#endif - - // Default constructor - ElectroMotiveForce(); - - // Destructor - ~ElectroMotiveForce(); - - private: - DataBlock *data; - Hydro *hydro; - -#ifdef WITH_MPI - enum {faceRight, faceLeft}; - - // Buffers for MPI calls - IdefixArray1D BufferSendX1[2]; - IdefixArray1D BufferSendX2[2]; - IdefixArray1D BufferSendX3[2]; - IdefixArray1D BufferRecvX1[2]; - IdefixArray1D BufferRecvX2[2]; - IdefixArray1D BufferRecvX3[2]; - - IdefixArray1D mapVars; - - int bufferSizeX1; - int bufferSizeX2; - int bufferSizeX3; - - // Requests for MPI persistent communications - MPI_Request sendRequestX1[2]; - MPI_Request sendRequestX2[2]; - MPI_Request sendRequestX3[2]; - MPI_Request recvRequestX1[2]; - MPI_Request recvRequestX2[2]; - MPI_Request recvRequestX3[2]; - - Kokkos::Timer timer; // Internal MPI timer -#endif -}; - -#endif // HYDRO_ELECTROMOTIVEFORCE_ELECTROMOTIVEFORCE_HPP_ diff --git a/src/hydro/fluxHD.hpp b/src/hydro/fluxHD.hpp deleted file mode 100644 index 9e49bb7e..00000000 --- a/src/hydro/fluxHD.hpp +++ /dev/null @@ -1,45 +0,0 @@ -// *********************************************************************************** -// Idefix MHD astrophysical code -// Copyright(C) Geoffroy R. J. Lesur -// and other code contributors -// Licensed under CeCILL 2.1 License, see COPYING for more information -// *********************************************************************************** - -#ifndef HYDRO_FLUXHD_HPP_ -#define HYDRO_FLUXHD_HPP_ -#include "idefix.hpp" -#include "hydro.hpp" - - -// Local Kokkos Inlined functions - -/******************************************************************************************** - * @fn void K_Flux(real F[], real V[], real U[], real Cs2Iso, - * const int Xn, const int Xt, const int Xb, - * const int BXn, const int BXt, const int BXb) - * @param F[] Array of flux variables (output) - * @param V[] Array of primitive variabless (input) - * @param U[] Array of conservative variables (input) - * @param cs2Iso Isothermal sound speed (only used when ISOTHERMAL is defined) - * @param Xn Index of the normal velocity component - * - * This routine computes the MHD out of V and U variables and stores it in F - ********************************************************************************************/ -KOKKOS_INLINE_FUNCTION void K_Flux(real *KOKKOS_RESTRICT F, const real *KOKKOS_RESTRICT V, - const real *KOKKOS_RESTRICT U, real Cs2Iso, const int Xn) { - F[RHO] = U[Xn]; - - EXPAND( F[MX1] = U[MX1]*V[Xn]; , - F[MX2] = U[MX2]*V[Xn]; , - F[MX3] = U[MX3]*V[Xn]; ) - -#if HAVE_ENERGY - F[ENG] = (U[ENG] + V[PRS])*V[Xn]; - F[Xn] += V[PRS]; -#else - // Add back pressure in the flux - F[Xn] += Cs2Iso * V[RHO]; -#endif -} - -#endif //HYDRO_FLUXHD_HPP_ diff --git a/src/hydro/fluxMHD.hpp b/src/hydro/fluxMHD.hpp deleted file mode 100644 index cc6422eb..00000000 --- a/src/hydro/fluxMHD.hpp +++ /dev/null @@ -1,62 +0,0 @@ -// *********************************************************************************** -// Idefix MHD astrophysical code -// Copyright(C) Geoffroy R. J. Lesur -// and other code contributors -// Licensed under CeCILL 2.1 License, see COPYING for more information -// *********************************************************************************** - -#ifndef HYDRO_FLUXMHD_HPP_ -#define HYDRO_FLUXMHD_HPP_ - -#include "idefix.hpp" -#include "hydro.hpp" - -/******************************************************************************************** - * @fn void K_Flux(real F[], real V[], real U[], real Cs2Iso, - * const int Xn, const int Xt, const int Xb, - * const int BXn, const int BXt, const int BXb) - * @param F[] Array of flux variables (output) - * @param V[] Array of primitive variabless (input) - * @param U[] Array of conservative variables (input) - * @param cs2Iso Isothermal sound speed (only used when ISOTHERMAL is defined) - * @param Xn Index of the normal velocity component - * @param Xt Index of the tangential velocity component - * @param Xb Indefx of the second tangential velocity component - * - * This routine computes the MHD out of V and U variables and stores it in F - ********************************************************************************************/ -KOKKOS_INLINE_FUNCTION void K_Flux(real F[], real V[], real U[], real Cs2Iso, - ARG_EXPAND(const int Xn, const int Xt, const int Xb), - ARG_EXPAND(const int BXn, const int BXt, const int BXb)) { - F[RHO] = U[Xn]; - EXPAND( F[MX1] = U[MX1]*V[Xn] - V[BXn]*V[BX1]; , - F[MX2] = U[MX2]*V[Xn] - V[BXn]*V[BX2]; , - F[MX3] = U[MX3]*V[Xn] - V[BXn]*V[BX3];) - - EXPAND(F[BXn] = ZERO_F; , - F[BXt] = V[Xn]*V[BXt] - V[BXn]*V[Xt]; , - F[BXb] = V[Xn]*V[BXb] - V[BXn]*V[Xb]; ) - - real Bmag2 = EXPAND(V[BX1]*V[BX1] , + V[BX2]*V[BX2], + V[BX3]*V[BX3]); - -#if HAVE_ENERGY - real ptot = V[PRS] + HALF_F*Bmag2; - -#elif defined(ISOTHERMAL) - real ptot = Cs2Iso * V[RHO] + HALF_F*Bmag2; - -#else - #error "K_Flux not defined for this EOS!" -#endif - -#if HAVE_ENERGY - F[ENG] = (U[ENG] + ptot)*V[Xn] - V[BXn] * (EXPAND( V[VX1]*V[BX1] , - + V[VX2]*V[BX2] , - + V[VX3]*V[BX3] )); -#endif - - // Add back pressure in the flux (not included in original PLUTO implementation) - F[Xn] += ptot; -} - -#endif // HYDRO_FLUXMHD_HPP_ diff --git a/src/hydro/hydro.cpp b/src/hydro/hydro.cpp deleted file mode 100644 index de737199..00000000 --- a/src/hydro/hydro.cpp +++ /dev/null @@ -1,685 +0,0 @@ -// *********************************************************************************** -// Idefix MHD astrophysical code -// Copyright(C) Geoffroy R. J. Lesur -// and other code contributors -// Licensed under CeCILL 2.1 License, see COPYING for more information -// *********************************************************************************** - -#include - -#include "idefix.hpp" -#include "hydro.hpp" -#include "dataBlock.hpp" - - -void Hydro::Init(Input &input, Grid &grid, DataBlock *datain) { - idfx::pushRegion("Hydro::Init"); - // Save the datablock to which we are attached from now on - this->data = datain; - - #if ORDER < 1 || ORDER > 4 - IDEFIX_ERROR("Reconstruction at chosen order is not implemented. Check your definitions file"); - #endif - - - #if HAVE_ENERGY - this->gamma = input.GetOrSet("Hydro","gamma",0, 5.0/3.0); - #endif - - #ifdef ISOTHERMAL - std::string isoString = input.Get("Hydro","csiso",0); - if(isoString.compare("constant") == 0) { - this->haveIsoSoundSpeed = Constant; - this->isoSoundSpeed = input.Get("Hydro","csiso",1); - } else if(isoString.compare("userdef") == 0) { - this->haveIsoSoundSpeed = UserDefFunction; - } else { - IDEFIX_ERROR("csiso admits only constant or userdef entries"); - } - #else - // set the isothermal soundspeed, even though it will not be used - this->isoSoundSpeed = -1.0; - #endif - - // read Solver from input file - std::string solverString = input.Get("Hydro","solver",0); - - if (solverString.compare("tvdlf") == 0) { - mySolver = TVDLF; - } else if (solverString.compare("hll") == 0) { - mySolver = HLL; -#if MHD == YES - } else if (solverString.compare("hlld") == 0) { - mySolver = HLLD; -#else - } else if (solverString.compare("hllc") == 0) { - mySolver = HLLC; -#endif - } else if (solverString.compare("roe") == 0) { - mySolver = ROE; - } else { - std::stringstream msg; -#if MHD == YES - msg << "Unknown MHD solver type " << solverString; -#else - msg << "Unknown HD solver type " << solverString; -#endif - IDEFIX_ERROR(msg); - } - - // Shock flattening - this->haveShockFlattening = input.CheckEntry("Hydro","shockFlattening")>=0; - - // Source terms (always activated when non-cartesian geometry because of curvature source terms) -#if GEOMETRY == CARTESIAN - this->haveSourceTerms = false; -#else - this->haveSourceTerms = true; -#endif - - // Check whether we have rotation - int rotation = input.CheckEntry("Hydro","rotation"); - - if(rotation>=0 ) { - this->haveSourceTerms = true; - this->haveRotation = true; - if(rotation >1 ) IDEFIX_ERROR("Rotation needs a a single rotation velocity (Omega_Z) \ - in idefix.ini"); - this->OmegaZ = input.Get("Hydro","rotation",0); - } - - // Check whether we have shearing box - int shearingbox = input.CheckEntry("Hydro","shearingBox"); - - if(shearingbox>=0 ) { - this->haveShearingBox = true; - this->haveSourceTerms = true; - if(shearingbox != 1) { - IDEFIX_ERROR("Shearing box needs a scalar value for the shear rate in idefix.ini"); - } - - this->sbS = input.Get("Hydro","shearingBox",0); - // Get box size - this->sbLx = grid.xend[IDIR] - grid.xbeg[IDIR]; - } - - // Gravitational potential (deprecated, use [Gravity] block in your input file) - if(input.CheckEntry("Hydro","gravPotential")>=0) { - IDEFIX_DEPRECATED("gravPotential in [Hydro] block is deprecated in the input file. " - "Use a [Gravity] block instead."); - std::string potentialString = input.Get("Hydro","gravPotential",0); - if(potentialString.compare("userdef") == 0) { - data->gravity.haveUserDefPotential = true; - data->gravity.havePotential = true; - data->gravity.Init(input,data); - } else { - IDEFIX_ERROR("Unknown type of gravitational potential in idefix.ini. " - "Only userdef is implemented"); - } - } - - // Body Force (deprecated, use [Gravity] block in your input file) - if(input.CheckEntry("Hydro","bodyForce")>=0) { - IDEFIX_DEPRECATED("bodyForce in [Hydro] block is deprecated in the input file. " - "Use a [Gravity] block instead."); - std::string potentialString = input.Get("Hydro","bodyForce",0); - if(potentialString.compare("userdef") == 0) { - data->gravity.haveBodyForce = true; - data->gravity.Init(input,data); - } else { - IDEFIX_ERROR("Unknown type of body force in idefix.ini. " - "Only userdef is implemented"); - } - } - - // Do we use fargo? (deprecated, use a full fargo block in idefix.ini, - // instead of including it in hydo) - if(input.CheckEntry("Hydro","fargo")>=0) { - IDEFIX_DEPRECATED("Fargo in [Hydro] block is deprecated in the input file. " - "Use a [Fargo] block instead."); - data->haveFargo = true; - data->fargo.Init(input, this->data); - } - - /////////////////////// - // Parabolic terms - /////////////////////// - - - // Check whether viscosity is enabled, if so, init a viscosity object - if(input.CheckEntry("Hydro","viscosity")>=0) { - std::string opType = input.Get("Hydro","viscosity",0); - if(opType.compare("explicit") == 0 ) { - haveExplicitParabolicTerms = true; - viscosityStatus.isExplicit = true; - } else if(opType.compare("rkl") == 0 ) { - haveRKLParabolicTerms = true; - viscosityStatus.isRKL = true; - } else { - std::stringstream msg; - msg << "Unknown integration type for viscosity: " << opType; - IDEFIX_ERROR(msg); - } - this->viscosity.Init(input, grid, this); - } - - // Check whether thermal diffusion is enabled, if so, init a thermal diffusion object - if(input.CheckEntry("Hydro","TDiffusion")>=0) { - std::string opType = input.Get("Hydro","TDiffusion",0); - if(opType.compare("explicit") == 0 ) { - haveExplicitParabolicTerms = true; - thermalDiffusionStatus.isExplicit = true; - } else if(opType.compare("rkl") == 0 ) { - haveRKLParabolicTerms = true; - thermalDiffusionStatus.isRKL = true; - } else { - std::stringstream msg; - msg << "Unknown integration type for thermal diffusion: " << opType; - IDEFIX_ERROR(msg); - } - this->thermalDiffusion.Init(input, grid, this); - } - -#if MHD == YES - if(input.CheckEntry("Hydro","resistivity")>=0 || - input.CheckEntry("Hydro","ambipolar")>=0 || - input.CheckEntry("Hydro","hall")>=0 ) { - // - this->haveCurrent = true; - - if(input.CheckEntry("Hydro","resistivity")>=0) { - std::string opType = input.Get("Hydro","resistivity",0); - if(opType.compare("explicit") == 0 ) { - haveExplicitParabolicTerms = true; - resistivityStatus.isExplicit = true; - needExplicitCurrent = true; - } else if(opType.compare("rkl") == 0 ) { - haveRKLParabolicTerms = true; - resistivityStatus.isRKL = true; - needRKLCurrent = true; - } else { - std::stringstream msg; - msg << "Unknown integration type for resistivity: " << opType; - IDEFIX_ERROR(msg); - } - if(input.Get("Hydro","resistivity",1).compare("constant") == 0) { - this->etaO = input.Get("Hydro","resistivity",2); - resistivityStatus.status = Constant; - } else if(input.Get("Hydro","resistivity",1).compare("userdef") == 0) { - resistivityStatus.status = UserDefFunction; - } else { - IDEFIX_ERROR("Unknown resistivity definition in idefix.ini. " - "Can only be constant or userdef."); - } - } - - if(input.CheckEntry("Hydro","ambipolar")>=0) { - std::string opType = input.Get("Hydro","ambipolar",0); - if(opType.compare("explicit") == 0 ) { - haveExplicitParabolicTerms = true; - ambipolarStatus.isExplicit = true; - needExplicitCurrent = true; - } else if(opType.compare("rkl") == 0 ) { - haveRKLParabolicTerms = true; - ambipolarStatus.isRKL = true; - needRKLCurrent = true; - } else { - std::stringstream msg; - msg << "Unknown integration type for ambipolar: " << opType; - IDEFIX_ERROR(msg); - } - if(input.Get("Hydro","ambipolar",1).compare("constant") == 0) { - this->xA = input.Get("Hydro","ambipolar",2); - ambipolarStatus.status = Constant; - } else if(input.Get("Hydro","ambipolar",1).compare("userdef") == 0) { - ambipolarStatus.status = UserDefFunction; - } else { - IDEFIX_ERROR("Unknown ambipolar definition in idefix.ini. " - "Can only be constant or userdef."); - } - } - - if(input.CheckEntry("Hydro","hall")>=0) { - // Check consistency - if(mySolver != HLL ) - IDEFIX_ERROR("Hall effect is only compatible with HLL Riemann solver."); - std::string opType = input.Get("Hydro","hall",0); - if(opType.compare("explicit") == 0 ) { - hallStatus.isExplicit = true; - needExplicitCurrent = true; - } else if(opType.compare("rkl") == 0 ) { - IDEFIX_ERROR("RKL inegration is incompatible with Hall"); - } else { - std::stringstream msg; - msg << "Unknown integration type for hall: " << opType; - IDEFIX_ERROR(msg); - } - if(input.Get("Hydro","hall",1).compare("constant") == 0) { - this->xH = input.Get("Hydro","hall",2); - hallStatus.status = Constant; - } else if(input.Get("Hydro","hall",1).compare("userdef") == 0) { - hallStatus.status = UserDefFunction; - } else { - IDEFIX_ERROR("Unknown Hall definition in idefix.ini. Can only be constant or userdef."); - } - } - } - #endif // MHD - - // Do we have to take care of the axis? - if(data->haveAxis) { - this->myAxis.Init(grid, this); - this->haveAxis = true; - } - ///////////////////////////////////////// - // ALLOCATION SECION /////////////////// - ///////////////////////////////////////// - - // We now allocate the fields required by the hydro solver - Vc = IdefixArray4D("Hydro_Vc", NVAR, - data->np_tot[KDIR], data->np_tot[JDIR], data->np_tot[IDIR]); - Uc = IdefixArray4D("Hydro_Uc", NVAR, - data->np_tot[KDIR], data->np_tot[JDIR], data->np_tot[IDIR]); - - data->states["current"].PushArray(Uc, State::center, "Hydro_Uc"); - - InvDt = IdefixArray3D("Hydro_InvDt", - data->np_tot[KDIR], data->np_tot[JDIR], data->np_tot[IDIR]); - cMax = IdefixArray3D("Hydro_cMax", - data->np_tot[KDIR], data->np_tot[JDIR], data->np_tot[IDIR]); - dMax = IdefixArray3D("Hydro_dMax", - data->np_tot[KDIR], data->np_tot[JDIR], data->np_tot[IDIR]); - FluxRiemann = IdefixArray4D("Hydro_FluxRiemann", NVAR, - data->np_tot[KDIR], data->np_tot[JDIR], data->np_tot[IDIR]); - - #if MHD == YES - Vs = IdefixArray4D("Hydro_Vs", DIMENSIONS, - data->np_tot[KDIR]+KOFFSET, data->np_tot[JDIR]+JOFFSET, data->np_tot[IDIR]+IOFFSET); - #ifdef EVOLVE_VECTOR_POTENTIAL - #if DIMENSIONS == 1 - IDEFIX_ERROR("EVOLVE_VECTOR_POTENTIAL is not compatible with 1D MHD"); - #else - Ve = IdefixArray4D("Hydro_Ve", AX3e+1, - data->np_tot[KDIR]+KOFFSET, data->np_tot[JDIR]+JOFFSET, data->np_tot[IDIR]+IOFFSET); - - data->states["current"].PushArray(Ve, State::center, "Hydro_Ve"); - #endif - #else // EVOLVE_VECTOR_POTENTIAL - data->states["current"].PushArray(Vs, State::center, "Hydro_Vs"); - #endif // EVOLVE_VECTOR_POTENTIAL - this->emf.Init(input, this); - #endif - - // Allocate sound speed array if needed - if(this->haveIsoSoundSpeed == UserDefFunction) { - this->isoSoundSpeedArray = IdefixArray3D("Hydro_csIso", - data->np_tot[KDIR], data->np_tot[JDIR], data->np_tot[IDIR]); - } - - if(this->haveCurrent) { - // Allocate current (when hydro needs it) - J = IdefixArray4D("Hydro_J", 3, - data->np_tot[KDIR], data->np_tot[JDIR], data->np_tot[IDIR]); - } - - // Allocate nonideal MHD effects array when a user-defined function is used - if(this->resistivityStatus.status == UserDefFunction) - etaOhmic = IdefixArray3D("Hydro_etaOhmic", - data->np_tot[KDIR], data->np_tot[JDIR], data->np_tot[IDIR]); - if(this->ambipolarStatus.status == UserDefFunction) - xAmbipolar = IdefixArray3D("Hydro_xAmbipolar", - data->np_tot[KDIR], data->np_tot[JDIR], data->np_tot[IDIR]); - if(this->hallStatus.status == UserDefFunction) - xHall = IdefixArray3D("Hydro_xHall", - data->np_tot[KDIR], data->np_tot[JDIR], data->np_tot[IDIR]); - - // Fill the names of the fields - for(int i = 0 ; i < NVAR ; i++) { - switch(i) { - case RHO: - VcName.push_back("RHO"); - break; - case VX1: - VcName.push_back("VX1"); - break; - case VX2: - VcName.push_back("VX2"); - break; - case VX3: - VcName.push_back("VX3"); - break; - case BX1: - VcName.push_back("BX1"); - break; - case BX2: - VcName.push_back("BX2"); - break; - case BX3: - VcName.push_back("BX3"); - break; -#if HAVE_ENERGY - case PRS: - VcName.push_back("PRS"); - break; -#endif - default: - VcName.push_back("Vc_"+std::to_string(i)); - } - } - - for(int i = 0 ; i < DIMENSIONS ; i++) { - switch(i) { - case 0: - VsName.push_back("BX1s"); - break; - case 1: - VsName.push_back("BX2s"); - break; - case 2: - VsName.push_back("BX3s"); - break; - default: - VsName.push_back("Vs_"+std::to_string(i)); - } - } - - #ifdef EVOLVE_VECTOR_POTENTIAL - #if DIMENSIONS < 3 - VeName.push_back("AX3e"); - #else - for(int i = 0 ; i < DIMENSIONS ; i++) { - switch(i) { - case 0: - VeName.push_back("AX1e"); - break; - case 1: - VeName.push_back("AX2e"); - break; - case 2: - VeName.push_back("AX3e"); - break; - default: - VeName.push_back("Ve_"+std::to_string(i)); - } - } - #endif - #endif - - // Initialise boundary conditions - boundary.Init(input, grid, this); - - // Init shock flattening - if(haveShockFlattening) { - this->shockFlattening = ShockFlattening(this,input.Get("Hydro","shockFlattening",0)); - } - - idfx::popRegion(); -} - -void Hydro::EnrollIsoSoundSpeed(IsoSoundSpeedFunc myFunc) { - if(this->haveIsoSoundSpeed != UserDefFunction) { - IDEFIX_WARNING("Isothermal sound speed enrollment requires Hydro/csiso " - " to be set to userdef in .ini file"); - } - #if HAVE_ENERGY - IDEFIX_ERROR("Isothermal sound speed enrollment requires ISOTHERMAL to be defined in" - "definitions.hpp"); - #endif - this->isoSoundSpeedFunc = myFunc; -} - -void Hydro::EnrollUserDefBoundary(UserDefBoundaryFunc myFunc) { - // This is a proxy for userdef enrollment - boundary.EnrollUserDefBoundary(myFunc); -} - -void Hydro::EnrollFluxBoundary(UserDefBoundaryFunc myFunc) { - // This is a proxy for userdef enrollment - boundary.EnrollFluxBoundary(myFunc); -} - - -void Hydro::EnrollInternalBoundary(InternalBoundaryFunc myFunc) { - // This is a proxy for userdef enrollment - boundary.EnrollInternalBoundary(myFunc); -} - -void Hydro::EnrollEmfBoundary(EmfBoundaryFunc myFunc) { - #if MHD == NO - IDEFIX_ERROR("This function can only be used with the MHD solver."); - #endif - this->emfBoundaryFunc = myFunc; - this->haveEmfBoundary = true; -} - -// Deprecated function -void Hydro::EnrollGravPotential(GravPotentialFunc myFunc) { - IDEFIX_DEPRECATED("Calling Hydro::EnrollGravPotential is deprecated." - "Use Gravity::EnrollPotential"); - data->gravity.EnrollPotential(myFunc); -} - -// Deprecated function -void Hydro::EnrollBodyForce(BodyForceFunc myFunc) { - IDEFIX_DEPRECATED("Calling Hydro::EnrollBodyForce is deprecated." - "Use Gravity::EnrollBodyForce"); - data->gravity.EnrollBodyForce(myFunc); -} - -void Hydro::EnrollUserSourceTerm(SrcTermFunc myFunc) { - this->userSourceTerm = myFunc; - this->haveUserSourceTerm = true; - this->haveSourceTerms = true; -} - -void Hydro::EnrollOhmicDiffusivity(DiffusivityFunc myFunc) { - #if MHD == NO - IDEFIX_ERROR("This function can only be used with the MHD solver."); - #endif - if(this->resistivityStatus.status < UserDefFunction) { - IDEFIX_WARNING("Ohmic diffusivity enrollment requires Hydro/Resistivity " - "to be set to userdef in .ini file"); - } - this->ohmicDiffusivityFunc = myFunc; -} - -void Hydro::EnrollAmbipolarDiffusivity(DiffusivityFunc myFunc) { - #if MHD == NO - IDEFIX_ERROR("This function can only be used with the MHD solver."); - #endif - if(this->ambipolarStatus.status < UserDefFunction) { - IDEFIX_WARNING("Ambipolar diffusivity enrollment requires Hydro/Ambipolar " - "to be set to userdef in .ini file"); - } - this->ambipolarDiffusivityFunc = myFunc; -} - -void Hydro::EnrollHallDiffusivity(DiffusivityFunc myFunc) { - #if MHD == NO - IDEFIX_ERROR("This function can only be used with the MHD solver."); - #endif - if(this->hallStatus.status < UserDefFunction) { - IDEFIX_WARNING("Hall diffusivity enrollment requires Hydro/Hall " - "to be set to userdef in .ini file"); - } - this->hallDiffusivityFunc = myFunc; -} - -Hydro::Hydro() { - // Do nothing ! -} - -real Hydro::GetGamma() { - return(this->gamma); -} - - -void Hydro::ResetStage() { - // Reset variables required at the beginning of each stage - // (essentially linked to timestep evaluation) - idfx::pushRegion("Hydro::ResetStage"); - - IdefixArray3D InvDt=this->InvDt; - - idefix_for("HydroResetStage",0,data->np_tot[KDIR],0,data->np_tot[JDIR],0,data->np_tot[IDIR], - KOKKOS_LAMBDA (int k, int j, int i) { - InvDt(k,j,i) = ZERO_F; - }); - - idfx::popRegion(); -} - -void Hydro::ShowConfig() { - idfx::cout << "Hydro: "; - #if MHD == YES - idfx::cout << "solving MHD equations." << std::endl; - #ifdef EVOLVE_VECTOR_POTENTIAL - idfx::cout << "Hydro: Using EXPERIMENTAL vector potential formulation for MHD." << std::endl; - #endif - #else - idfx::cout << "solving HD equations." << std::endl; - #endif - idfx::cout << "Hydro: Reconstruction: "; - #if ORDER == 1 - idfx::cout << "1st order (donor cell)" << std::endl; - #elif ORDER == 2 - idfx::cout << "2nd order (PLM Van Leer)" << std::endl; - #elif ORDER == 3 - idfx::cout << "3rd order (LimO3)" << std::endl; - #elif ORDER == 4 - idfx::cout << "4th order (PPM)" << std::endl; - #endif - - #if HAVE_ENERGY - idfx::cout << "Hydro: EOS: ideal with gamma=" << this->gamma << std::endl; - #endif - #ifdef ISOTHERMAL - if(haveIsoSoundSpeed == Constant) { - idfx::cout << "Hydro: EOS: isothermal with cs=" << isoSoundSpeed << "." << std::endl; - } else if(haveIsoSoundSpeed == UserDefFunction) { - idfx::cout << "Hydro: EOS: isothermal with user-defined cs function." << std::endl; - if(!isoSoundSpeedFunc) { - IDEFIX_ERROR("No user-defined isothermal sound speed function has been enrolled."); - } - } - #endif// ISOTHERMAL - idfx::cout << "Hydro: Riemann solver: "; - switch(mySolver) { - case TVDLF: - idfx::cout << "tvdlf." << std::endl; - break; - case HLL: - idfx::cout << "hll." << std::endl; - break; - #if MHD==YES - case HLLD: - idfx::cout << "hlld." << std::endl; - break; - #else - case HLLC: - idfx::cout << "hllc." << std::endl; - break; - #endif - case ROE: - idfx::cout << "roe." << std::endl; - break; - default: - IDEFIX_ERROR("Unknown Riemann solver"); - } - if(haveRotation) { - idfx::cout << "Hydro: Rotation ENABLED with Omega=" << this->OmegaZ << std::endl; - } - if(haveShearingBox) { - idfx::cout << "Hydro: ShearingBox ENABLED with shear rate= " << this->sbS - << " and Lx= " << sbLx << std::endl; - } - if(haveShockFlattening) { - idfx::cout << "Hydro: Shock Flattening ENABLED." << std::endl; - } - - - if(resistivityStatus.status != Disabled) { - if(resistivityStatus.status == Constant) { - idfx::cout << "Hydro: Ohmic resistivity ENABLED with constant resistivity eta=" - << etaO << std::endl; - } else if(resistivityStatus.status == UserDefFunction) { - idfx::cout << "Hydro: Ohmic resistivity ENABLED with user-defined resistivity function." - << std::endl; - if(!ohmicDiffusivityFunc) { - IDEFIX_ERROR("No user-defined Ihmic resistivity function has been enrolled."); - } - } else { - IDEFIX_ERROR("Unknown Ohmic resistivity mode"); - } - if(resistivityStatus.isExplicit) { - idfx::cout << "Hydro: Ohmic resistivity uses an explicit time integration." << std::endl; - } else if(resistivityStatus.isRKL) { - idfx::cout << "Hydro: Ohmic resistivity uses a Runge-Kutta-Legendre time integration." - << std::endl; - } else { - IDEFIX_ERROR("Unknown time integrator for Ohmic resistivity"); - } - } - - if(ambipolarStatus.status != Disabled) { - if(ambipolarStatus.status == Constant) { - idfx::cout << "Hydro: Ambipolar diffusion ENABLED with constant diffusivity xA=" - << xA << std::endl; - } else if(ambipolarStatus.status == UserDefFunction) { - idfx::cout << "Hydro: Ambipolar diffusion ENABLED with user-defined diffusivity function." - << std::endl; - if(!ambipolarDiffusivityFunc) { - IDEFIX_ERROR("No user-defined ambipolar diffusion function has been enrolled."); - } - } else { - IDEFIX_ERROR("Unknown Ambipolar diffusion mode"); - } - if(ambipolarStatus.isExplicit) { - idfx::cout << "Hydro: Ambipolar diffusion uses an explicit time integration." << std::endl; - } else if(ambipolarStatus.isRKL) { - idfx::cout << "Hydro: Ambipolar diffusion uses a Runge-Kutta-Legendre time integration." - << std::endl; - } else { - IDEFIX_ERROR("Unknown time integrator for ambipolar diffusion"); - } - } - - if(hallStatus.status != Disabled) { - if(hallStatus.status == Constant) { - idfx::cout << "Hydro: Hall effect ENABLED with constant diffusivity xH=" - << xH << std::endl; - } else if(hallStatus.status == UserDefFunction) { - idfx::cout << "Hydro: Hall effect ENABLED with user-defined diffusivity function." - << std::endl; - if(!hallDiffusivityFunc) { - IDEFIX_ERROR("No user-defined Hall diffusivity function has been enrolled."); - } - } else { - IDEFIX_ERROR("Unknown Hall effect mode"); - } - if(hallStatus.isExplicit) { - idfx::cout << "Hydro: Hall effect uses an explicit time integration." << std::endl; - } else { - IDEFIX_ERROR("Unknown time integrator for Hall effect"); - } - } - - if(emfBoundaryFunc) { - idfx::cout << "Hydro: user-defined EMF boundaries ENABLED." << std::endl; - } - if(userSourceTerm) { - idfx::cout << "Hydro: user-defined source terms ENABLED." << std::endl; - } - #if MHD == YES - emf.ShowConfig(); - #endif - if(viscosityStatus.isExplicit || viscosityStatus.isRKL) { - viscosity.ShowConfig(); - } - if(thermalDiffusionStatus.isExplicit || thermalDiffusionStatus.isRKL) { - thermalDiffusion.ShowConfig(); - } - if(haveAxis) { - myAxis.ShowConfig(); - } -} diff --git a/src/hydro/hydro.hpp b/src/hydro/hydro.hpp deleted file mode 100644 index 13cbb868..00000000 --- a/src/hydro/hydro.hpp +++ /dev/null @@ -1,211 +0,0 @@ -// *********************************************************************************** -// Idefix MHD astrophysical code -// Copyright(C) Geoffroy R. J. Lesur -// and other code contributors -// Licensed under CeCILL 2.1 License, see COPYING for more information -// *********************************************************************************** - -#ifndef HYDRO_HYDRO_HPP_ -#define HYDRO_HYDRO_HPP_ - -#include -#include -#include "idefix.hpp" - -#include "grid.hpp" -#include "hydro_defs.hpp" -#include "electroMotiveForce.hpp" -#include "viscosity.hpp" -#include "thermalDiffusion.hpp" -#include "axis.hpp" -#include "hydroboundary.hpp" -#include "shockFlattening.hpp" - -// forward class declaration -class DataBlock; - - -class Hydro { - public: - Hydro(); - void Init(Input &, Grid &, DataBlock *); - void ConvertConsToPrim(); - void ConvertPrimToCons(); - template void CalcRiemannFlux(const real); - template void CalcParabolicFlux(const real); - template void AddNonIdealMHDFlux(const real); - template void CalcRightHandSide(real, real ); - void CalcCurrent(); - void AddSourceTerms(real, real ); - void CoarsenFlow(IdefixArray4D&); - void CoarsenMagField(IdefixArray4D&); - void CoarsenVectorPotential(); - real GetGamma(); - real CheckDivB(); - void ResetStage(); - void ShowConfig(); - - // Our boundary conditions - HydroBoundary boundary; - - // Source terms - bool haveSourceTerms{false}; - - // Parabolic terms - bool haveExplicitParabolicTerms{false}; - bool haveRKLParabolicTerms{false}; - - // Current - bool haveCurrent{false}; - bool needExplicitCurrent{false}; - bool needRKLCurrent{false}; - - // Nonideal MHD effects coefficients - ParabolicModuleStatus resistivityStatus, ambipolarStatus, hallStatus; - - // Whether or not we have viscosity - ParabolicModuleStatus viscosityStatus; - - // Whether or not we have thermal diffusion - ParabolicModuleStatus thermalDiffusionStatus; - - // Viscosity object - Viscosity viscosity; - - // Thermal Diffusion object - ThermalDiffusion thermalDiffusion; - - // Whether or not we have to treat the axis - bool haveAxis{false}; - Axis myAxis; - - // Rotation vector - bool haveRotation{false}; - real OmegaZ; - - bool haveShearingBox{false}; - // Shear rate for shearing box problems - real sbS; - // Box width for shearing box problems - real sbLx; - - // ShockFlattening - bool haveShockFlattening{false}; - ShockFlattening shockFlattening; - - // Enroll user-defined boundary conditions - void EnrollUserDefBoundary(UserDefBoundaryFunc); - void EnrollInternalBoundary(InternalBoundaryFunc); - void EnrollEmfBoundary(EmfBoundaryFunc); - void EnrollFluxBoundary(UserDefBoundaryFunc); - - // Add some user source terms - void EnrollUserSourceTerm(SrcTermFunc); - - // DEPRECATED gravity enrollment - void EnrollGravPotential(GravPotentialFunc); - void EnrollBodyForce(BodyForceFunc); - - // Enroll user-defined ohmic, ambipolar and Hall diffusivities - void EnrollOhmicDiffusivity(DiffusivityFunc); - void EnrollAmbipolarDiffusivity(DiffusivityFunc); - void EnrollHallDiffusivity(DiffusivityFunc); - - // Enroll user-defined isothermal sound speed - void EnrollIsoSoundSpeed(IsoSoundSpeedFunc); - - // Riemann Solvers -#if MHD == YES - template - void HlldMHD(); - template - void HllMHD(); - template - void RoeMHD(); - template - void TvdlfMHD(); -#else - template - void HllcHD(); - template - void HllHD(); - template - void RoeHD(); - template - void TvdlfHD(); -#endif - - // Arrays required by the Hydro object - IdefixArray4D Vc; // Main cell-centered primitive variables index - IdefixArray4D Vs; // Main face-centered varariables - IdefixArray4D Ve; // Main edge-centered varariables (only when EVOLVE_VECTOR_POTENTIAL) - IdefixArray4D Uc; // Main cell-centered conservative variables - IdefixArray4D J; // Electrical current - // (only defined when non-ideal MHD effects are enabled) - - // Name of the fields (used in outputs) - std::vector VcName; - std::vector VsName; - std::vector VeName; - - // Storing all of the electromotive forces - ElectroMotiveForce emf; - - // Required by time integrator - IdefixArray4D Uc0; - IdefixArray4D Vs0; - IdefixArray4D Ve0; - IdefixArray3D InvDt; - - IdefixArray4D FluxRiemann; - IdefixArray3D dMax; // Maximum diffusion speed - - - - private: - friend class ElectroMotiveForce; - friend class Viscosity; - friend class ThermalDiffusion; - friend class Fargo; - friend class Axis; - friend class RKLegendre; - friend class HydroBoundary; - friend class ShockFlattening; - - // Isothermal EOS parameters - real isoSoundSpeed; - HydroModuleStatus haveIsoSoundSpeed{Disabled}; - IdefixArray3D isoSoundSpeedArray; - IsoSoundSpeedFunc isoSoundSpeedFunc{NULL}; - - // Adiabatic EOS parameters - real gamma; - - Solver mySolver; - - DataBlock *data; - - // Emf boundary conditions - bool haveEmfBoundary{false}; - EmfBoundaryFunc emfBoundaryFunc{NULL}; - - // User defined source term - SrcTermFunc userSourceTerm{NULL}; - bool haveUserSourceTerm{false}; - - real etaO, xH, xA; // Ohmic resistivity, Hall, ambipolar (when constant) - - // Ohmic, Hall and ambipolar diffusivity (when function-defined) - DiffusivityFunc ohmicDiffusivityFunc{NULL}; - DiffusivityFunc ambipolarDiffusivityFunc{NULL}; - DiffusivityFunc hallDiffusivityFunc{NULL}; - - IdefixArray3D cMax; // Maximum propagation speed - - // Nonideal effect diffusion coefficient (only allocated when needed) - IdefixArray3D etaOhmic; - IdefixArray3D xHall; - IdefixArray3D xAmbipolar; -}; - -#endif // HYDRO_HYDRO_HPP_ diff --git a/src/hydro/shockFlattening.cpp b/src/hydro/shockFlattening.cpp deleted file mode 100644 index 660e7945..00000000 --- a/src/hydro/shockFlattening.cpp +++ /dev/null @@ -1,127 +0,0 @@ -// *********************************************************************************** -// Idefix MHD astrophysical code -// Copyright(C) Geoffroy R. J. Lesur -// and other code contributors -// Licensed under CeCILL 2.1 License, see COPYING for more information -// *********************************************************************************** - -#include "dataBlock.hpp" -#include "hydro.hpp" -#include "shockFlattening.hpp" - -ShockFlattening::ShockFlattening(Hydro *h, real smoothing) { - hydro = h; - flagArray = IdefixArray3D("flagArray",h->data->np_tot[KDIR], - h->data->np_tot[JDIR], - h->data->np_tot[IDIR]); - this->isActive = true; - this->smoothing = smoothing; -} - -void ShockFlattening::FindShock() { - idfx::pushRegion("ShockFlattening::FindShock"); - auto flags = flagArray; - auto Vc = hydro->Vc; - real smoothing = this->smoothing; - - #if GEOMETRY == CARTESIAN - auto dx1 = hydro->data->dx[IDIR]; - auto dx2 = hydro->data->dx[JDIR]; - auto dx3 = hydro->data->dx[KDIR]; - #else - auto Ax1 = hydro->data->A[IDIR]; - auto Ax2 = hydro->data->A[JDIR]; - auto Ax3 = hydro->data->A[KDIR]; - auto dV = hydro->data->dV; - #endif - #ifdef ISOTHERMAL - auto cs = hydro->isoSoundSpeedArray; - HydroModuleStatus haveIsoCs = hydro->haveIsoSoundSpeed; - real csIso = hydro->isoSoundSpeed; - #endif - - auto beg = hydro->data->beg; - auto end = hydro->data->end; - idefix_for("findshocks", - beg[KDIR]-KOFFSET, end[KDIR]+KOFFSET, - beg[JDIR]-JOFFSET, end[JDIR]+JOFFSET, - beg[IDIR]-IOFFSET, end[IDIR]+IOFFSET, - KOKKOS_LAMBDA(int k, int j, int i) { - flags(k,j,i) = FlagShock::None; - #if GEOMETRY == CARTESIAN - real divV = D_EXPAND( (Vc(VX1,k,j,i+1) - Vc(VX1,k,j,i-1))/(2*dx1(i)) , - + (Vc(VX2,k,j+1,i) - Vc(VX2,k,j-1,i))/(2*dx2(j)) , - + (Vc(VX3,k+1,j,i) - Vc(VX3,k-1,j,i))/(2*dx3(k)) ); - #else - real divV = D_EXPAND( (Vc(VX1,k,j,i+1) + Vc(VX1,k,j,i))*Ax1(k,j,i+1) - - (Vc(VX1,k,j,i-1) + Vc(VX1,k,j,i))*Ax1(k,j,i) , - + (Vc(VX2,k,j+1,i) + Vc(VX2,k,j,i))*Ax2(k,j+1,i) - - (Vc(VX2,k,j-1,i) + Vc(VX2,k,j,i))*Ax2(k,j,i) , - + (Vc(VX3,k+1,j,i) + Vc(VX3,k,j,i))*Ax3(k+1,j,i) - - (Vc(VX3,k-1,j,i) + Vc(VX3,k,j,i))*Ax3(k,j,i) ); - divV = 0.5*divV/dV(k,j,i); - #endif - - if(divV= 2 - pmin = FMIN(pmin,Vc(RHO,k,j+1,i)*cs(k,j+1,i)*cs(k,j+1,i)); - pmin = FMIN(pmin,Vc(RHO,k,j-1,i)*cs(k,j-1,i)*cs(k,j-1,i)); - gradP += FABS(Vc(RHO,k,j+1,i)*cs(k,j+1,i)*cs(k,j+1,i) - - Vc(RHO,k,j-1,i)*cs(k,j-1,i)*cs(k,j-1,i)); - #endif - #if DIMENSIONS == 3 - pmin = FMIN(pmin,Vc(RHO,k+1,j,i)*cs(k+1,j,i)*cs(k+1,j,i)); - pmin = FMIN(pmin,Vc(RHO,k-1,j,i)*cs(k-1,j,i)*cs(k-1,j,i)); - gradP += FABS(Vc(RHO,k+1,j,i)*cs(k+1,j,i)*cs(k+1,j,i) - - Vc(RHO,k-1,j,i)*cs(k-1,j,i)*cs(k-1,j,i)); - #endif - } else { - pmin = Vc(RHO,k,j,i)*csIso*csIso; - pmin = FMIN(pmin,Vc(RHO,k,j,i+1)*csIso*csIso); - pmin = FMIN(pmin,Vc(RHO,k,j,i-1)*csIso*csIso); - gradP = FABS(Vc(RHO,k,j,i+1)*csIso*csIso - - Vc(RHO,k,j,i-1)*csIso*csIso); - #if DIMENSIONS >= 2 - pmin = FMIN(pmin,Vc(RHO,k,j+1,i)*csIso*csIso); - pmin = FMIN(pmin,Vc(RHO,k,j-1,i)*csIso*csIso); - gradP += FABS(Vc(RHO,k,j+1,i)*csIso*csIso - - Vc(RHO,k,j-1,i)*csIso*csIso); - #endif - #if DIMENSIONS == 3 - pmin = FMIN(pmin,Vc(RHO,k+1,j,i)*csIso*csIso); - pmin = FMIN(pmin,Vc(RHO,k-1,j,i)*csIso*csIso); - gradP += FABS(Vc(RHO,k+1,j,i)*csIso*csIso - - Vc(RHO,k-1,j,i)*csIso*csIso); - #endif - } - #else // ISOTHERMAL - real pmin = Vc(PRS,k,j,i); - pmin = FMIN(pmin,Vc(PRS,k,j,i+1)); - pmin = FMIN(pmin,Vc(PRS,k,j,i-1)); - real gradP = FABS(Vc(PRS,k,j,i+1) - Vc(PRS,k,j,i-1)); - #if DIMENSIONS >= 2 - pmin = FMIN(pmin,Vc(PRS,k,j+1,i)); - pmin = FMIN(pmin,Vc(PRS,k,j-1,i)); - gradP += FABS(Vc(PRS,k,j+1,i) - Vc(PRS,k,j-1,i)); - #endif - #if DIMENSIONS == 3 - pmin = FMIN(pmin,Vc(PRS,k+1,j,i)); - pmin = FMIN(pmin,Vc(PRS,k-1,j,i)); - gradP += FABS(Vc(PRS,k+1,j,i) - Vc(PRS,k-1,j,i)); - #endif - #endif // ISOTHERMAL - if(gradP > smoothing*pmin) { - flags(k,j,i) = FlagShock::Shock; - } - } - }); - idfx::popRegion(); -} diff --git a/src/hydro/slopeLimiter.hpp b/src/hydro/slopeLimiter.hpp deleted file mode 100644 index 91dfe9da..00000000 --- a/src/hydro/slopeLimiter.hpp +++ /dev/null @@ -1,337 +0,0 @@ -// *********************************************************************************** -// Idefix MHD astrophysical code -// Copyright(C) Geoffroy R. J. Lesur -// and other code contributors -// Licensed under CeCILL 2.1 License, see COPYING for more information -// *********************************************************************************** -#ifndef HYDRO_SLOPELIMITER_HPP_ -#define HYDRO_SLOPELIMITER_HPP_ - -#include "hydro.hpp" -#include "dataBlock.hpp" -#include "shockFlattening.hpp" - -// Build a left and right extrapolation of the primitive variables along direction dir - -// These functions extrapolate the cell prim vars to the faces. Definitions are as followed -// -// | cell i-1 interface i cell i -// |-----------------------------------|------------------------------------|| -// Vc(i-1) PrimL(i) PrimR(i) Vc(i) -template -class SlopeLimiter { - public: - SlopeLimiter(IdefixArray4D &Vc, IdefixArray1D &dx, ShockFlattening &sf): - Vc(Vc), dx(dx), flags(sf.flagArray), shockFlattening(sf.isActive) {} - - KOKKOS_FORCEINLINE_FUNCTION real MinModLim(const real dvp, const real dvm) const { - real dq= 0.0; - // MinMod - if(dvp*dvm >0.0) { - real dq = ( fabs(dvp) < fabs(dvm) ? dvp : dvm); - } - return(dq); - } - - KOKKOS_FORCEINLINE_FUNCTION real LimO3Lim(const real dvp, const real dvm, const real dx) const { - real r = 0.1; - real a,b,c,q, th, lim; - real eta, psi, eps = 1.e-12; - - th = dvm/(dvp + 1.e-16); - - q = (2.0 + th)/3.0; - - a = FMIN(1.5,2.0*th); - a = FMIN(q,a); - b = FMAX(-0.5*th,a); - c = FMIN(q,b); - psi = FMAX(0.0,c); - - eta = r*dx; - eta = (dvm*dvm + dvp*dvp)/(eta*eta); - if ( eta <= 1.0 - eps) { - lim = q; - } else if (eta >= 1.0 + eps) { - lim = psi; - } else { - psi = (1.0 - (eta - 1.0)/eps)*q - + (1.0 + (eta - 1.0)/eps)*psi; - lim = 0.5*psi; - } - return (lim); - } - - KOKKOS_FORCEINLINE_FUNCTION real VanLeerLim(const real dvp, const real dvm) const { - real dq= 0.0; - dq = (dvp*dvm > ZERO_F ? TWO_F*dvp*dvm/(dvp + dvm) : ZERO_F); - return(dq); - } - - KOKKOS_FORCEINLINE_FUNCTION real McLim(const real dvp, const real dvm) const { - real dq = 0; - if(dvp*dvm >0.0) { - real dqc = 0.5*(dvp+dvm); - real d2q = 2.0*( fabs(dvp) < fabs(dvm) ? dvp : dvm); - dq= fabs(d2q) < fabs(dqc) ? d2q : dqc; - } - return(dq); - } - - KOKKOS_FORCEINLINE_FUNCTION real PLMLim(const real dvp, const real dvm) const { - if constexpr(limiter == Limiter::VanLeer) return(VanLeerLim(dvp,dvm)); - if constexpr(limiter == Limiter::McLim) return(McLim(dvp,dvm)); - if constexpr(limiter == Limiter::MinMod) return(MinModLim(dvp,dvm)); - } - - template - KOKKOS_FORCEINLINE_FUNCTION int sign(T val) const { - return (T(0) < val) - (val < T(0)); -} - - KOKKOS_FORCEINLINE_FUNCTION void limitPPMFaceValues(const real vm1, const real v0, const real vp1, - const real vp2, real &vph) const { - // if local extremum, then use limited curvature estimate - if( (vp1-vph)*(vph-v0) < 0.0) { - // Collela, eqns. 85 - const real deltaL = (vm1-2*v0+vp1); - const real deltaC = 3*(v0-2*vph+vp1); - const real deltaR = (v0-2*vp1+vp2); - // Compute limited curvature estimate - real delta = 0.0; - - if(sign(deltaL) == sign(deltaC) && sign(deltaR) == sign(deltaC)) { - const real C = 1.25; - delta = C * FMIN(FABS(deltaL), FABS(deltaR)); - delta = sign(deltaC) * FMIN(delta, FABS(deltaC)); - } - vph = 0.5*(v0+vp1) - delta / 6.0; - } - } - - // This implementation follows the PPM4 scheme of Peterson & Hammet (PH13) - // SIAM J. Sci Comput (2013) - - KOKKOS_FORCEINLINE_FUNCTION void getPPMStates(const real vm2, const real vm1, const real v0, - const real vp1, const real vp2, real &vl, real &vr) - const { - const int n = 2; - - vr = 7.0/12.0*(v0+vp1) - 1.0/12.0*(vm1+vp2); - vl = 7.0/12.0*(vm1+v0) - 1.0/12.0*(vm2+vp1); - - limitPPMFaceValues(vm2,vm1,v0,vp1,vl); - limitPPMFaceValues(vm1,v0,vp1,vp2,vr); - - real d2qf = 6.0*(vl + vr - 2.0*v0); - real d2qc0 = vm1 + vp1 - 2.0*v0; - real d2qcp1 = v0 + vp2 - 2.0*vp1; - real d2qcm1 = vm2 + v0 - 2.0*vm1; - - real d2q = 0.0; - if(sign(d2qf) == sign(d2qc0) && sign(d2qf) == sign(d2qcp1) && sign(d2qf) == sign(d2qcm1)) { - // smooth extrememum - const real C = 1.25; - d2q = FMIN(FABS(d2qc0),FABS(d2qcp1)); - d2q = C * FMIN(FABS(d2qcm1), d2q); - d2q = sign(d2qf) * FMIN(FABS(d2qf), d2q); - } - - real qmax = FMAX(FMAX(FABS(vm1),FABS(v0)),FABS(vp1)); - real rho = 0.0; - // todo(GL): replace 1e-12 by mixed precision value - if(FABS(d2qf) > 1e-12*qmax) { - rho = d2q / d2qf; - } - - // PH13 3.31 - if( ((vr - v0)*(v0 - vl) <= 0) || (vm1 - v0)*(v0 - vp1) <= 0 ) { - if(rho <= (1.0 - 1e-12)) { - vl = v0 - rho * (v0 - vl); - vr = v0 + rho * (vr - v0); - } - } else { - // PH13 3.32 - if(FABS(vr-v0) >= n*FABS(v0-vl)) { - vr = v0 + n*(v0-vl); - } - if(FABS(vl-v0) >= n*FABS(v0-vr)) { - vl = v0 + n*(v0-vr); - } - } - } - - KOKKOS_FORCEINLINE_FUNCTION void ExtrapolatePrimVar(const int i, - const int j, - const int k, - real vL[], real vR[]) const { - // 1-- Store the primitive variables on the left, right, and averaged states - constexpr int ioffset = (dir==IDIR ? 1 : 0); - constexpr int joffset = (dir==JDIR ? 1 : 0); - constexpr int koffset = (dir==KDIR ? 1 : 0); - - for(int nv = 0 ; nv < nvmax ; nv++) { - if constexpr(order == 1) { - vL[nv] = Vc(nv,k-koffset,j-joffset,i-ioffset); - vR[nv] = Vc(nv,k,j,i); - } else if constexpr(order == 2) { - real dvm = Vc(nv,k-koffset,j-joffset,i-ioffset) - -Vc(nv,k-2*koffset,j-2*joffset,i-2*ioffset); - real dvp = Vc(nv,k,j,i)-Vc(nv,k-koffset,j-joffset,i-ioffset); - - real dv; - if(shockFlattening) { - if(flags(k-koffset,j-joffset,i-ioffset) == FlagShock::Shock) { - // Force slope limiter to minmod - dv = MinModLim(dvp,dvm); - } else { - dv = PLMLim(dvp,dvm); - } - } else { // No shock flattening - dv = PLMLim(dvp,dvm); - } - - vL[nv] = Vc(nv,k-koffset,j-joffset,i-ioffset) + HALF_F*dv; - - dvm = dvp; - dvp = Vc(nv,k+koffset,j+joffset,i+ioffset) - Vc(nv,k,j,i); - - if(shockFlattening) { - if(flags(k,j,i) == FlagShock::Shock) { - dv = MinModLim(dvp,dvm); - } else { - dv = PLMLim(dvp,dvm); - } - } else { // No shock flattening - dv = PLMLim(dvp,dvm); - } - - vR[nv] = Vc(nv,k,j,i) - HALF_F*dv; - } else if constexpr(order == 3) { - // 1D index along the chosen direction - const int index = ioffset*i + joffset*j + koffset*k; - real dvm = Vc(nv,k-koffset,j-joffset,i-ioffset) - -Vc(nv,k-2*koffset,j-2*joffset,i-2*ioffset); - real dvp = Vc(nv,k,j,i)-Vc(nv,k-koffset,j-joffset,i-ioffset); - - // Limo3 limiter - real dv = dvp * LimO3Lim(dvp, dvm, dx(index-1)); - vL[nv] = Vc(nv,k-koffset,j-joffset,i-ioffset) + HALF_F*dv; - - // Check positivity - if(nv==RHO) { - // If face element is negative, revert to vanleer - if(vL[nv] <= 0.0) { - dv = (dvp*dvm > ZERO_F ? TWO_F*dvp*dvm/(dvp + dvm) : ZERO_F); - vL[nv] = Vc(nv,k-koffset,j-joffset,i-ioffset) + HALF_F*dv; - } - } - #if HAVE_ENERGY - if(nv==PRS) { - // If face element is negative, revert to vanleer - if(vL[nv] <= 0.0) { - dv = (dvp*dvm > ZERO_F ? TWO_F*dvp*dvm/(dvp + dvm) : ZERO_F); - vL[nv] = Vc(nv,k-koffset,j-joffset,i-ioffset) + HALF_F*dv; - } - } - #endif - - dvm = dvp; - dvp = Vc(nv,k+koffset,j+joffset,i+ioffset) - Vc(nv,k,j,i); - - // Limo3 limiter - dv = dvm * LimO3Lim(dvm, dvp, dx(index)); - vR[nv] = Vc(nv,k,j,i) - HALF_F*dv; - - // Check positivity - if(nv==RHO) { - // If face element is negative, revert to vanleer - if(vR[nv] <= 0.0) { - dv = (dvp*dvm > ZERO_F ? TWO_F*dvp*dvm/(dvp + dvm) : ZERO_F); - vR[nv] = Vc(nv,k,j,i) - HALF_F*dv; - } - } - #if HAVE_ENERGY - if(nv==PRS) { - // If face element is negative, revert to vanleer - if(vR[nv] <= 0.0) { - dv = (dvp*dvm > ZERO_F ? TWO_F*dvp*dvm/(dvp + dvm) : ZERO_F); - vR[nv] = Vc(nv,k,j,i) - HALF_F*dv; - } - } - #endif - } else if constexpr(order == 4) { - // Reconstruction in cell i-1 - real vm2 = Vc(nv,k-3*koffset,j-3*joffset,i-3*ioffset);; - real vm1 = Vc(nv,k-2*koffset,j-2*joffset,i-2*ioffset); - real v0 = Vc(nv,k-koffset,j-joffset,i-ioffset); - real vp1 = Vc(nv,k,j,i); - real vp2 = Vc(nv,k+koffset,j+joffset,i+ioffset); - - real vr,vl; - getPPMStates(vm2, vm1, v0, vp1, vp2, vl, vr); - // vL= left side of current interface (i-1/2)= right side of cell i-1 - - // Check positivity - if(nv==RHO) { - // If face element is negative, revert to vanleer - if(vr <= 0.0) { - real dv = PLMLim(vp1-v0,v0-vm1); - vr = v0+HALF_F*dv; - } - } - #if HAVE_ENERGY - if(nv==PRS) { - // If face element is negative, revert to vanleer - if(vr <= 0.0) { - real dv = PLMLim(vp1-v0,v0-vm1); - vr = v0+HALF_F*dv; - } - } - #endif - - vL[nv] = vr; - // Reconstruction in cell i - - vm2 = vm1; - vm1 = v0; - v0 = vp1; - vp1 = vp2; - vp2 = Vc(nv,k+2*koffset,j+2*joffset,i+2*ioffset); - - getPPMStates(vm2, vm1, v0, vp1, vp2, vl, vr); - - // Check positivity - if(nv==RHO) { - // If face element is negative, revert to vanleer - if(vl <= 0.0) { - real dv = PLMLim(vp1-v0,v0-vm1); - vl = v0-HALF_F*dv; - } - } - #if HAVE_ENERGY - if(nv==PRS) { - // If face element is negative, revert to vanleer - if(vl <= 0.0) { - real dv = PLMLim(vp1-v0,v0-vm1); - vl = v0-HALF_F*dv; - } - } - #endif - - vR[nv] = vl; - } - } - } - - IdefixArray4D Vc; - IdefixArray1D dx; - IdefixArray3D flags; - bool shockFlattening{false}; -}; - - -#endif // HYDRO_SLOPELIMITER_HPP_ diff --git a/src/hydro/thermalDiffusion.hpp b/src/hydro/thermalDiffusion.hpp deleted file mode 100644 index b3d6c763..00000000 --- a/src/hydro/thermalDiffusion.hpp +++ /dev/null @@ -1,51 +0,0 @@ -// *********************************************************************************** -// Idefix MHD astrophysical code -// Copyright(C) Geoffroy R. J. Lesur -// and other code contributors -// Licensed under CeCILL 2.1 License, see COPYING for more information -// *********************************************************************************** - -#ifndef HYDRO_THERMALDIFFUSION_HPP_ -#define HYDRO_THERMALDIFFUSION_HPP_ - -#include "idefix.hpp" -#include "input.hpp" -#include "grid.hpp" -#include "hydro_defs.hpp" - - -// Forward class hydro declaration -class Hydro; -class DataBlock; - -class ThermalDiffusion { - public: - ThermalDiffusion(); // Default (empty) constructor - - void Init(Input &, Grid &, Hydro *); // Initialisation - - void ShowConfig(); // display configuration - - void AddDiffusiveFlux(int, const real); - - // Enroll user-defined viscous diffusivity - void EnrollThermalDiffusivity(DiffusivityFunc); - - IdefixArray4D viscSrc; // Source terms of the viscous operator - IdefixArray3D kappaArr; - - // pre-computed geometrical factors in non-cartesian geometry - IdefixArray1D one_dmu; - - private: - Hydro *hydro; // My parent hydro object - - // type of viscosity function - HydroModuleStatus haveThermalDiffusion{Disabled}; - DiffusivityFunc diffusivityFunc; - - // constant diffusion coefficient (when needed) - real kappa; -}; - -#endif // HYDRO_THERMALDIFFUSION_HPP_ diff --git a/src/hydro/viscosity.hpp b/src/hydro/viscosity.hpp deleted file mode 100644 index f124ae2a..00000000 --- a/src/hydro/viscosity.hpp +++ /dev/null @@ -1,54 +0,0 @@ -// *********************************************************************************** -// Idefix MHD astrophysical code -// Copyright(C) Geoffroy R. J. Lesur -// and other code contributors -// Licensed under CeCILL 2.1 License, see COPYING for more information -// *********************************************************************************** - -#ifndef HYDRO_VISCOSITY_HPP_ -#define HYDRO_VISCOSITY_HPP_ - -#include "idefix.hpp" -#include "input.hpp" -#include "grid.hpp" -#include "hydro_defs.hpp" - - -// Forward class hydro declaration -class Hydro; -class DataBlock; - -using ViscousDiffusivityFunc = void (*) (DataBlock &, const real t, - IdefixArray3D &, IdefixArray3D &); - -class Viscosity { - public: - Viscosity(); // Default (empty) constructor - - void Init(Input &, Grid &, Hydro *); // Initialisation - void ShowConfig(); // print configuration - - void AddViscousFlux(int, const real); - - // Enroll user-defined viscous diffusivity - void EnrollViscousDiffusivity(ViscousDiffusivityFunc); - - IdefixArray4D viscSrc; // Source terms of the viscous operator - IdefixArray3D eta1Arr; - IdefixArray3D eta2Arr; - - // pre-computed geometrical factors in non-cartesian geometry - IdefixArray1D one_dmu; - - private: - Hydro *hydro; // My parent hydro object - - // type of viscosity function - HydroModuleStatus haveViscosity{Disabled}; - ViscousDiffusivityFunc viscousDiffusivityFunc; - - // constant diffusion coefficient (when needed) - real eta1, eta2; -}; - -#endif // HYDRO_VISCOSITY_HPP_ diff --git a/src/idefix.hpp b/src/idefix.hpp index 57766b6e..a44bf4c2 100644 --- a/src/idefix.hpp +++ b/src/idefix.hpp @@ -20,7 +20,6 @@ using Layout = Kokkos::LayoutRight; /// Type of loops we admit in idefix (see loop.hpp for details) enum class LoopPattern { SIMDFOR, RANGE, MDRANGE, TPX, TPTTRTVR, UNDEFINED }; -enum class Limiter {VanLeer, MinMod, McLim}; #define YES 255 #define NO 0 @@ -76,7 +75,7 @@ enum class Limiter {VanLeer, MinMod, McLim}; #define MX2 (COMPONENTS >= 2 ? 2: 255) #define MX3 (COMPONENTS == 3 ? 3: 254) #if MHD == YES -#define BX1 (COMPONENTS + 1) +#define BX1 (COMPONENTS + 1 + HAVE_ENERGY) #define BX2 (COMPONENTS >= 2 ? (BX1+1): 252) #define BX3 (COMPONENTS == 3 ? (BX1+2): 251) @@ -86,14 +85,12 @@ enum class Limiter {VanLeer, MinMod, McLim}; #define BX3 251 #endif -#if HAVE_ENERGY -#if MHD == YES - #define ENG (2*COMPONENTS + 1) -#else +#if HAVE_ENERGY == 1 #define ENG (COMPONENTS + 1) +#else + #define ENG 250 #endif - #define PRS ENG -#endif +#define PRS ENG #define VX1 MX1 #define VX2 MX2 @@ -101,8 +98,12 @@ enum class Limiter {VanLeer, MinMod, McLim}; #if MHD == YES #define NFLX (1 + 2*COMPONENTS + HAVE_ENERGY) + #define TRG NFLX // Gas Tracer index + #define TRD (NFLX-COMPONENTS) // Dust Tracer index #else #define NFLX (1 + COMPONENTS + HAVE_ENERGY) + #define TRG NFLX + #define TRD NFLX #endif // Face-centered variables @@ -217,6 +218,7 @@ using IdfxFileHandler = FILE*; // Types of boundary which can be treated enum BoundaryType { internal, periodic, reflective, outflow, shearingbox, axis, userdef}; enum BoundarySide { left, right}; +enum class SliceType {Cut, Average}; // Type of grid coarsening enum GridCoarsening{disabled, diff --git a/src/input.cpp b/src/input.cpp index 4c8fbc3d..86c6275e 100644 --- a/src/input.cpp +++ b/src/input.cpp @@ -18,7 +18,8 @@ #include "idefix.hpp" #include "input.hpp" -#include "gitversion.hpp" +#include "version.hpp" +#include "profiler.hpp" // Flag will be set if a signal has been received bool Input::abortRequested = false; @@ -147,37 +148,11 @@ void Input::ParseCommandLine(int argc, char **argv) { sirestart = std::string(argv[++i]); } else { // implicitly restart from the latest existing dumpfile - // implementation detail: we look for the existing dumpfile with the highest - // number, not necessarilly the latest timestamp ! - const std::vector files = Input::getDirectoryFiles(); - int ifile{-1}; - int irestart{-1}; - - for (const auto& file : files) { - if (Input::getFileExtension(file).compare("dmp") != 0) continue; - // parse the dumpfile number from filename "dump.????.dmp" - if(file.substr(0,5) != "dump.") continue; - try { - ifile = std::stoi(file.substr(5, 4)); - } catch (...) { - // woops, pattern doesn't match! - ifile = -1; - } - irestart = std::max(irestart, ifile); - } - sirestart = std::to_string(irestart); - if(irestart==-1) { - IDEFIX_WARNING("cannot find a valid restart dump file in current directory"); - } - } - int restartn = std::stoi(sirestart); - if(restartn>=0) { - inputParameters["CommandLine"]["restart"].push_back(sirestart); - this->restartRequested = true; - this->restartFileNumber = restartn; - } else { - IDEFIX_WARNING("Invalid -restart option, I will ignore it."); + sirestart = "-1"; } + inputParameters["CommandLine"]["restart"].push_back(sirestart); + this->restartRequested = true; + this->restartFileNumber = std::stoi(sirestart); } else if(std::string(argv[i]) == "-i") { // Loop on dimensions if((++i) >= argc) IDEFIX_ERROR( @@ -192,14 +167,25 @@ void Input::ParseCommandLine(int argc, char **argv) { } this->maxCycles = std::stoi(std::string(argv[++i])); inputParameters["CommandLine"]["maxCycles"].push_back(std::to_string(maxCycles)); + } else if(std::string(argv[i]) == "-force_init") { + this->forceInitRequested = true; } else if(std::string(argv[i]) == "-nowrite") { this->forceNoWrite = true; enableLogs = false; } else if(std::string(argv[i]) == "-nolog") { enableLogs = false; + } else if(std::string(argv[i]) == "-profile") { + idfx::prof.EnablePerformanceProfiling(); } else if(std::string(argv[i]) == "-Werror") { idfx::warningsAreErrors = true; + } else if(std::string(argv[i]) == "-version" || std::string(argv[i]) == "-v") { + PrintVersion(); + idfx::safeExit(0); + } else if(std::string(argv[i]) == "-help" || std::string(argv[i]) == "-h") { + PrintOptions(); + idfx::safeExit(0); } else { + PrintOptions(); msg << "Unknown option " << argv[i]; IDEFIX_ERROR(msg); } @@ -308,32 +294,6 @@ bool Input::CheckForAbort() { #endif } -std::vector Input::getDirectoryFiles() { - // List files in the current directory - // adapted from - // http://www.codebind.com/cpp-tutorial/cpp-program-list-files-directory-windows-linux/ - const std::string& dir = std::string("."); - std::vector files; - std::shared_ptr directory_ptr(opendir(dir.c_str()), [](DIR* dir){ dir && closedir(dir); }); - struct dirent *dirent_ptr; - if (!directory_ptr) { - idfx::cout << "Error opening : " << std::strerror(errno) << dir << std::endl; - return files; - } - - while ((dirent_ptr = readdir(directory_ptr.get())) != nullptr) { - files.push_back(std::string(dirent_ptr->d_name)); - } - return files; -} - - -std::string Input::getFileExtension(const std::string file_name) { - int position = file_name.find_last_of("."); - std::string ext = file_name.substr(position+1); - return ext; -} - // Get a string in a block, parameter, position of the file std::string Input::GetString(std::string blockName, std::string paramName, int num) { IDEFIX_DEPRECATED("Input::GetString is deprecated. Use Input::Get instead"); @@ -417,7 +377,47 @@ void Input::PrintLogo() { idfx::cout << " `'''4x.dX +^ `''MMMMHM?L.."<< std::endl; idfx::cout << " ``' `'`'`'`"<< std::endl; idfx::cout << std::endl; + PrintVersion(); idfx::cout << std::endl; idfx::cout << std::endl; - idfx::cout << " This is Idefix " << GITVERSION << std::endl; +} + +void Input::PrintOptions() { + idfx::cout << "List of valid arguments:" << std::endl << std::endl; + #ifdef WITH_MPI + idfx::cout << " -dec " + D_SELECT(<< " nx1 ", + << " nx1 nx2", + << " nx1 nx2 nx3") + << std::endl; + idfx::cout << " Force an mpi domain decomposition with n processes in each direction" + << std::endl; + #endif + idfx::cout << " -restart n" << std::endl; + idfx::cout << " Restart from dumpfile n. If n is ommited, Idefix restart from the latest" + << " generated dump file." << std::endl; + idfx::cout << " -i xxx" << std::endl; + idfx::cout << " Use the input file xxx instead of the default idefix.ini" << std::endl; + idfx::cout << " -maxcycles n" << std::endl; + idfx::cout << " Perform at most n integration cycles." << std::endl; + idfx::cout << " -force_init" << std::endl; + idfx::cout << " Call initial conditions before reading dump file "; + idfx::cout << "(this has no effect if -restart is not also passed)" << std::endl; + idfx::cout << " -nowrite" << std::endl; + idfx::cout << " Do not generate any output file." << std::endl; + idfx::cout << " -nolog" << std::endl; + idfx::cout << " Do not write any log file." << std::endl; + idfx::cout << " -profile" << std::endl; + idfx::cout << " Enable on-the-fly performance profiling." << std::endl; + idfx::cout << " -Werror" << std::endl; + idfx::cout << " Consider warnings as errors." << std::endl; + idfx::cout << " -v/-version" << std::endl; + idfx::cout << " Show Idefix and kokkos version." << std::endl; + idfx::cout << " -h/-help" << std::endl; + idfx::cout << " Show this message." << std::endl; +} + +void Input::PrintVersion() { + idfx::cout << " Idefix version " << IDEFIX_VERSION << std::endl; + idfx::cout << " Built against Kokkos " << KOKKOS_VERSION << std::endl; } diff --git a/src/input.hpp b/src/input.hpp index 7216b0e8..18fcf260 100644 --- a/src/input.hpp +++ b/src/input.hpp @@ -52,9 +52,12 @@ class Input { Input(); void PrintLogo(); + void PrintOptions(); + void PrintVersion(); bool restartRequested{false}; //< Should we restart? int restartFileNumber; //< if yes, from which file? + bool forceInitRequested{false}; // call DataBlock::InitFlow even on restarts ? static bool abortRequested; //< Did we receive an abort signal (USR2) from the system? @@ -67,8 +70,6 @@ class Input { IdefixInputContainer inputParameters; void ParseCommandLine(int , char **argv); static void signalHandler(int); - std::vector getDirectoryFiles(); - std::string getFileExtension(const std::string file_name); Kokkos::Timer timer; double lastStopFileCheck; diff --git a/src/kokkos b/src/kokkos index a1d045da..1a3ea28f 160000 --- a/src/kokkos +++ b/src/kokkos @@ -1 +1 @@ -Subproject commit a1d045da4cc299217e2831f71dedff72e4f2aecf +Subproject commit 1a3ea28f6e97b4c9dd2c8ceed53ad58ed5f94dfe diff --git a/src/loop.hpp b/src/loop.hpp index a958dd31..8ad345de 100644 --- a/src/loop.hpp +++ b/src/loop.hpp @@ -66,6 +66,9 @@ template inline void idefix_for(const std::string & NAME, const int & IB, const int & IE, Function function) { + #ifdef DEBUG + idfx::pushRegion("idefix_for("+NAME+")"); + #endif const int NI = IE - IB; Kokkos::parallel_for(NAME, NI, KOKKOS_LAMBDA (const int& IDX) { @@ -73,6 +76,10 @@ inline void idefix_for(const std::string & NAME, i += IB; function(i); }); + #ifdef DEBUG + Kokkos::fence(); + idfx::popRegion(); + #endif } @@ -82,6 +89,9 @@ inline void idefix_for(const std::string & NAME, const int & JB, const int & JE, const int & IB, const int & IE, Function function) { + #ifdef DEBUG + idfx::pushRegion("idefix_for("+NAME+")"); + #endif // Kokkos 1D Range if constexpr(defaultLoop == LoopPattern::RANGE) { const int NJ = JE - JB; @@ -123,6 +133,10 @@ inline void idefix_for(const std::string & NAME, } else { throw std::runtime_error("Unknown/undefined LoopPattern used."); } + #ifdef DEBUG + Kokkos::fence(); + idfx::popRegion(); + #endif } @@ -133,6 +147,9 @@ inline void idefix_for(const std::string & NAME, const int & JB, const int & JE, const int & IB, const int & IE, Function function) { + #ifdef DEBUG + idfx::pushRegion("idefix_for("+NAME+")"); + #endif // Kokkos 1D Range if constexpr(defaultLoop == LoopPattern::RANGE) { const int NK = KE - KB; @@ -201,6 +218,10 @@ inline void idefix_for(const std::string & NAME, } else { throw std::runtime_error("Unknown/undefined LoopPattern used."); } + #ifdef DEBUG + Kokkos::fence(); + idfx::popRegion(); + #endif } // 4D loop @@ -211,6 +232,9 @@ inline void idefix_for(const std::string & NAME, const int JB, const int JE, const int IB, const int IE, Function function) { + #ifdef DEBUG + idfx::pushRegion("idefix_for("+NAME+")"); + #endif // Kokkos 1D Range if constexpr(defaultLoop == LoopPattern::RANGE) { const int NN = (NE) - (NB); @@ -293,6 +317,10 @@ inline void idefix_for(const std::string & NAME, } else { throw std::runtime_error("Unknown/undefined LoopPattern used."); } + #ifdef DEBUG + Kokkos::fence(); + idfx::popRegion(); + #endif } #endif // LOOP_HPP_ diff --git a/src/macros.hpp b/src/macros.hpp index 1d8229f4..9a484c3d 100644 --- a/src/macros.hpp +++ b/src/macros.hpp @@ -14,24 +14,21 @@ #if COMPONENTS == 1 #define EXPAND(a,b,c) a #define SELECT(a,b,c) a - #define ARG_EXPAND(a,b,c) a #endif #if COMPONENTS == 2 #define EXPAND(a,b,c) a b #define SELECT(a,b,c) b - #define ARG_EXPAND(a,b,c) a,b #endif #if COMPONENTS == 3 #define EXPAND(a,b,c) a b c #define SELECT(a,b,c) c - #define ARG_EXPAND(a,b,c) a,b,c + #endif #if DIMENSIONS == 1 #define D_EXPAND(a,b,c) a - #define ARG_DEXPAND(a,b,c) a #define D_SELECT(a,b,c) a #define IOFFSET 1 #define JOFFSET 0 @@ -40,7 +37,6 @@ #if DIMENSIONS == 2 #define D_EXPAND(a,b,c) a b - #define ARG_DEXPAND(a,b,c) a,b #define D_SELECT(a,b,c) b #define IOFFSET 1 #define JOFFSET 1 @@ -49,7 +45,6 @@ #if DIMENSIONS == 3 #define D_EXPAND(a,b,c) a b c - #define ARG_DEXPAND(a,b,c) a,b,c #define D_SELECT(a,b,c) c #define IOFFSET 1 #define JOFFSET 1 @@ -168,4 +163,36 @@ } \ } + +#ifdef RUNTIME_CHECKS + #define RUNTIME_CHECK_HOST(condition, ...) { \ + if(!(condition)) { \ + char msg[255]; \ + snprintf(msg, sizeof(msg), __VA_ARGS__); \ + IDEFIX_ERROR(msg); \ + } \ + } + + #if defined(KOKKOS_ENABLE_HIP) || defined(KOKKOS_ENABLE_CUDA) + // string formatting functions can't be accessed from GPU kernels, + // so we fallback to a simple error message + #define RUNTIME_CHECK_KERNEL(condition, fallback_msg, ...) { \ + if(!(condition)) Kokkos::abort(fallback_msg); \ + } + #else + #define RUNTIME_CHECK_KERNEL(condition, fallback_msg, ...) { \ + if(!(condition)) { \ + char msg[255]; \ + snprintf(msg, sizeof(msg), __VA_ARGS__); \ + Kokkos::abort(msg); \ + } \ + } + #endif // if defined(KOKKOS_ENABLE_HIP) || defined(KOKKOS_ENABLE_CUDA) + +#else + #define RUNTIME_CHECK_HOST(condition, msg, ...) {} + #define RUNTIME_CHECK_KERNEL(condition, fallback_msg, ...) {} +#endif // ifdef RUNTIME_CHECKS + + #endif // MACROS_HPP_ diff --git a/src/main.cpp b/src/main.cpp index 5e1ff7ab..0e67ad09 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -9,7 +9,7 @@ //@HEADER // ************************************************************************ // -// IDEFIX v 1.0-alpha +// IDEFIX v 2.0.0 // // ************************************************************************ //@HEADER @@ -31,7 +31,7 @@ #include "input.hpp" #include "grid.hpp" #include "gridHost.hpp" -#include "hydro.hpp" +#include "fluid.hpp" #include "dataBlock.hpp" #include "timeIntegrator.hpp" #include "setup.hpp" @@ -88,8 +88,7 @@ int main( int argc, char* argv[] ) { gridHost.SyncToDevice(); // instantiate required objects. - DataBlock data; - data.InitFromGrid(grid, input); + DataBlock data(grid, input); TimeIntegrator Tint(input,data); Output output(input, data); Setup mysetup(input, grid, data, output); @@ -112,21 +111,30 @@ int main( int argc, char* argv[] ) { /////////////////////////////// // Are we restarting? if(input.restartRequested) { + if(input.forceInitRequested) { + idfx::pushRegion("Setup::Initflow"); + mysetup.InitFlow(data); + data.DeriveVectorPotential(); + idfx::popRegion(); + } idfx::cout << "Main: Restarting from dump file." << std::endl; - output.RestartFromDump(data,input.restartFileNumber); - data.SetBoundaries(); - output.CheckForWrites(data); - } else { + bool restartSuccess = output.RestartFromDump(data,input.restartFileNumber); + if(!restartSuccess) { + idfx::cout << "Main: restart aborted." << std::endl; + input.restartRequested = false; + } else { + data.SetBoundaries(); + } + } + if(!input.restartRequested) { idfx::cout << "Main: Creating initial conditions." << std::endl; idfx::pushRegion("Setup::Initflow"); mysetup.InitFlow(data); idfx::popRegion(); - #if MHD == YES && defined(EVOLVE_VECTOR_POTENTIAL) - data.hydro.emf.ComputeMagFieldFromA(data.hydro.Ve, data.hydro.Vs); - #endif + data.DeriveVectorPotential(); // This does something only when evolveVectorPotential is on data.SetBoundaries(); - output.CheckForWrites(data); data.Validate(); + output.CheckForWrites(data); } /////////////////////////////// diff --git a/src/mpi.cpp b/src/mpi.cpp index fe59f052..9ca66b68 100644 --- a/src/mpi.cpp +++ b/src/mpi.cpp @@ -11,6 +11,7 @@ #include #include // NOLINT [build/c++11] #include // NOLINT [build/c++11] +#include #include "idefix.hpp" #include "dataBlock.hpp" @@ -76,6 +77,7 @@ void Mpi::Init(Grid *grid, std::vector inputMap, bufferSizeX1 = nghost[IDIR] * nint[JDIR] * nint[KDIR] * mapNVars; if(haveVs) { + bufferSizeX1 += nghost[IDIR] * nint[JDIR] * nint[KDIR]; #if DIMENSIONS>=2 bufferSizeX1 += nghost[IDIR] * (nint[JDIR]+1) * nint[KDIR]; #endif @@ -86,10 +88,10 @@ void Mpi::Init(Grid *grid, std::vector inputMap, } - BufferRecvX1[faceLeft ] = IdefixArray1D("BufferRecvX1Left", bufferSizeX1); - BufferRecvX1[faceRight] = IdefixArray1D("BufferRecvX1Right",bufferSizeX1); - BufferSendX1[faceLeft ] = IdefixArray1D("BufferSendX1Left", bufferSizeX1); - BufferSendX1[faceRight] = IdefixArray1D("BufferSendX1Right",bufferSizeX1); + BufferRecvX1[faceLeft ] = Buffer(bufferSizeX1); + BufferRecvX1[faceRight] = Buffer(bufferSizeX1); + BufferSendX1[faceLeft ] = Buffer(bufferSizeX1); + BufferSendX1[faceRight] = Buffer(bufferSizeX1); // Number of cells in X2 boundary condition (only required when problem >2D): #if DIMENSIONS >= 2 @@ -97,15 +99,18 @@ void Mpi::Init(Grid *grid, std::vector inputMap, if(haveVs) { // IDIR bufferSizeX2 += (ntot[IDIR]+1) * nghost[JDIR] * nint[KDIR]; + #if DIMENSIONS>=2 + bufferSizeX2 += ntot[IDIR] * nghost[JDIR] * nint[KDIR]; + #endif #if DIMENSIONS==3 bufferSizeX2 += ntot[IDIR] * nghost[JDIR] * (nint[KDIR]+1); #endif // DIMENSIONS } - BufferRecvX2[faceLeft ] = IdefixArray1D("BufferRecvX2Left", bufferSizeX2); - BufferRecvX2[faceRight] = IdefixArray1D("BufferRecvX2Right",bufferSizeX2); - BufferSendX2[faceLeft ] = IdefixArray1D("BufferSendX2Left", bufferSizeX2); - BufferSendX2[faceRight] = IdefixArray1D("BufferSendX2Right",bufferSizeX2); + BufferRecvX2[faceLeft ] = Buffer(bufferSizeX2); + BufferRecvX2[faceRight] = Buffer(bufferSizeX2); + BufferSendX2[faceLeft ] = Buffer(bufferSizeX2); + BufferSendX2[faceRight] = Buffer(bufferSizeX2); #endif // Number of cells in X3 boundary condition (only required when problem is 3D): @@ -117,12 +122,14 @@ void Mpi::Init(Grid *grid, std::vector inputMap, bufferSizeX3 += (ntot[IDIR]+1) * ntot[JDIR] * nghost[KDIR]; // JDIR bufferSizeX3 += ntot[IDIR] * (ntot[JDIR]+1) * nghost[KDIR]; + // KDIR + bufferSizeX3 += ntot[IDIR] * ntot[JDIR] * nghost[KDIR]; } - BufferRecvX3[faceLeft ] = IdefixArray1D("BufferRecvX3Left", bufferSizeX3); - BufferRecvX3[faceRight] = IdefixArray1D("BufferRecvX3Right",bufferSizeX3); - BufferSendX3[faceLeft ] = IdefixArray1D("BufferSendX3Left", bufferSizeX3); - BufferSendX3[faceRight] = IdefixArray1D("BufferSendX3Right",bufferSizeX3); + BufferRecvX3[faceLeft ] = Buffer(bufferSizeX3); + BufferRecvX3[faceRight] = Buffer(bufferSizeX3); + BufferSendX3[faceLeft ] = Buffer(bufferSizeX3); + BufferSendX3[faceRight] = Buffer(bufferSizeX3); #endif // DIMENSIONS #ifdef MPI_PERSISTENT @@ -235,10 +242,9 @@ void Mpi::ExchangeX1(IdefixArray4D Vc, IdefixArray4D Vs) { idfx::pushRegion("Mpi::ExchangeX1"); // Load the buffers with data - int ibeg,iend,jbeg,jend,kbeg,kend,offset; - int nx,ny,nz; - IdefixArray1D BufferLeft=BufferSendX1[faceLeft]; - IdefixArray1D BufferRight=BufferSendX1[faceRight]; + int ibeg,iend,jbeg,jend,kbeg,kend,offset,nx; + Buffer BufferLeft = BufferSendX1[faceLeft]; + Buffer BufferRight = BufferSendX1[faceRight]; IdefixArray1D map = this->mapVars; // If MPI Persistent, start receiving even before the buffers are filled @@ -260,41 +266,53 @@ void Mpi::ExchangeX1(IdefixArray4D Vc, IdefixArray4D Vs) { offset = end[IDIR]; // Distance between beginning of left and right ghosts jbeg = beg[JDIR]; jend = end[JDIR]; - ny = jend - jbeg; + kbeg = beg[KDIR]; kend = end[KDIR]; - nz = kend - kbeg; - idefix_for("LoadBufferX1Vc",0,mapNVars,kbeg,kend,jbeg,jend,ibeg,iend, - KOKKOS_LAMBDA (int n, int k, int j, int i) { - BufferLeft(i + (j-jbeg)*nx + (k-kbeg)*nx*ny + n*nx*ny*nz ) = Vc(map(n),k,j,i+nx); - BufferRight(i + (j-jbeg)*nx + (k-kbeg)*nx*ny + n*nx*ny*nz ) = Vc(map(n),k,j,i+offset-nx); - } - ); + + BufferLeft.ResetPointer(); + BufferRight.ResetPointer(); + + BufferLeft.Pack(Vc, map, std::make_pair(ibeg+nx, iend+nx), + std::make_pair(jbeg , jend), + std::make_pair(kbeg , kend)); + + BufferRight.Pack(Vc, map, std::make_pair(ibeg+offset-nx, iend+offset-nx), + std::make_pair(jbeg , jend), + std::make_pair(kbeg , kend)); // Load face-centered field in the buffer if(haveVs) { + BufferLeft.Pack(Vs, BX1s,std::make_pair(ibeg+nx+1, iend+nx+1), + std::make_pair(jbeg , jend), + std::make_pair(kbeg , kend)); + + BufferRight.Pack(Vs, BX1s, std::make_pair(ibeg+offset-nx, iend+offset-nx), + std::make_pair(jbeg , jend), + std::make_pair(kbeg , kend)); + #if DIMENSIONS >= 2 - int VsIndex = mapNVars*nx*ny*nz; - idefix_for("LoadBufferX1JDIR",kbeg,kend,jbeg,jend+1,ibeg,iend, - KOKKOS_LAMBDA (int k, int j, int i) { - BufferLeft(i + (j-jbeg)*nx + (k-kbeg)*nx*(ny+1) + VsIndex ) = Vs(JDIR,k,j,i+nx); - BufferRight(i + (j-jbeg)*nx + (k-kbeg)*nx*(ny+1) + VsIndex ) = Vs(JDIR,k,j,i+offset-nx); - } - ); + BufferLeft.Pack(Vs, BX2s,std::make_pair(ibeg+nx, iend+nx), + std::make_pair(jbeg , jend+1), + std::make_pair(kbeg , kend)); + + BufferRight.Pack(Vs, BX2s, std::make_pair(ibeg+offset-nx, iend+offset-nx), + std::make_pair(jbeg , jend+1), + std::make_pair(kbeg , kend)); #endif #if DIMENSIONS == 3 - VsIndex = mapNVars*nx*ny*nz + nx*(ny+1)*nz; - idefix_for("LoadBufferX1KDIR",kbeg,kend+1,jbeg,jend,ibeg,iend, - KOKKOS_LAMBDA (int k, int j, int i) { - BufferLeft(i + (j-jbeg)*nx + (k-kbeg)*nx*ny + VsIndex ) = Vs(KDIR,k,j,i+nx); - BufferRight(i + (j-jbeg)*nx + (k-kbeg)*nx*ny + VsIndex ) = Vs(KDIR,k,j,i+offset-nx); - } - ); + BufferLeft.Pack(Vs, BX3s,std::make_pair(ibeg+nx, iend+nx), + std::make_pair(jbeg , jend), + std::make_pair(kbeg , kend+1)); + + BufferRight.Pack(Vs, BX3s, std::make_pair(ibeg+offset-nx, iend+offset-nx), + std::make_pair(jbeg , jend), + std::make_pair(kbeg , kend+1)); #endif } @@ -364,35 +382,45 @@ void Mpi::ExchangeX1(IdefixArray4D Vc, IdefixArray4D Vs) { BufferLeft=BufferRecvX1[faceLeft]; BufferRight=BufferRecvX1[faceRight]; + BufferLeft.ResetPointer(); + BufferRight.ResetPointer(); + + BufferLeft.Unpack(Vc, map,std::make_pair(ibeg, iend), + std::make_pair(jbeg , jend), + std::make_pair(kbeg , kend)); + + BufferRight.Unpack(Vc, map,std::make_pair(ibeg+offset, iend+offset), + std::make_pair(jbeg , jend), + std::make_pair(kbeg , kend)); // We fill the ghost zones - idefix_for("StoreBufferX1Vc",0,mapNVars,kbeg,kend,jbeg,jend,ibeg,iend, - KOKKOS_LAMBDA (int n, int k, int j, int i) { - Vc(map(n),k,j,i) = BufferLeft(i + (j-jbeg)*nx + (k-kbeg)*nx*ny + n*nx*ny*nz ); - Vc(map(n),k,j,i+offset) = BufferRight(i + (j-jbeg)*nx + (k-kbeg)*nx*ny + n*nx*ny*nz ); - } - ); if(haveVs) { + BufferLeft.Unpack(Vs, BX1s, std::make_pair(ibeg, iend), + std::make_pair(jbeg , jend), + std::make_pair(kbeg , kend)); + + BufferRight.Unpack(Vs, BX1s, std::make_pair(ibeg+offset+1, iend+offset+1), + std::make_pair(jbeg , jend), + std::make_pair(kbeg , kend)); + #if DIMENSIONS >= 2 - int VsIndex = mapNVars*nx*ny*nz; + BufferLeft.Unpack(Vs, BX2s, std::make_pair(ibeg, iend), + std::make_pair(jbeg , jend+1), + std::make_pair(kbeg , kend)); - idefix_for("StoreBufferX1JDIR",kbeg,kend,jbeg,jend+1,ibeg,iend, - KOKKOS_LAMBDA (int k, int j, int i) { - Vs(JDIR,k,j,i) = BufferLeft(i + (j-jbeg)*nx + (k-kbeg)*nx*(ny+1) + VsIndex ); - Vs(JDIR,k,j,i+offset) = BufferRight(i + (j-jbeg)*nx + (k-kbeg)*nx*(ny+1) + VsIndex ); - } - ); + BufferRight.Unpack(Vs, BX2s, std::make_pair(ibeg+offset, iend+offset), + std::make_pair(jbeg , jend+1), + std::make_pair(kbeg , kend)); #endif #if DIMENSIONS == 3 - VsIndex = mapNVars*nx*ny*nz + nx*(ny+1)*nz; + BufferLeft.Unpack(Vs, BX3s, std::make_pair(ibeg, iend), + std::make_pair(jbeg , jend), + std::make_pair(kbeg , kend+1)); - idefix_for("StoreBufferX1KDIR",kbeg,kend+1,jbeg,jend,ibeg,iend, - KOKKOS_LAMBDA ( int k, int j, int i) { - Vs(KDIR,k,j,i) = BufferLeft(i + (j-jbeg)*nx + (k-kbeg)*nx*ny + VsIndex ); - Vs(KDIR,k,j,i+offset) = BufferRight(i + (j-jbeg)*nx + (k-kbeg)*nx*ny + VsIndex ); - } - ); + BufferRight.Unpack(Vs, BX3s, std::make_pair(ibeg+offset, iend+offset), + std::make_pair(jbeg , jend), + std::make_pair(kbeg , kend+1)); #endif } @@ -416,10 +444,9 @@ void Mpi::ExchangeX2(IdefixArray4D Vc, IdefixArray4D Vs) { idfx::pushRegion("Mpi::ExchangeX2"); // Load the buffers with data - int ibeg,iend,jbeg,jend,kbeg,kend,offset; - int nx,ny,nz; - IdefixArray1D BufferLeft=BufferSendX2[faceLeft]; - IdefixArray1D BufferRight=BufferSendX2[faceRight]; + int ibeg,iend,jbeg,jend,kbeg,kend,offset,ny; + Buffer BufferLeft=BufferSendX2[faceLeft]; + Buffer BufferRight=BufferSendX2[faceRight]; IdefixArray1D map = this->mapVars; // If MPI Persistent, start receiving even before the buffers are filled @@ -437,43 +464,53 @@ void Mpi::ExchangeX2(IdefixArray4D Vc, IdefixArray4D Vs) { // Coordinates of the ghost region which needs to be transfered ibeg = 0; iend = ntot[IDIR]; - nx = ntot[IDIR]; // Number of points in x + jbeg = 0; jend = nghost[JDIR]; offset = end[JDIR]; // Distance between beginning of left and right ghosts ny = nghost[JDIR]; + kbeg = beg[KDIR]; kend = end[KDIR]; - nz = kend - kbeg; - idefix_for("LoadBufferX2Vc",0,mapNVars,kbeg,kend,jbeg,jend,ibeg,iend, - KOKKOS_LAMBDA (int n, int k, int j, int i) { - BufferLeft(i + (j-jbeg)*nx + (k-kbeg)*nx*ny + n*nx*ny*nz ) = Vc(map(n),k,j+ny,i); - BufferRight(i + (j-jbeg)*nx + (k-kbeg)*nx*ny + n*nx*ny*nz ) = Vc(map(n),k,j+offset-ny,i); - } - ); + BufferLeft.ResetPointer(); + BufferRight.ResetPointer(); + + BufferLeft.Pack(Vc, map, std::make_pair(ibeg , iend), + std::make_pair(jbeg+ny , jend+ny), + std::make_pair(kbeg , kend)); + + BufferRight.Pack(Vc, map, std::make_pair(ibeg , iend), + std::make_pair(jbeg+offset-ny , jend+offset-ny), + std::make_pair(kbeg , kend)); // Load face-centered field in the buffer if(haveVs) { - int VsIndex = mapNVars*nx*ny*nz; - - idefix_for("LoadBufferX2IDIR",kbeg,kend,jbeg,jend,ibeg,iend+1, - KOKKOS_LAMBDA (int k, int j, int i) { - BufferLeft(i + (j-jbeg)*(nx+1) + (k-kbeg)*(nx+1)*ny + VsIndex ) = Vs(IDIR,k,j+ny,i); - BufferRight(i + (j-jbeg)*(nx+1) + (k-kbeg)*(nx+1)*ny + VsIndex ) = Vs(IDIR,k,j+offset-ny,i); - } - ); + BufferLeft.Pack(Vs, BX1s,std::make_pair(ibeg , iend+1), + std::make_pair(jbeg+ny , jend+ny), + std::make_pair(kbeg , kend)); + BufferRight.Pack(Vs, BX1s, std::make_pair(ibeg , iend+1), + std::make_pair(jbeg+offset-ny , jend+offset-ny), + std::make_pair(kbeg , kend)); + #if DIMENSIONS >= 2 + BufferLeft.Pack(Vs, BX2s,std::make_pair(ibeg , iend), + std::make_pair(jbeg+ny+1 , jend+ny+1), + std::make_pair(kbeg , kend)); + BufferRight.Pack(Vs, BX2s, std::make_pair(ibeg , iend), + std::make_pair(jbeg+offset-ny , jend+offset-ny), + std::make_pair(kbeg , kend)); + #endif #if DIMENSIONS == 3 - VsIndex = mapNVars*nx*ny*nz + (nx+1)*ny*nz; - idefix_for("LoadBufferX2KDIR",kbeg,kend+1,jbeg,jend,ibeg,iend, - KOKKOS_LAMBDA (int k, int j, int i) { - BufferLeft(i + (j-jbeg)*nx + (k-kbeg)*nx*ny + VsIndex ) = Vs(KDIR,k,j+ny,i); - BufferRight(i + (j-jbeg)*nx + (k-kbeg)*nx*ny + VsIndex ) = Vs(KDIR,k,j+offset-ny,i); - } - ); + BufferLeft.Pack(Vs, BX3s,std::make_pair(ibeg , iend), + std::make_pair(jbeg+ny , jend+ny), + std::make_pair(kbeg , kend+1)); + + BufferRight.Pack(Vs, BX3s, std::make_pair(ibeg , iend), + std::make_pair(jbeg+offset-ny , jend+offset-ny), + std::make_pair(kbeg , kend+1)); #endif } @@ -543,35 +580,44 @@ void Mpi::ExchangeX2(IdefixArray4D Vc, IdefixArray4D Vs) { BufferLeft=BufferRecvX2[faceLeft]; BufferRight=BufferRecvX2[faceRight]; + BufferLeft.ResetPointer(); + BufferRight.ResetPointer(); + // We fill the ghost zones + BufferLeft.Unpack(Vc, map,std::make_pair(ibeg, iend), + std::make_pair(jbeg , jend), + std::make_pair(kbeg , kend)); - idefix_for("StoreBufferX2Vc",0,mapNVars,kbeg,kend,jbeg,jend,ibeg,iend, - KOKKOS_LAMBDA (int n, int k, int j, int i) { - Vc(map(n),k,j,i) = BufferLeft(i + (j-jbeg)*nx + (k-kbeg)*nx*ny + n*nx*ny*nz ); - Vc(map(n),k,j+offset,i) = BufferRight(i + (j-jbeg)*nx + (k-kbeg)*nx*ny + n*nx*ny*nz ); - } - ); + BufferRight.Unpack(Vc, map,std::make_pair(ibeg , iend), + std::make_pair(jbeg+offset , jend+offset), + std::make_pair(kbeg , kend)); + // We fill the ghost zones - // Load face-centered field in the buffer if(haveVs) { - int VsIndex = mapNVars*nx*ny*nz; + BufferLeft.Unpack(Vs, BX1s, std::make_pair(ibeg, iend+1), + std::make_pair(jbeg , jend), + std::make_pair(kbeg , kend)); - idefix_for("StoreBufferX2IDIR",kbeg,kend,jbeg,jend,ibeg,iend+1, - KOKKOS_LAMBDA (int k, int j, int i) { - Vs(IDIR,k,j,i) = BufferLeft(i + (j-jbeg)*(nx+1) + (k-kbeg)*(nx+1)*ny + VsIndex ); - Vs(IDIR,k,j+offset,i) = BufferRight(i + (j-jbeg)*(nx+1) + (k-kbeg)*(nx+1)*ny + VsIndex ); - } - ); + BufferRight.Unpack(Vs, BX1s, std::make_pair(ibeg, iend+1), + std::make_pair(jbeg+offset , jend+offset), + std::make_pair(kbeg , kend)); + #if DIMENSIONS >= 2 + BufferLeft.Unpack(Vs, BX2s, std::make_pair(ibeg, iend), + std::make_pair(jbeg , jend), + std::make_pair(kbeg , kend)); + BufferRight.Unpack(Vs, BX2s, std::make_pair(ibeg, iend), + std::make_pair(jbeg+offset+1, jend+offset+1), + std::make_pair(kbeg , kend)); + #endif #if DIMENSIONS == 3 - VsIndex = mapNVars*nx*ny*nz + (nx+1)*ny*nz; + BufferLeft.Unpack(Vs, BX3s, std::make_pair(ibeg, iend), + std::make_pair(jbeg, jend), + std::make_pair(kbeg, kend+1)); - idefix_for("StoreBufferX2KDIR",kbeg,kend+1,jbeg,jend,ibeg,iend, - KOKKOS_LAMBDA ( int k, int j, int i) { - Vs(KDIR,k,j,i) = BufferLeft(i + (j-jbeg)*nx + (k-kbeg)*nx*ny + VsIndex ); - Vs(KDIR,k,j+offset,i) = BufferRight(i + (j-jbeg)*nx + (k-kbeg)*nx*ny + VsIndex ); - } - ); + BufferRight.Unpack(Vs, BX3s,std::make_pair(ibeg , iend), + std::make_pair(jbeg+offset, jend+offset), + std::make_pair(kbeg , kend+1)); #endif } @@ -596,10 +642,9 @@ void Mpi::ExchangeX3(IdefixArray4D Vc, IdefixArray4D Vs) { // Load the buffers with data - int ibeg,iend,jbeg,jend,kbeg,kend,offset; - int nx,ny,nz; - IdefixArray1D BufferLeft=BufferSendX3[faceLeft]; - IdefixArray1D BufferRight=BufferSendX3[faceRight]; + int ibeg,iend,jbeg,jend,kbeg,kend,offset,nz; + Buffer BufferLeft=BufferSendX3[faceLeft]; + Buffer BufferRight=BufferSendX3[faceRight]; IdefixArray1D map = this->mapVars; // If MPI Persistent, start receiving even before the buffers are filled @@ -617,43 +662,56 @@ void Mpi::ExchangeX3(IdefixArray4D Vc, IdefixArray4D Vs) { // Coordinates of the ghost region which needs to be transfered ibeg = 0; iend = ntot[IDIR]; - nx = ntot[IDIR]; // Number of points in x + jbeg = 0; jend = ntot[JDIR]; - ny = ntot[JDIR]; + kbeg = 0; kend = nghost[KDIR]; offset = end[KDIR]; // Distance between beginning of left and right ghosts nz = nghost[KDIR]; - idefix_for("LoadBufferX3Vc",0,mapNVars,kbeg,kend,jbeg,jend,ibeg,iend, - KOKKOS_LAMBDA (int n, int k, int j, int i) { - int nt = map(n); - BufferLeft((i-ibeg) + (j-jbeg)*nx + (k-kbeg)*nx*ny + n*nx*ny*nz ) = Vc(nt,k+nz,j,i); - BufferRight((i-ibeg) + (j-jbeg)*nx + (k-kbeg)*nx*ny + n*nx*ny*nz ) = Vc(nt,k+offset-nz,j,i); - } - ); + BufferLeft.ResetPointer(); + BufferRight.ResetPointer(); + + BufferLeft.Pack(Vc, map, std::make_pair(ibeg , iend), + std::make_pair(jbeg , jend), + std::make_pair(kbeg+nz, kend+nz)); + + BufferRight.Pack(Vc, map, std::make_pair(ibeg , iend), + std::make_pair(jbeg , jend), + std::make_pair(kbeg + offset-nz, kend+ offset-nz)); // Load face-centered field in the buffer if(haveVs) { - int VsIndex = mapNVars*nx*ny*nz; + BufferLeft.Pack(Vs, BX1s,std::make_pair(ibeg , iend+1), + std::make_pair(jbeg , jend), + std::make_pair(kbeg+nz , kend+nz)); - idefix_for("LoadBufferX3IDIR",kbeg,kend,jbeg,jend,ibeg,iend+1, - KOKKOS_LAMBDA (int k, int j, int i) { - BufferLeft(i + (j-jbeg)*(nx+1) + (k-kbeg)*(nx+1)*ny + VsIndex ) = Vs(IDIR,k+nz,j,i); - BufferRight(i + (j-jbeg)*(nx+1) + (k-kbeg)*(nx+1)*ny + VsIndex ) = Vs(IDIR,k+offset-nz,j,i); - } - ); + BufferRight.Pack(Vs, BX1s, std::make_pair(ibeg , iend+1), + std::make_pair(jbeg , jend), + std::make_pair(kbeg + offset-nz, kend+ offset-nz)); #if DIMENSIONS >= 2 - VsIndex = mapNVars*nx*ny*nz + (nx+1)*ny*nz; - idefix_for("LoadBufferX3JDIR",kbeg,kend,jbeg,jend+1,ibeg,iend, - KOKKOS_LAMBDA (int k, int j, int i) { - BufferLeft(i + (j-jbeg)*nx + (k-kbeg)*nx*(ny+1) + VsIndex ) = Vs(JDIR,k+nz,j,i); - BufferRight(i + (j-jbeg)*nx + (k-kbeg)*nx*(ny+1) + VsIndex ) = Vs(JDIR,k+offset-nz,j,i); - } - ); + BufferLeft.Pack(Vs, BX2s,std::make_pair(ibeg , iend), + std::make_pair(jbeg , jend+1), + std::make_pair(kbeg+nz , kend+nz)); + + BufferRight.Pack(Vs, BX2s, std::make_pair(ibeg , iend), + std::make_pair(jbeg , jend+1), + std::make_pair(kbeg + offset-nz, kend+ offset-nz)); + + #endif + + #if DIMENSIONS == 3 + BufferLeft.Pack(Vs, BX3s,std::make_pair(ibeg , iend), + std::make_pair(jbeg , jend), + std::make_pair(kbeg+nz+1 , kend+nz+1)); + + BufferRight.Pack(Vs, BX3s, std::make_pair(ibeg , iend), + std::make_pair(jbeg , jend), + std::make_pair(kbeg + offset-nz, kend+ offset-nz)); #endif } @@ -722,34 +780,47 @@ void Mpi::ExchangeX3(IdefixArray4D Vc, IdefixArray4D Vs) { BufferLeft=BufferRecvX3[faceLeft]; BufferRight=BufferRecvX3[faceRight]; + BufferLeft.ResetPointer(); + BufferRight.ResetPointer(); + + // We fill the ghost zones - idefix_for("StoreBufferX3Vc",0,mapNVars,kbeg,kend,jbeg,jend,ibeg,iend, - KOKKOS_LAMBDA (int n, int k, int j, int i) { - Vc(map(n),k,j,i) = BufferLeft((i-ibeg) + (j-jbeg)*nx + (k-kbeg)*nx*ny + n*nx*ny*nz ); - Vc(map(n),k+offset,j,i) = BufferRight((i-ibeg) + (j-jbeg)*nx + (k-kbeg)*nx*ny + n*nx*ny*nz ); - } - ); + BufferLeft.Unpack(Vc, map,std::make_pair(ibeg, iend), + std::make_pair(jbeg , jend), + std::make_pair(kbeg , kend)); - // Load face-centered field in the buffer - if(haveVs) { - int VsIndex = mapNVars*nx*ny*nz; + BufferRight.Unpack(Vc, map,std::make_pair(ibeg , iend), + std::make_pair(jbeg , jend), + std::make_pair(kbeg+offset , kend+offset)); + // We fill the ghost zones - idefix_for("StoreBufferX3IDIR",kbeg,kend,jbeg,jend,ibeg,iend+1, - KOKKOS_LAMBDA (int k, int j, int i) { - Vs(IDIR,k,j,i) = BufferLeft(i + (j-jbeg)*(nx+1) + (k-kbeg)*(nx+1)*ny + VsIndex ); - Vs(IDIR,k+offset,j,i) = BufferRight(i + (j-jbeg)*(nx+1) + (k-kbeg)*(nx+1)*ny + VsIndex ); - } - ); + if(haveVs) { + BufferLeft.Unpack(Vs, BX1s, std::make_pair(ibeg, iend+1), + std::make_pair(jbeg , jend), + std::make_pair(kbeg , kend)); + + BufferRight.Unpack(Vs, BX1s, std::make_pair(ibeg, iend+1), + std::make_pair(jbeg , jend), + std::make_pair(kbeg+offset , kend+offset)); + + #if DIMENSIONS >=2 + BufferLeft.Unpack(Vs, BX2s, std::make_pair(ibeg, iend), + std::make_pair(jbeg, jend+1), + std::make_pair(kbeg, kend)); + + BufferRight.Unpack(Vs, BX2s,std::make_pair(ibeg , iend), + std::make_pair(jbeg , jend+1), + std::make_pair(kbeg+offset, kend+offset)); + #endif - #if DIMENSIONS >= 2 - VsIndex = mapNVars*nx*ny*nz + (nx+1)*ny*nz; + #if DIMENSIONS == 3 + BufferLeft.Unpack(Vs, BX3s, std::make_pair(ibeg, iend), + std::make_pair(jbeg, jend), + std::make_pair(kbeg, kend)); - idefix_for("StoreBufferX3JDIR",kbeg,kend,jbeg,jend+1,ibeg,iend, - KOKKOS_LAMBDA (int k, int j, int i) { - Vs(JDIR,k,j,i) = BufferLeft(i + (j-jbeg)*nx + (k-kbeg)*nx*(ny+1) + VsIndex ); - Vs(JDIR,k+offset,j,i) = BufferRight(i + (j-jbeg)*nx + (k-kbeg)*nx*(ny+1) + VsIndex ); - } - ); + BufferRight.Unpack(Vs, BX3s,std::make_pair(ibeg , iend), + std::make_pair(jbeg , jend), + std::make_pair(kbeg+offset+1, kend+offset+1)); #endif } @@ -763,7 +834,7 @@ void Mpi::ExchangeX3(IdefixArray4D Vc, IdefixArray4D Vs) { MPI_Waitall(2, sendRequestX3, sendStatus); #endif myTimer += MPI_Wtime(); - bytesSentOrReceived += 4*bufferSizeX2*sizeof(real); + bytesSentOrReceived += 4*bufferSizeX3*sizeof(real); idfx::popRegion(); } diff --git a/src/mpi.hpp b/src/mpi.hpp index f33880de..03aceb03 100644 --- a/src/mpi.hpp +++ b/src/mpi.hpp @@ -10,11 +10,176 @@ #include #include +#include #include "idefix.hpp" #include "grid.hpp" class DataBlock; +class Buffer { + public: + Buffer() = default; + explicit Buffer(size_t size): pointer{0}, array{IdefixArray1D("BufferArray",size)} { }; + + void* data() { + return(array.data()); + } + + int Size() { + return(array.size()); + } + + void ResetPointer() { + this->pointer = 0; + } + + void Pack(IdefixArray3D& in, + std::pair ib, + std::pair jb, + std::pair kb) { + const int ni = ib.second-ib.first; + const int ninj = (jb.second-jb.first)*ni; + const int ninjnk = (kb.second-kb.first)*ninj; + const int ibeg = ib.first; + const int jbeg = jb.first; + const int kbeg = kb.first; + const int offset = this->pointer; + + auto arr = this->array; + idefix_for("LoadBuffer3D",kb.first,kb.second,jb.first,jb.second,ib.first,ib.second, + KOKKOS_LAMBDA (int k, int j, int i) { + arr(i-ibeg + (j-jbeg)*ni + (k-kbeg)*ninj + offset ) = in(k,j,i); + }); + + // Update pointer + this->pointer += ninjnk; + } + + void Pack(IdefixArray4D& in, + const int var, + std::pair ib, + std::pair jb, + std::pair kb) { + const int ni = ib.second-ib.first; + const int ninj = (jb.second-jb.first)*ni; + const int ninjnk = (kb.second-kb.first)*ninj; + const int ibeg = ib.first; + const int jbeg = jb.first; + const int kbeg = kb.first; + const int offset = this->pointer; + + auto arr = this->array; + idefix_for("LoadBuffer4D",kb.first,kb.second,jb.first,jb.second,ib.first,ib.second, + KOKKOS_LAMBDA (int k, int j, int i) { + arr(i-ibeg + (j-jbeg)*ni + (k-kbeg)*ninj + offset ) = in(var, k,j,i); + }); + + // Update pointer + this->pointer += ninjnk; + } + + void Pack(IdefixArray4D& in, + IdefixArray1D& map, + std::pair ib, + std::pair jb, + std::pair kb) { + const int ni = ib.second-ib.first; + const int ninj = (jb.second-jb.first)*ni; + const int ninjnk = (kb.second-kb.first)*ninj; + const int ibeg = ib.first; + const int jbeg = jb.first; + const int kbeg = kb.first; + const int offset = this->pointer; + auto arr = this->array; + + idefix_for("LoadBuffer4D",0,map.size(), + kb.first,kb.second, + jb.first,jb.second, + ib.first,ib.second, + KOKKOS_LAMBDA (int n, int k, int j, int i) { + arr(i-ibeg + (j-jbeg)*ni + (k-kbeg)*ninj + n*ninjnk + offset ) = in(map(n), k,j,i); + }); + + // Update pointer + this->pointer += ninjnk*map.size(); + } + + void Unpack(IdefixArray3D& out, + std::pair ib, + std::pair jb, + std::pair kb) { + const int ni = ib.second-ib.first; + const int ninj = (jb.second-jb.first)*ni; + const int ninjnk = (kb.second-kb.first)*ninj; + const int ibeg = ib.first; + const int jbeg = jb.first; + const int kbeg = kb.first; + const int offset = this->pointer; + auto arr = this->array; + + idefix_for("LoadBuffer3D",kb.first,kb.second,jb.first,jb.second,ib.first,ib.second, + KOKKOS_LAMBDA (int k, int j, int i) { + out(k,j,i) = arr(i-ibeg + (j-jbeg)*ni + (k-kbeg)*ninj + offset ); + }); + + // Update pointer + this->pointer += ninjnk; + } + + void Unpack(IdefixArray4D& out, + const int var, + std::pair ib, + std::pair jb, + std::pair kb) { + const int ni = ib.second-ib.first; + const int ninj = (jb.second-jb.first)*ni; + const int ninjnk = (kb.second-kb.first)*ninj; + const int ibeg = ib.first; + const int jbeg = jb.first; + const int kbeg = kb.first; + const int offset = this->pointer; + + auto arr = this->array; + idefix_for("LoadBuffer3D",kb.first,kb.second,jb.first,jb.second,ib.first,ib.second, + KOKKOS_LAMBDA (int k, int j, int i) { + out(var,k,j,i) = arr(i-ibeg + (j-jbeg)*ni + (k-kbeg)*ninj + offset ); + }); + + // Update pointer + this->pointer += ninjnk; + } + + void Unpack(IdefixArray4D& out, + IdefixArray1D& map, + std::pair ib, + std::pair jb, + std::pair kb) { + const int ni = ib.second-ib.first; + const int ninj = (jb.second-jb.first)*ni; + const int ninjnk = (kb.second-kb.first)*ninj; + const int ibeg = ib.first; + const int jbeg = jb.first; + const int kbeg = kb.first; + const int offset = this->pointer; + + auto arr = this->array; + idefix_for("LoadBuffer4D",0,map.size(), + kb.first,kb.second, + jb.first,jb.second, + ib.first,ib.second, + KOKKOS_LAMBDA (int n, int k, int j, int i) { + out(map(n),k,j,i) = arr(i-ibeg + (j-jbeg)*ni + (k-kbeg)*ninj + n*ninjnk + offset ); + }); + + // Update pointer + this->pointer += ninjnk*map.size(); + } + + + private: + size_t pointer; + IdefixArray1D array; +}; class Mpi { public: @@ -54,12 +219,12 @@ class Mpi { enum {faceRight, faceLeft}; // Buffers for MPI calls - IdefixArray1D BufferSendX1[2]; - IdefixArray1D BufferSendX2[2]; - IdefixArray1D BufferSendX3[2]; - IdefixArray1D BufferRecvX1[2]; - IdefixArray1D BufferRecvX2[2]; - IdefixArray1D BufferRecvX3[2]; + Buffer BufferSendX1[2]; + Buffer BufferSendX2[2]; + Buffer BufferSendX3[2]; + Buffer BufferRecvX1[2]; + Buffer BufferRecvX2[2]; + Buffer BufferRecvX3[2]; IdefixArray1D mapVars; int mapNVars{0}; diff --git a/src/output/CMakeLists.txt b/src/output/CMakeLists.txt index af581ec8..8c54eaeb 100644 --- a/src/output/CMakeLists.txt +++ b/src/output/CMakeLists.txt @@ -1,8 +1,11 @@ target_sources(idefix + PUBLIC ${CMAKE_CURRENT_LIST_DIR}/slice.cpp + PUBLIC ${CMAKE_CURRENT_LIST_DIR}/slice.hpp PUBLIC ${CMAKE_CURRENT_LIST_DIR}/dump.cpp PUBLIC ${CMAKE_CURRENT_LIST_DIR}/dump.hpp PUBLIC ${CMAKE_CURRENT_LIST_DIR}/output.cpp PUBLIC ${CMAKE_CURRENT_LIST_DIR}/output.hpp + PUBLIC ${CMAKE_CURRENT_LIST_DIR}/scalarField.hpp PUBLIC ${CMAKE_CURRENT_LIST_DIR}/vtk.cpp PUBLIC ${CMAKE_CURRENT_LIST_DIR}/vtk.hpp ) diff --git a/src/output/dump.cpp b/src/output/dump.cpp index 4c3b7c75..9bd10164 100644 --- a/src/output/dump.cpp +++ b/src/output/dump.cpp @@ -5,33 +5,87 @@ // Licensed under CeCILL 2.1 License, see COPYING for more information // *********************************************************************************** +#include +#include +#include #include "dump.hpp" -#include "gitversion.hpp" +#include "version.hpp" #include "dataBlockHost.hpp" #include "gridHost.hpp" #include "output.hpp" +#include "fluid.hpp" // Max size of array name #define NAMESIZE 16 #define FILENAMESIZE 256 #define HEADERSIZE 128 -void Dump::Init(Input &input, DataBlock &data) { - // Init the output period +// Register a variable to be dumped (and read) +void Dump::RegisterVariable(IdefixArray3D& in, + std::string name, + int dir, + DumpField::ArrayLocation loc) { + dumpFieldMap.emplace(name, DumpField(in, loc, dir)); +} + +void Dump::RegisterVariable(IdefixHostArray3D& in, + std::string name, + int dir, + DumpField::ArrayLocation loc) { + dumpFieldMap.emplace(name, DumpField(in, loc, dir)); +} + +void Dump::RegisterVariable(IdefixArray4D& in, + std::string name, + int varnum, + int dir, + DumpField::ArrayLocation loc) { + dumpFieldMap.emplace(name, DumpField(in, varnum, loc, dir)); +} + +void Dump::RegisterVariable(IdefixHostArray4D& in, + std::string name, + int varnum, + int dir, + DumpField::ArrayLocation loc) { + dumpFieldMap.emplace(name, DumpField(in, varnum, loc, dir)); +} + + +void Dump::Init(DataBlock *datain) { + idfx::pushRegion("Dump::Init"); + this->data = datain; for (int dir=0; dir<3; dir++) { - this->periodicity[dir] = (data.mygrid->lbound[dir] == periodic); + this->periodicity[dir] = (data->mygrid->lbound[dir] == periodic); } this->dumpFileNumber = 0; + if(idfx::prank==0) { + if(!std::filesystem::is_directory(outputDirectory)) { + try { + if(!std::filesystem::create_directory(outputDirectory)) { + std::stringstream msg; + msg << "Cannot create directory " << outputDirectory << std::endl; + IDEFIX_ERROR(msg); + } + } catch(std::exception &e) { + std::stringstream msg; + msg << "Cannot create directory " << outputDirectory << std::endl; + msg << e.what(); + IDEFIX_ERROR(msg); + } + } + } + // Allocate scratch Array - this->scrch = new real[ (data.np_int[IDIR]+IOFFSET)* - (data.np_int[JDIR]+JOFFSET)* - (data.np_int[KDIR]+KOFFSET)]; + this->scrch = new real[ (data->np_int[IDIR]+IOFFSET)* + (data->np_int[JDIR]+JOFFSET)* + (data->np_int[KDIR]+KOFFSET)]; #ifdef WITH_MPI - Grid *grid = data.mygrid; + Grid *grid = data->mygrid; int start[3]; int size[3]; @@ -40,8 +94,8 @@ void Dump::Init(Input &input, DataBlock &data) { // Dimensions for cell-centered fields for(int dir = 0; dir < 3 ; dir++) { size[2-dir] = grid->np_int[dir]; - start[2-dir] = data.gbeg[dir]-data.nghost[dir]; - subsize[2-dir] = data.np_int[dir]; + start[2-dir] = data->gbeg[dir]-data->nghost[dir]; + subsize[2-dir] = data->np_int[dir]; } MPI_SAFE_CALL(MPI_Type_create_subarray(3, size, subsize, start, @@ -52,8 +106,8 @@ void Dump::Init(Input &input, DataBlock &data) { for(int face = 0; face < 3 ; face++) { for(int dir = 0; dir < 3 ; dir++) { size[2-dir] = grid->np_int[dir]; - start[2-dir] = data.gbeg[dir]-data.nghost[dir]; - subsize[2-dir] = data.np_int[dir]; + start[2-dir] = data->gbeg[dir]-data->nghost[dir]; + subsize[2-dir] = data->np_int[dir]; } // Add the extra guy in the face direction size[2-face]++; @@ -71,57 +125,74 @@ void Dump::Init(Input &input, DataBlock &data) { MPI_SAFE_CALL(MPI_Type_commit(&this->descSW[face])); } // Dimensions for edge-centered field - #ifdef EVOLVE_VECTOR_POTENTIAL - for(int nv = 0; nv <= AX3e ; nv++) { - int edge; // Vector direction(=edge) - - // Map nv to a vector direction - #if DIMENSIONS == 2 - if(nv==AX3e) { - edge = KDIR; - } else { - IDEFIX_ERROR("Wrong direction for vector potential"); - } - #elif DIMENSIONS == 3 - edge = nv; - #else - IDEFIX_ERROR("Cannot treat vector potential with that number of dimensions"); - #endif - - // load the array size - for(int dir = 0; dir < 3 ; dir++) { - size[2-dir] = grid->np_int[dir]; - start[2-dir] = data.gbeg[dir]-data.nghost[dir]; - subsize[2-dir] = data.np_int[dir]; - } + for(int nv = 0; nv < 3 ; nv++) { + // load the array size + for(int dir = 0; dir < 3 ; dir++) { + size[2-dir] = grid->np_int[dir]; + start[2-dir] = data->gbeg[dir]-data->nghost[dir]; + subsize[2-dir] = data->np_int[dir]; + } - // Extra cell in the dirs perp to field - for(int i = 0 ; i < DIMENSIONS ; i++) { - if(i!=edge) { - size[2-i]++; - subsize[2-i]++; // valid only for reading - //since it involves an overlap of data between procs - } + // Extra cell in the dirs perp to field + for(int i = 0 ; i < DIMENSIONS ; i++) { + if(i!=nv) { + size[2-i]++; + subsize[2-i]++; // valid only for reading + //since it involves an overlap of data between procs } - MPI_SAFE_CALL(MPI_Type_create_subarray(3, size, subsize, start, - MPI_ORDER_C, realMPI, &this->descER[nv])); - MPI_SAFE_CALL(MPI_Type_commit(&this->descER[nv])); - - // Now for writing, it is only the last proc which keeps one additional cell, - // so we remove what we added for reads - for(int i = 0 ; i < DIMENSIONS ; i++) { - if(i!=edge) { - if(grid->xproc[i] != grid->nproc[i] - 1 ) { - subsize[2-i]--; - } + } + MPI_SAFE_CALL(MPI_Type_create_subarray(3, size, subsize, start, + MPI_ORDER_C, realMPI, &this->descER[nv])); + MPI_SAFE_CALL(MPI_Type_commit(&this->descER[nv])); + + // Now for writing, it is only the last proc which keeps one additional cell, + // so we remove what we added for reads + for(int i = 0 ; i < DIMENSIONS ; i++) { + if(i!=nv) { + if(grid->xproc[i] != grid->nproc[i] - 1 ) { + subsize[2-i]--; } } - MPI_SAFE_CALL(MPI_Type_create_subarray(3, size, subsize, start, - MPI_ORDER_C, realMPI, &this->descEW[nv])); - MPI_SAFE_CALL(MPI_Type_commit(&this->descEW[nv])); } - #endif + MPI_SAFE_CALL(MPI_Type_create_subarray(3, size, subsize, start, + MPI_ORDER_C, realMPI, &this->descEW[nv])); + MPI_SAFE_CALL(MPI_Type_commit(&this->descEW[nv])); + } + #endif + + // Register variables that are needed in restart dumps + this->RegisterVariable(&dumpFileNumber, "dumpFileNumber"); + this->RegisterVariable(&geometry, "geometry"); + this->RegisterVariable(periodicity, "periodicity", 3); + + idfx::popRegion(); +} + + +Dump::Dump(Input &input, DataBlock *datain) { + // Constructor with an input object, in which case, + // We use the outputdirectory provided in the input + + // initialize output path + if(input.CheckEntry("Output","dmp_dir")>=0) { + outputDirectory = input.Get("Output","dmp_dir",0); + } else { + outputDirectory = "./"; + } + Init(datain); +} + +Dump::Dump(DataBlock *datain) { + // Constructor without an input object, in which case, + // We use the default output directory + + outputDirectory = "./"; + Init(datain); +} + +Dump::~Dump() { + delete scrch; } void Dump::WriteString(IdfxFileHandler fileHdl, char *str, int size) { @@ -450,8 +521,49 @@ void Dump::ReadDistributed(IdfxFileHandler fileHdl, int ndim, int *dim, int *gdi #endif } -int Dump::Read(DataBlock &data, Output& output, int readNumber ) { - char filename[FILENAMESIZE]; +// Helper function to convert filesystem::file_time into std::time_t +// see https://stackoverflow.com/questions/56788745/ +// This conversion "hack" is required in C++17 as no proper conversion bewteen +// std::filesystem::last_write_time and std::time_t +// exists in the standard library until C++20 +template +std::time_t to_time_t(TP tp) { + auto sctp = std::chrono::time_point_cast + (tp - TP::clock::now() + std::chrono::system_clock::now()); + return std::chrono::system_clock::to_time_t(sctp); +} + +int Dump::GetLastDumpInDirectory(std::filesystem::path &directory) { + int num = -1; + + std::time_t youngFileTime; + bool first = true; + for (const auto & entry : std::filesystem::directory_iterator(directory)) { + // Check file extension + if(entry.path().extension().string().compare(".dmp")==0) { + auto fileTime = to_time_t(std::filesystem::last_write_time(entry.path())); + // Check which one is the most recent + if(first || fileTime>youngFileTime) { + // std::tm *gmt = std::gmtime(&fileTime); + // idfx::cout << "file " << entry.path() << "is the most recent with " + // << std::put_time(gmt, "%d %B %Y %H:%M:%S") << std::endl; + + // Ours is more recent, extract the dump file number + try { + num = std::stoi(entry.path().filename().string().substr(5,4)); + first = false; + youngFileTime = fileTime; + } catch (...) { + // the file name does not follow the convention "filebase.xxxx.dmp" + // ->We skip it + } + } + } + } + return(num); +} +bool Dump::Read(Output& output, int readNumber ) { + std::filesystem::path filename; int nx[3]; int nxglob[3]; std::string fieldName; @@ -462,24 +574,43 @@ int Dump::Read(DataBlock &data, Output& output, int readNumber ) { idfx::pushRegion("Dump::Read"); - idfx::cout << "Dump: Reading restart file n " << readNumber << "..." << std::flush; + std::filesystem::path readDir = this->outputDirectory; + + if(readNumber<0) { + // We actually don't know which file we're supposed to read, so let's guess + readNumber = GetLastDumpInDirectory(readDir); + if(readNumber < 0) { + idfx::cout << "Dump: cannot find a valid dump in " << this->outputDirectory << std::endl; + if(outputDirectory.compare("./")!=0) { + idfx::cout << "Dump: reverting to the current directory." << std::endl; + readDir = "."; + readNumber = GetLastDumpInDirectory(readDir); + } + if(readNumber<0) { + IDEFIX_WARNING("cannot find a valid restart dump."); + return(false); + } + } + } // Reset timer timer.reset(); // Set filename - std::snprintf (filename, FILENAMESIZE, "dump.%04d.dmp", readNumber); - - // Make a local image of the datablock - DataBlockHost dataHost(data); + std::stringstream ssdumpFileNum,ssFileName; + ssdumpFileNum << std::setfill('0') << std::setw(4) << readNumber; + ssFileName << "dump." << ssdumpFileNum.str() << ".dmp"; + filename = readDir/ssFileName.str(); + idfx::cout << "Dump: Reading " << filename << "..." << std::flush; // open file #ifdef WITH_MPI - MPI_SAFE_CALL(MPI_File_open(MPI_COMM_WORLD, filename, MPI_MODE_RDONLY | MPI_MODE_UNIQUE_OPEN, + MPI_SAFE_CALL(MPI_File_open(MPI_COMM_WORLD, filename.c_str(), + MPI_MODE_RDONLY | MPI_MODE_UNIQUE_OPEN, MPI_INFO_NULL, &fileHdl)); this->offset = 0; #else - fileHdl = fopen(filename,"rb"); + fileHdl = fopen(filename.c_str(),"rb"); if(fileHdl == NULL) { std::stringstream msg; msg << "Failed to open dump file: " << std::string(filename) << std::endl; @@ -499,7 +630,7 @@ int Dump::Read(DataBlock &data, Output& output, int readNumber ) { for(int dir=0 ; dir < 3; dir++) { ReadNextFieldProperties(fileHdl, ndim, nx, type, fieldName); if(ndim>1) IDEFIX_ERROR("Wrong coordinate array dimensions while reading restart dump"); - if(nx[0] != data.mygrid->np_int[dir]) { + if(nx[0] != data->mygrid->np_int[dir]) { idfx::cout << "dir " << dir << ", restart has " << nx[0] << " points " << std::endl; IDEFIX_ERROR("Domain size from the restart dump is different from the current one"); } @@ -514,167 +645,108 @@ int Dump::Read(DataBlock &data, Output& output, int readNumber ) { } // Todo: check that coordinates are identical } + std::unordered_set notFound {}; + for(auto it = dumpFieldMap.begin(); it != dumpFieldMap.end(); it++) { + notFound.insert(it->first); + } + // Coordinates are ok, load the bulk while(true) { ReadNextFieldProperties(fileHdl, ndim, nxglob, type, fieldName); - idfx::cout << "Next field is " << fieldName << " with " << ndim << " dimensions and ("; + + /*idfx::cout << "Next field is " << fieldName << " with " << ndim << " dimensions and ("; for(int i = 0 ; i < ndim ; i++) idfx::cout << nxglob[i] << " "; - idfx::cout << ") points." << std::endl; + idfx::cout << ") points." << std::endl;*/ if(fieldName.compare(eof) == 0) { + // We have reached end of dump file break; - } else if(fieldName.compare(0,3,"Vc-") == 0) { - // Next field is a cell-centered field - - // Matching Name is Vc-<> - int nv = -1; - for(int n = 0 ; n < NVAR; n++) { - if(fieldName.compare(3,3,data.hydro.VcName[n],0,3)==0) nv=n; // Found matching field - } - // Load it - for(int dir = 0 ; dir < 3; dir++) { - nx[dir] = dataHost.np_int[dir]; - } - ReadDistributed(fileHdl, ndim, nx, nxglob, descC, scrch); + } else { + if(auto it = dumpFieldMap.find(fieldName) ; it != dumpFieldMap.end()) { + // This key has been registered + notFound.erase(fieldName); + DumpField &scalar = it->second; + if(scalar.GetType() == DumpField::Type::IdefixArray) { + // Distributed idefix array + int direction = scalar.GetDirection(); - if(nv<0) { - IDEFIX_WARNING("Cannot find a field matching " + fieldName - + " in current running code. Skipping."); - } else { - // Load the scratch space in designated field - for(int k = 0; k < nx[KDIR]; k++) { - for(int j = 0 ; j < nx[JDIR]; j++) { - for(int i = 0; i < nx[IDIR]; i++) { - dataHost.Vc(nv,k+dataHost.beg[KDIR],j+dataHost.beg[JDIR],i+dataHost.beg[IDIR]) = - scrch[i + j*nx[IDIR] + k*nx[IDIR]*nx[JDIR]]; - } - } - } - } - } else if(fieldName.compare(0,3,"Vs-") == 0) { - // Next field is a face-centered field - - // Matching Name is Vs-<> - #if MHD == YES - int nv = -1; - for(int n = 0 ; n < DIMENSIONS; n++) { - if(fieldName.compare(3,4,data.hydro.VsName[n],0,4)==0) nv=n; // Found matching field - } - if(nv<0) { - IDEFIX_ERROR("Cannot find a field matching " + fieldName - + " in current running code."); - } else { // Load it - for(int dir = 0 ; dir < 3; dir++) nx[dir] = dataHost.np_int[dir]; - nx[nv]++; // Extra cell in the dir direction for cell-centered fields - ReadDistributed(fileHdl, ndim, nx, nxglob, descSR[nv], scrch); + for(int dir = 0 ; dir < 3; dir++) { + nx[dir] = data->np_int[dir]; + } - for(int k = 0; k < nx[KDIR]; k++) { - for(int j = 0 ; j < nx[JDIR]; j++) { - for(int i = 0; i < nx[IDIR]; i++) { - dataHost.Vs(nv,k+dataHost.beg[KDIR],j+dataHost.beg[JDIR],i+dataHost.beg[IDIR]) - = scrch[i + j*nx[IDIR] + k*nx[IDIR]*nx[JDIR]]; - } - } + if(scalar.GetLocation() == DumpField::ArrayLocation::Face) { + nx[direction]++; // Extra cell in the dir direction for face-centered fields } - } - #else - IDEFIX_WARNING("Code configured without MHD. Face-centered magnetic field components \ - from the restart dump are skipped"); - #endif - } else if(fieldName.compare(0,3,"Ve-") == 0) { - #if MHD == YES && defined(EVOLVE_VECTOR_POTENTIAL) - int nv = -1; - for(int n = 0 ; n <= AX3e; n++) { - if(fieldName.compare(3,4,data.hydro.VeName[n],0,4)==0) nv=n; // Found matching field - } - if(nv<0) { - IDEFIX_ERROR("Cannot find a field matching " + fieldName - + " in current running code."); - } else { - int dir = 0; - #if DIMENSIONS == 2 - if(nv==AX3e) { - dir = KDIR; - } else { - IDEFIX_ERROR("Wrong direction for vector potential"); + if(scalar.GetLocation() == DumpField::ArrayLocation::Edge) { + // Extra cell in the dirs perp to field + for(int i = 0 ; i < DIMENSIONS ; i++) { + if(i!=direction) nx[i] ++; } - #elif DIMENSIONS == 3 - dir = nv; - #else - IDEFIX_ERROR("Cannot treat vector potential with that number of dimensions"); - #endif - // Load it - for(int i = 0 ; i < 3; i++) { - nx[i] = dataHost.np_int[i]; } - // Extra cell in the dirs perp to field - for(int i = 0 ; i < DIMENSIONS ; i++) { - if(i!=dir) nx[i] ++; + if(scalar.GetLocation() == DumpField::ArrayLocation::Center) { + ReadDistributed(fileHdl, ndim, nx, nxglob, descC, scrch); + } else if(scalar.GetLocation() == DumpField::ArrayLocation::Face) { + ReadDistributed(fileHdl, ndim, nx, nxglob, descSR[direction], scrch); + } else if(scalar.GetLocation() == DumpField::ArrayLocation::Edge) { + ReadDistributed(fileHdl, ndim, nx, nxglob, descER[direction], scrch); } - - ReadDistributed(fileHdl, ndim, nx, nxglob, descER[nv], scrch); - + auto toRead = scalar.GetHostField>(); + // Load the scratch space in designated field for(int k = 0; k < nx[KDIR]; k++) { for(int j = 0 ; j < nx[JDIR]; j++) { for(int i = 0; i < nx[IDIR]; i++) { - dataHost.Ve(nv,k+dataHost.beg[KDIR],j+dataHost.beg[JDIR],i+dataHost.beg[IDIR]) - = scrch[i + j*nx[IDIR] + k*nx[IDIR]*nx[JDIR]]; + toRead(k+data->beg[KDIR],j+data->beg[JDIR],i+data->beg[IDIR]) = + scrch[i + j*nx[IDIR] + k*nx[IDIR]*nx[JDIR]]; } } } - } + scalar.SyncFrom(toRead); + } else { + // Fundamental Type + // Check that size matches + if(nxglob[0] != scalar.GetSize()) { + idfx::cout << "nxglob=" << nxglob[0] << " scalar=" << scalar.GetSize() << std::endl; + IDEFIX_ERROR("Size of field "+fieldName+" do not match"); + } + // Todo: check that type matches + void *ptr = scalar.GetHostField(); - #else - IDEFIX_WARNING("Code configured without vector potential support. Vector potentials \ - from the restart dump are skipped"); - Skip(fileHdl, ndim, nxglob, type); - - #endif - } else if(fieldName.compare("time") == 0) { - ReadSerial(fileHdl, ndim, nxglob, type, &data.t); - } else if(fieldName.compare("dt") == 0) { - ReadSerial(fileHdl, ndim, nxglob, type, &data.dt); - } else if(fieldName.compare("vtkFileNumber")==0) { - ReadSerial(fileHdl, ndim, nxglob, type, &output.vtk.vtkFileNumber); - } else if(fieldName.compare("vtkLast")==0) { - ReadSerial(fileHdl, ndim, nxglob, type, &output.vtkLast); - } else if(fieldName.compare("dumpFileNumber")==0) { - ReadSerial(fileHdl, ndim, nxglob, type, &this->dumpFileNumber); - } else if(fieldName.compare("dumpLast")==0) { - ReadSerial(fileHdl, ndim, nxglob, type, &output.dumpLast); - } else if(fieldName.compare("analysisLast")==0) { - ReadSerial(fileHdl, ndim, nxglob, type, &output.analysisLast); - } else if(fieldName.compare("geometry")==0) { - ReadSerial(fileHdl, ndim, nxglob, type, &this->geometry); - } else if(fieldName.compare("periodicity")==0) { - ReadSerial(fileHdl, ndim, nxglob, type, &this->periodicity); - } else { - Skip(fileHdl, ndim, nxglob, type); - IDEFIX_WARNING("Unknown field "+fieldName+" in restart dump. Skipping."); + ReadSerial(fileHdl, ndim, nxglob, type, ptr); + } + } else { + Skip(fileHdl, ndim, nxglob, type); + // Key has not been registered, throw a warning + IDEFIX_WARNING("Cannot find a field matching " + fieldName + + " in current running code. Skipping."); + } } } - + if (notFound.size() > 0) { + std::stringstream msg {}; + msg << "The following fields were not found in " << filename << ": "; + for(auto it = notFound.begin(); it != notFound.end(); it++) { + msg << *it << ' '; + } + IDEFIX_WARNING(msg); + } #ifdef WITH_MPI MPI_SAFE_CALL(MPI_File_close(&fileHdl)); #else fclose(fileHdl); #endif - // Send to device - dataHost.SyncToDevice(); - idfx::cout << "done in " << timer.seconds() << " s." << std::endl; - idfx::cout << "Restarting from t=" << data.t << "." << std::endl; + idfx::cout << "Restarting from t=" << data->t << "." << std::endl; idfx::popRegion(); - return(0); + return(true); } -int Dump::Write(DataBlock &data, Output& output) { - char filename[FILENAMESIZE]; +int Dump::Write(Output& output) { + std::filesystem::path filename; char fieldName[NAMESIZE+1]; // +1 is just in case int nx[3]; int nxtot[3]; @@ -693,35 +765,37 @@ int Dump::Write(DataBlock &data, Output& output) { // Reset timer timer.reset(); - // Set filename - std::snprintf(filename, FILENAMESIZE, "dump.%04d.dmp", dumpFileNumber); + + // Set filenames + std::stringstream ssdumpFileNum,ssFileName; + ssdumpFileNum << std::setfill('0') << std::setw(4) << dumpFileNumber; + ssFileName << "dump." << ssdumpFileNum.str() << ".dmp"; + filename = outputDirectory/ssFileName.str(); + dumpFileNumber++; // For next one + // Check if file exists, if yes, delete it + if(idfx::prank==0) { + if(std::filesystem::exists(filename)) { + std::filesystem::remove(filename); + } + } + // open file #ifdef WITH_MPI -// Open file for creating, return error if file already exists. - int err = MPI_File_open(MPI_COMM_WORLD, filename, - MPI_MODE_CREATE | MPI_MODE_RDWR - | MPI_MODE_EXCL | MPI_MODE_UNIQUE_OPEN, - MPI_INFO_NULL, &fileHdl); - if (err != MPI_SUCCESS) { - // File exists, delete it before reopening - if(idfx::prank == 0) { - MPI_File_delete(filename,MPI_INFO_NULL); - } - MPI_SAFE_CALL(MPI_File_open(MPI_COMM_WORLD, filename, + MPI_Barrier(MPI_COMM_WORLD); + // Open file for creating, return error if file already exists. + MPI_SAFE_CALL(MPI_File_open(MPI_COMM_WORLD, filename.c_str(), MPI_MODE_CREATE | MPI_MODE_RDWR | MPI_MODE_EXCL | MPI_MODE_UNIQUE_OPEN, MPI_INFO_NULL, &fileHdl)); - } - this->offset = 0; #else - fileHdl = fopen(filename,"wb"); + fileHdl = fopen(filename.c_str(),"wb"); #endif // File is open // First thing we need are coordinates: init a host mirror and sync it - GridHost gridHost(*data.mygrid); + GridHost gridHost(*data->mygrid); gridHost.SyncFromDevice(); // Test endianness @@ -735,7 +809,8 @@ int Dump::Write(DataBlock &data, Output& output) { } char header[HEADERSIZE]; - std::snprintf(header, HEADERSIZE, "Idefix %s Dump Data %s endian", GITVERSION, endian.c_str()); + std::snprintf(header, HEADERSIZE, "Idefix %s Dump Data %s endian", + IDEFIX_VERSION, endian.c_str()); WriteString(fileHdl, header, HEADERSIZE); for(int dir = 0; dir < 3 ; dir++) { @@ -754,120 +829,70 @@ int Dump::Write(DataBlock &data, Output& output) { } // Then write raw data from Vc - DataBlockHost dataHost(data); - dataHost.SyncFromDevice(); - - for(int nv = 0 ; nv descC, scrch); - } - #if MHD == YES - // write staggered field components - for(int nv = 0 ; nv >(); + int dir = scalar.GetDirection(); for(int i = 0; i < 3 ; i++) { - nx[i] = dataHost.np_int[i]; + nx[i] = data->np_int[i]; nxtot[i] = gridHost.np_int[i]; } - // If it is the last datablock of the dimension, increase the size by one to get the last - //active face of the staggered mesh. - if(data.mygrid->xproc[nv] == data.mygrid->nproc[nv] - 1 ) nx[nv]++; - nxtot[nv]++; - for(int k = 0; k < nx[KDIR]; k++) { - for(int j = 0 ; j < nx[JDIR]; j++) { - for(int i = 0; i < nx[IDIR]; i++) { - scrch[i + j*nx[IDIR] + k*nx[IDIR]*nx[JDIR] ] = dataHost.Vs(nv,k+dataHost.beg[KDIR], - j+dataHost.beg[JDIR], - i+dataHost.beg[IDIR]); - } - } + if(scalar.GetLocation() == DumpField::ArrayLocation::Face) { + // If it is the last datablock of the dimension, increase the size by one to get the last + //active face of the staggered mesh. + if(data->mygrid->xproc[dir] == data->mygrid->nproc[dir] - 1 ) nx[dir]++; + nxtot[dir]++; } - WriteDistributed(fileHdl, 3, nx, nxtot, fieldName, this->descSW[nv], scrch); - } - #ifdef EVOLVE_VECTOR_POTENTIAL - // write edge field components - for(int nv = 0 ; nv <= AX3e ; nv++) { - std::snprintf(fieldName,NAMESIZE,"Ve-%s",data.hydro.VeName[nv].c_str()); - int edge = 0; - #if DIMENSIONS == 2 - if(nv==AX3e) { - edge = KDIR; - } else { - IDEFIX_ERROR("Wrong direction for vector potential"); - } - #elif DIMENSIONS == 3 - edge = nv; - #else - IDEFIX_ERROR("Cannot treat vector potential with that number of dimensions"); - #endif - // Load the active domain in the scratch space - for(int i = 0; i < 3 ; i++) { - nx[i] = dataHost.np_int[i]; - nxtot[i] = gridHost.np_int[i]; - } + + if(scalar.GetLocation() == DumpField::ArrayLocation::Edge) { // If it is the last datablock of the dimension, increase the size by one in the direction // perpendicular to the vector. - for(int i = 0 ; i < DIMENSIONS ; i++) { - if(i != edge) { - if(data.mygrid->xproc[i] == data.mygrid->nproc[i] - 1) nx[i]++; + if(i != dir) { + if(data->mygrid->xproc[i] == data->mygrid->nproc[i] - 1) nx[i]++; nxtot[i]++; } } - for(int k = 0; k < nx[KDIR]; k++) { - for(int j = 0 ; j < nx[JDIR]; j++) { - for(int i = 0; i < nx[IDIR]; i++) { - scrch[i + j*nx[IDIR] + k*nx[IDIR]*nx[JDIR] ] = dataHost.Ve(nv,k+dataHost.beg[KDIR], - j+dataHost.beg[JDIR], - i+dataHost.beg[IDIR]); - } + } + + // Load the dataset in the scratch array + for(int k = 0; k < nx[KDIR]; k++) { + for(int j = 0 ; j < nx[JDIR]; j++) { + for(int i = 0; i < nx[IDIR]; i++) { + scrch[i + j*nx[IDIR] + k*nx[IDIR]*nx[JDIR]] = toWrite(k+data->beg[KDIR], + j+data->beg[JDIR], + i+data->beg[IDIR]); } } - WriteDistributed(fileHdl, 3, nx, nxtot, fieldName, this->descEW[nv], scrch); } - #endif - #endif - // Write some raw data - nx[0] = 1; - std::snprintf(fieldName,NAMESIZE, "time"); - WriteSerial(fileHdl, 1, nx, realType, fieldName, &data.t); - std::snprintf(fieldName,NAMESIZE, "dt"); - WriteSerial(fileHdl, 1, nx, realType, fieldName, &data.dt); - std::snprintf(fieldName,NAMESIZE, "vtkFileNumber"); - WriteSerial(fileHdl, 1, nx, IntegerType, fieldName, &output.vtk.vtkFileNumber); - std::snprintf(fieldName,NAMESIZE, "vtkLast"); - WriteSerial(fileHdl, 1, nx, realType, fieldName, &output.vtkLast); - std::snprintf(fieldName,NAMESIZE, "dumpFileNumber"); - WriteSerial(fileHdl, 1, nx, IntegerType, fieldName, &this->dumpFileNumber); - std::snprintf(fieldName,NAMESIZE, "dumpLast"); - WriteSerial(fileHdl, 1, nx, realType, fieldName, &output.dumpLast); - std::snprintf(fieldName,NAMESIZE, "analysisLast"); - WriteSerial(fileHdl, 1, nx, realType, fieldName, &output.analysisLast); - std::snprintf(fieldName,NAMESIZE, "geometry"); - WriteSerial(fileHdl, 1, nx, IntegerType, fieldName, &this->geometry); - - nx[0] = 3; - std::snprintf(fieldName,NAMESIZE, "periodicity"); - WriteSerial(fileHdl, 1, nx, IntegerType, fieldName, &this->periodicity); + if(scalar.GetLocation() == DumpField::ArrayLocation::Center) { + WriteDistributed(fileHdl, 3, nx, nxtot, fieldName, this->descC, scrch); + } else if(scalar.GetLocation() == DumpField::ArrayLocation::Face) { + WriteDistributed(fileHdl, 3, nx, nxtot, fieldName, this->descSW[dir], scrch); + } else if(scalar.GetLocation() == DumpField::ArrayLocation::Edge) { + WriteDistributed(fileHdl, 3, nx, nxtot, fieldName, this->descEW[dir], scrch); + } else { + IDEFIX_ERROR("Unknown scalar type for dump write"); + } + + } else { + // Scalar type if a fundamental type, not distributed + DataType thisType; + if(scalar.GetType()==DumpField::Type::Int) thisType = DataType::IntegerType; + if(scalar.GetType()==DumpField::Type::Single) thisType = DataType::SingleType; + if(scalar.GetType()==DumpField::Type::Double) thisType = DataType::DoubleType; + if(scalar.GetType()==DumpField::Type::Bool) thisType = DataType::BoolType; + + nx[0] = scalar.GetSize(); + + WriteSerial(fileHdl, 1, nx, thisType, fieldName, scalar.GetHostField()); + } + } // Write end of file scrch[0] = 0.0; diff --git a/src/output/dump.hpp b/src/output/dump.hpp index 4bca76a0..657c9ed0 100644 --- a/src/output/dump.hpp +++ b/src/output/dump.hpp @@ -8,6 +8,9 @@ #ifndef OUTPUT_DUMP_HPP_ #define OUTPUT_DUMP_HPP_ #include +#include +#include + #include "idefix.hpp" #include "input.hpp" #include "dataBlock.hpp" @@ -25,23 +28,189 @@ enum DataType {DoubleType, SingleType, IntegerType, BoolType}; // Forward class declaration //class Vtk; class Output; +class DataBlock; + + +class DumpField { + public: + enum Type {Int, Single, Double, Bool, IdefixArray}; + enum ArrayType {Device3D, Device4D, Host3D, Host4D}; + enum ArrayLocation {Center, Face, Edge}; + + DumpField(IdefixArray4D& in, const int varnum, const ArrayLocation loc, const int dir): + d4Darray{in}, var{varnum}, arrayType{Device4D}, + type{IdefixArray}, arrayLocation{loc}, direction{dir} {}; + + DumpField(IdefixHostArray4D& in, const int varnum, const ArrayLocation loc, const int dir): + h4Darray{in}, var{varnum}, arrayType{Host4D}, + type{IdefixArray}, arrayLocation{loc}, direction{dir} {}; + + DumpField(IdefixArray3D& in, const ArrayLocation loc, const int dir): + d3Darray{in}, arrayType{Device3D}, + type{IdefixArray}, arrayLocation{loc}, direction{dir} {}; + + DumpField(IdefixHostArray3D& in, const ArrayLocation loc, const int dir): + h3Darray{in}, arrayType{Host3D}, + type{IdefixArray}, arrayLocation{loc}, direction{dir} {}; + + explicit DumpField(int * in, const int size = 1 ): + rawData{static_cast(in)}, rawSize{size}, type{Int} {}; + + explicit DumpField(float * in, const int size = 1 ): + rawData{static_cast(in)}, rawSize{size}, type{Single} {}; + + explicit DumpField(double * in, const int size = 1 ): + rawData{static_cast(in)}, rawSize{size}, type{Double} {}; + + explicit DumpField(bool * in, const int size = 1 ): + rawData{static_cast(in)}, rawSize{size}, type{Bool} {}; + + + + + template + T GetHostField() const { + // Check that return type is consistent with what is stored internally + if constexpr(std::is_same>::value) { + // User is expecting an IdefixArray, so let's convert what we have into an idefix array + if(arrayType==Host3D) { + return(h3Darray); + } else if(arrayType==Host4D) { + IdefixHostArray3D arr3D = Kokkos::subview( + h4Darray, var, Kokkos::ALL, Kokkos::ALL, Kokkos::ALL); + return(arr3D); + } else if(arrayType==Device3D) { + IdefixHostArray3D arr3D = Kokkos::create_mirror(d3Darray); + Kokkos::deep_copy(arr3D,d3Darray); + return(arr3D); + } else if(arrayType==Device4D) { + IdefixArray3D arrDev3D = Kokkos::subview( + d4Darray, var, Kokkos::ALL, Kokkos::ALL, Kokkos::ALL); + IdefixHostArray3D arr3D = Kokkos::create_mirror(arrDev3D); + Kokkos::deep_copy(arr3D,arrDev3D); + return(arr3D); + } else { + IDEFIX_ERROR("unknown field"); + return(h3Darray); + } + } else { + // If it's not a raw pointer, then it must be a simple scalar + return(static_cast(rawData)); + } + } + + // Synchronise field to Host + template + void SyncFrom(T in) const { + if constexpr(std::is_same>::value) { + if(arrayType==Host3D) { + Kokkos::deep_copy(h3Darray, in); + } else if(arrayType==Host4D) { + IdefixHostArray3D arr3D = Kokkos::subview( + h4Darray, var, Kokkos::ALL, Kokkos::ALL, Kokkos::ALL); + Kokkos::deep_copy(arr3D, in); + } else if(arrayType==Device3D) { + Kokkos::deep_copy(d3Darray,in); + } else if(arrayType==Device4D) { + IdefixArray3D arrDev3D = Kokkos::subview( + d4Darray, var, Kokkos::ALL, Kokkos::ALL, Kokkos::ALL); + Kokkos::deep_copy(arrDev3D,in); + } + } + // Nothing to sync otherwise + } + + Type GetType() const { + return type; + } + + ArrayType GetArrayType() const { + return(arrayType); + } + + ArrayLocation GetLocation() const { + return arrayLocation; + } + + int GetSize() const { + return(rawSize); + } + + int GetDirection() const { + return(direction); + } + + + + private: + IdefixArray4D d4Darray; + IdefixArray3D d3Darray; + IdefixHostArray4D h4Darray; + IdefixHostArray3D h3Darray; + + void *rawData; + int rawSize; + + int var; + int direction; + ArrayLocation arrayLocation; + ArrayType arrayType; + Type type; +}; class Dump { friend class DumpImage; // Allow dumpimag to have access to dump API public: - void Init(Input &, DataBlock &); // Create Dump Object + explicit Dump(Input &, DataBlock *); // Create Dump Object + explicit Dump(DataBlock *); // Create a dump object independent of input + ~Dump(); + // Create a Dump file from the current state of the code - int Write(DataBlock &, Output&); + int Write(Output&); // Read and load a dump file as current state of the code - int Read(DataBlock &, Output&, int); + bool Read(Output&, int); + + // Register IdefixArrays + void RegisterVariable(IdefixArray3D&, + std::string, + int dir = -1, + DumpField::ArrayLocation loc = DumpField::ArrayLocation::Center ); + + void RegisterVariable(IdefixHostArray3D&, + std::string, + int dir = -1, + DumpField::ArrayLocation loc = DumpField::ArrayLocation::Center ); + + void RegisterVariable(IdefixArray4D&, + std::string, + int varnum, + int dir = -1, + DumpField::ArrayLocation loc = DumpField::ArrayLocation::Center ); + + void RegisterVariable(IdefixHostArray4D&, + std::string, + int varnum, + int dir = -1, + DumpField::ArrayLocation loc = DumpField::ArrayLocation::Center ); + + // Register any other fundamental type + template + void RegisterVariable(T*, + std::string, + int size = 1); private: + void Init(DataBlock*); + DataBlock *data; int dumpFileNumber; int geometry{GEOMETRY}; int periodicity[3]; real *scrch; // Scratch array in host space + std::map dumpFieldMap; + + // Timer Kokkos::Timer timer; @@ -63,6 +232,24 @@ class Dump { void ReadSerial(IdfxFileHandler, int, int*, DataType, void*); void ReadDistributed(IdfxFileHandler, int, int*, int*, IdfxDataDescriptor&, void*); void Skip(IdfxFileHandler, int, int *, DataType); + int GetLastDumpInDirectory(std::filesystem::path &); + + std::filesystem::path outputDirectory; }; + + +template +void Dump::RegisterVariable(T* in, std::string name, int size) { + if(dumpFieldMap.count(name)>0) { + std::stringstream msg; + msg << "Error while registering variable in Dump I/O: " << std::endl + << name << " has already been registered." << std::endl; + IDEFIX_ERROR(msg); + } + dumpFieldMap.emplace(name, DumpField(in, size)); +} + + + #endif // OUTPUT_DUMP_HPP_ diff --git a/src/output/output.cpp b/src/output/output.cpp index bfa930b4..15f0bf8f 100644 --- a/src/output/output.cpp +++ b/src/output/output.cpp @@ -5,7 +5,11 @@ // Licensed under CeCILL 2.1 License, see COPYING for more information // *********************************************************************************** +#include #include "output.hpp" +#include "dataBlock.hpp" +#include "fluid.hpp" +#include "slice.hpp" Output::Output(Input &input, DataBlock &data) { idfx::pushRegion("Output::Output"); @@ -22,7 +26,30 @@ Output::Output(Input &input, DataBlock &data) { vtkEnabled = true; } } - vtk.Init(input,data); // Always initialised in case of emergency vtk output + + // Look for slice outputs (in the form of VTK files) + if(input.CheckEntry("Output","vtkSlice1")>0) { + sliceEnabled = true; + int n = 1; + while(input.CheckEntry("Output","vtkSlice"+std::to_string(n))>0) { + std::string sliceStr = "vtkSlice"+std::to_string(n); + real period = input.Get("Output", sliceStr,0); + int direction = input.Get("Output", sliceStr,1); + real x0 = input.Get("Output", sliceStr, 2); + std::string typeStr = input.Get("Output",sliceStr,3); + SliceType type; + if(typeStr.compare("cut")==0) { + type = SliceType::Cut; + } else if(typeStr.compare("average")==0) { + type = SliceType::Average; + } else { + IDEFIX_ERROR("Unknown slice type "+typeStr); + } + slices.emplace_back(std::make_unique(input, data, n, type, direction, x0, period)); + // Next iteration + n++; + } + } // Initialise xdmf outputs if(input.CheckEntry("Output","xdmf")>0) { @@ -37,9 +64,6 @@ Output::Output(Input &input, DataBlock &data) { #endif } } - #ifdef WITH_HDF5 - xdmf.Init(input,data); // Always initialised in case of emergency xdmf output - #endif // intialise dump outputs if(input.CheckEntry("Output","dmp")>0) { @@ -49,7 +73,6 @@ Output::Output(Input &input, DataBlock &data) { // Backwards compatibility: negative period means no dump if(dumpPeriod<0.0) dumpEnabled = false; } - dump.Init(input,data); // Always initialised since it is required on restarts // initialise analysis outputs if(input.CheckEntry("Output","analysis")>0) { @@ -69,9 +92,19 @@ Output::Output(Input &input, DataBlock &data) { data.np_tot[KDIR], data.np_tot[JDIR], data.np_tot[IDIR]); + data.vtk->RegisterVariable(userDefVariables[arrayName],arrayName); } userDefVariablesEnabled = true; } + + // Register variables that are needed in restart dumps + data.dump->RegisterVariable(&dumpLast, "dumpLast"); + data.dump->RegisterVariable(&analysisLast, "analysisLast"); + data.dump->RegisterVariable(&vtkLast, "vtkLast"); + #ifdef WITH_HDF5 + data.dump->RegisterVariable(&xdmfLast, "xdmfLast"); + #endif + idfx::popRegion(); } @@ -84,24 +117,6 @@ int Output::CheckForWrites(DataBlock &data) { idfx::popRegion(); return(0); } - // Do we need a restart dump? - if(dumpEnabled) { - if(data.t >= dumpLast + dumpPeriod) { - elapsedTime -= timer.seconds(); - dumpLast += dumpPeriod; - dump.Write(data,*this); - nfiles++; - elapsedTime += timer.seconds(); - - // Check if our next predicted output should already have happened - if((dumpLast+dumpPeriod <= data.t) && dumpPeriod>0.0) { - // Move forward dumpLast - while(dumpLast <= data.t - dumpPeriod) { - dumpLast += dumpPeriod; - } - } - } - } // Do we need a VTK output? if(vtkEnabled) { @@ -119,7 +134,7 @@ int Output::CheckForWrites(DataBlock &data) { } } vtkLast += vtkPeriod; - vtk.Write(data, *this); + data.vtk->Write(); nfiles++; elapsedTime += timer.seconds(); @@ -149,7 +164,7 @@ int Output::CheckForWrites(DataBlock &data) { } } xdmfLast += xdmfPeriod; - xdmf.Write(data, *this); + data.xdmf->Write(); nfiles++; elapsedTime += timer.seconds(); @@ -189,23 +204,50 @@ int Output::CheckForWrites(DataBlock &data) { } } + if(sliceEnabled) { + for(int i = 0 ; i < slices.size() ; i++) { + slices[i]->CheckForWrite(data); + } + } + // Do we need a restart dump? + if(dumpEnabled) { + // Dumps contain metadata about the most recent outputs of other types, + // so it's important that this part happens last. + if(data.t >= dumpLast + dumpPeriod) { + elapsedTime -= timer.seconds(); + dumpLast += dumpPeriod; + data.dump->Write(*this); + nfiles++; + elapsedTime += timer.seconds(); + + // Check if our next predicted output should already have happened + if((dumpLast+dumpPeriod <= data.t) && dumpPeriod>0.0) { + // Move forward dumpLast + while(dumpLast <= data.t - dumpPeriod) { + dumpLast += dumpPeriod; + } + } + } + } idfx::popRegion(); return(nfiles); } -void Output::RestartFromDump(DataBlock &data, int readNumber) { +bool Output::RestartFromDump(DataBlock &data, int readNumber) { idfx::pushRegion("Output::RestartFromDump"); - dump.Read(data, *this, readNumber); + bool result = data.dump->Read(*this, readNumber); + if(result) data.DeriveVectorPotential(); idfx::popRegion(); + return(result); } void Output::ForceWriteDump(DataBlock &data) { idfx::pushRegion("Output::ForceWriteDump"); - if(!forceNoWrite) dump.Write(data,*this); + if(!forceNoWrite) data.dump->Write(*this); idfx::popRegion(); } @@ -226,7 +268,7 @@ void Output::ForceWriteVtk(DataBlock &data) { } } vtkLast += vtkPeriod; - vtk.Write(data, *this); + data.vtk->Write(); } idfx::popRegion(); } @@ -248,7 +290,7 @@ void Output::ForceWriteXdmf(DataBlock &data) { } } xdmfLast += xdmfPeriod; - xdmf.Write(data, *this); + data.xdmf->Write(); } idfx::popRegion(); } diff --git a/src/output/output.hpp b/src/output/output.hpp index db754e93..bb6b7688 100644 --- a/src/output/output.hpp +++ b/src/output/output.hpp @@ -9,6 +9,8 @@ #define OUTPUT_OUTPUT_HPP_ #include #include +#include +#include #include "idefix.hpp" #include "input.hpp" #include "dataBlock.hpp" @@ -17,7 +19,7 @@ #include "xdmf.hpp" #endif #include "dump.hpp" - +#include "slice.hpp" using AnalysisFunc = void (*) (DataBlock &); @@ -35,7 +37,7 @@ class Output { public: Output(Input &, DataBlock &); // Create Output Object int CheckForWrites(DataBlock &); // Check if outputs are needed at this stage - void RestartFromDump(DataBlock &, int); // Restart from a dump file. + bool RestartFromDump(DataBlock &, int); // Restart from a dump file. void ForceWriteDump(DataBlock &); // Force write dumps (needed during an abort) void ForceWriteVtk(DataBlock &); // Force write vtks #ifdef WITH_HDF5 @@ -47,11 +49,6 @@ class Output { void EnrollUserDefVariables(UserDefVariablesFunc); private: - Vtk vtk; // local instance of Vtk class - Dump dump; // local instance of Dump class - #ifdef WITH_HDF5 - Xdmf xdmf; // local instance of Xdmf class - #endif bool forceNoWrite = false; //< explicitely disable all writes bool vtkEnabled = false; real vtkPeriod = 0.0; // periodicity of vtk outputs @@ -77,6 +74,9 @@ class Output { UserDefVariablesFunc userDefVariablesFunc; UserDefVariablesContainer userDefVariables; + bool sliceEnabled = false; + std::vector> slices; + Kokkos::Timer timer; double elapsedTime{0.0}; }; diff --git a/src/output/scalarField.hpp b/src/output/scalarField.hpp new file mode 100644 index 00000000..9af7b198 --- /dev/null +++ b/src/output/scalarField.hpp @@ -0,0 +1,60 @@ +// *********************************************************************************** +// Idefix MHD astrophysical code +// Copyright(C) Geoffroy R. J. Lesur +// and other code contributors +// Licensed under CeCILL 2.1 License, see COPYING for more information +// *********************************************************************************** + +#ifndef OUTPUT_SCALARFIELD_HPP_ +#define OUTPUT_SCALARFIELD_HPP_ +#include "idefix.hpp" + + +// Forward class declaration + +class ScalarField { + public: + enum Type {Device3D, Device4D, Host3D, Host4D}; + + explicit ScalarField(IdefixArray4D& in, const int varnum): + d4Darray{in}, var{varnum}, type{Device4D} {}; + explicit ScalarField(IdefixHostArray4D& in, const int varnum): + h4Darray{in}, var{varnum}, type{Host4D} {}; + explicit ScalarField(IdefixArray3D& in): + d3Darray{in}, type{Device3D} {}; + explicit ScalarField(IdefixHostArray3D& in): + h3Darray{in}, type{Host3D} {}; + + IdefixHostArray3D GetHostField() const { + if(type==Host3D) { + return(h3Darray); + } else if(type==Host4D) { + IdefixHostArray3D arr3D = Kokkos::subview( + h4Darray, var, Kokkos::ALL, Kokkos::ALL, Kokkos::ALL); + return(arr3D); + } else if(type==Device3D) { + IdefixHostArray3D arr3D = Kokkos::create_mirror(d3Darray); + Kokkos::deep_copy(arr3D,d3Darray); + return(arr3D); + } else if(type==Device4D) { + IdefixArray3D arrDev3D = Kokkos::subview( + d4Darray, var, Kokkos::ALL, Kokkos::ALL, Kokkos::ALL); + IdefixHostArray3D arr3D = Kokkos::create_mirror(arrDev3D); + Kokkos::deep_copy(arr3D,arrDev3D); + return(arr3D); + } else { + IDEFIX_ERROR("unknown field"); + return(h3Darray); + } + } + + private: + IdefixArray4D d4Darray; + IdefixArray3D d3Darray; + IdefixHostArray4D h4Darray; + IdefixHostArray3D h3Darray; + int var; + Type type; +}; + +#endif // OUTPUT_SCALARFIELD_HPP_ diff --git a/src/output/slice.cpp b/src/output/slice.cpp new file mode 100644 index 00000000..6c4a63b5 --- /dev/null +++ b/src/output/slice.cpp @@ -0,0 +1,131 @@ +// *********************************************************************************** +// Idefix MHD astrophysical code +// Copyright(C) Geoffroy R. J. Lesur +// and other code contributors +// Licensed under CeCILL 2.1 License, see COPYING for more information +// *********************************************************************************** + +#include +#include "slice.hpp" +#include "input.hpp" +#include "physics.hpp" +#include "dataBlock.hpp" +#include "fluid.hpp" +#include "vtk.hpp" + +Slice::Slice(Input &input, DataBlock & data, int nSlice, SliceType type, + int direction, real x0, real period) { + idfx::pushRegion("Slice::Slice"); + std::string prefix = "slice"+std::to_string(nSlice); + this->slicePeriod = period; + if(slicePeriod> 0) { + sliceLast = data.t - slicePeriod; + } + // Create the slice. + this->type = type; + this->direction = direction; + // Initialize the subgrid + this->subgrid = std::make_unique(data.mygrid, type, direction, x0); + // Initialize the associated dataBlock + this->sliceData = std::make_unique(subgrid.get()); + this->containsX0 = (data.xbeg[direction] <= x0) + && (data.xend[direction] >= x0); + + // Initialize the vtk routines + this->vtk = std::make_unique(input, sliceData.get(),prefix); + + // Allocate array to compute the slice + this->Vc = IdefixArray4D("Slice_Vc", NVAR, + sliceData->np_tot[KDIR], + sliceData->np_tot[JDIR], + sliceData->np_tot[IDIR]); + + vtk->RegisterVariable(Vc, "RHO", RHO); + vtk->RegisterVariable(Vc, "VX1", VX1); + + #if MHD == YES + vtk->RegisterVariable(Vc, "BX1", BX1); + vtk->RegisterVariable(Vc, "BX2", BX2); + vtk->RegisterVariable(Vc, "BX3", BX3); + #endif + + // todo(glesur): add variables for dust and other fluids. + + idfx::popRegion(); +} + +void Slice::CheckForWrite(DataBlock &data) { + idfx::pushRegion("Slice:CheckForWrite"); + + if(data.t >= sliceLast + slicePeriod) { + auto Vcin=data.hydro->Vc; + auto Vcout=this->Vc; + if(this->type == SliceType::Cut && containsX0) { + // index of element in current datablock + const int idx = subgrid->index - data.gbeg[direction] + + data.beg[direction]; + + if(direction == IDIR) { + idefix_for("slice",0,NVAR,0,data.np_tot[KDIR],0,data.np_tot[JDIR], + KOKKOS_LAMBDA(int n,int k,int j) { + Vcout(n,k,j,0) = Vcin(n,k,j,idx); + }); + } else if(direction == JDIR) { + idefix_for("slice",0,NVAR,0,data.np_tot[KDIR],0,data.np_tot[IDIR], + KOKKOS_LAMBDA(int n,int k,int i) { + Vcout(n,k,0,i) = Vcin(n,k,idx,i); + }); + } else if(direction == KDIR) { + idefix_for("slice",0,NVAR,0,data.np_tot[JDIR],0,data.np_tot[IDIR], + KOKKOS_LAMBDA(int n,int j,int i) { + Vcout(n,0,j,i) = Vcin(n,idx,j,i); + }); + } + vtk->Write(); + } + if(this->type == SliceType::Average) { + // Perform a point average (NB: this does not perform a volume average!) + // This is a naive approach which could be optimised using threaded loops + // However, since this is only for I/O, we don't do any proper optimisation + idefix_for("Zero",0,NVAR,0,Vcout.extent(1),0,Vcout.extent(2), + KOKKOS_LAMBDA(int n,int k,int j) { + Vcout(n,k,j,0) = 0.0; + }); + int beg = data.beg[direction]; + int end = data.end[direction]; + int ntot = data.mygrid->np_int[direction]; + const int dir = direction; + idefix_for("average",0,NVAR,data.beg[KDIR],data.end[KDIR], + data.beg[JDIR],data.end[JDIR], + data.beg[IDIR],data.end[IDIR], + KOKKOS_LAMBDA(int n,int k,int j, int i) { + const int it = (dir == IDIR ? 0 : i); + const int jt = (dir == JDIR ? 0 : j); + const int kt = (dir == KDIR ? 0 : k); + + Kokkos::atomic_add(&Vcout(n,kt,jt,it) , Vcin(n,k,j,i)/ntot); + }); + #ifdef WITH_MPI + // Create a communicator on which we can do the sum accross processors + int remainDims[3] = {false, false, false}; + remainDims[direction] = true; + MPI_Comm avgComm; + MPI_Cart_sub(subgrid->parentGrid->CartComm, remainDims, &avgComm); + MPI_Allreduce(MPI_IN_PLACE, Vcout.data(), + Vcout.extent(0)*Vcout.extent(1)*Vcout.extent(2)*Vcout.extent(3), + realMPI, MPI_SUM, avgComm); + #endif + if(containsX0) { + vtk->Write(); + } + } + + sliceLast += slicePeriod; + if((sliceLast+slicePeriod <= data.t) && slicePeriod > 0.0) { + while(sliceLast <= data.t - slicePeriod) { + sliceLast += slicePeriod; + } + } + } + idfx::popRegion(); +} diff --git a/src/output/slice.hpp b/src/output/slice.hpp new file mode 100644 index 00000000..645d35a7 --- /dev/null +++ b/src/output/slice.hpp @@ -0,0 +1,38 @@ +// *********************************************************************************** +// Idefix MHD astrophysical code +// Copyright(C) Geoffroy R. J. Lesur +// and other code contributors +// Licensed under CeCILL 2.1 License, see COPYING for more information +// *********************************************************************************** + + +#ifndef OUTPUT_SLICE_HPP_ +#define OUTPUT_SLICE_HPP_ + +#include +#include "idefix.hpp" +#include "dataBlock.hpp" +#include "input.hpp" + +class Grid; +class SubGrid; +class Vtk; + + +class Slice { + public: + Slice(Input &, DataBlock &, int, SliceType, int, real, real); + void CheckForWrite(DataBlock &); + real slicePeriod = 0.0; + real sliceLast = 0.0; + private: + bool containsX0; + IdefixArray4D Vc; + SliceType type; + int direction; + std::unique_ptr subgrid; + std::unique_ptr sliceData; + std::unique_ptr vtk; +}; + +#endif // OUTPUT_SLICE_HPP_ diff --git a/src/output/vtk.cpp b/src/output/vtk.cpp index 80bb053e..85401e3f 100644 --- a/src/output/vtk.cpp +++ b/src/output/vtk.cpp @@ -8,12 +8,14 @@ #include #include #include +#include #include "vtk.hpp" -#include "gitversion.hpp" +#include "version.hpp" #include "idefix.hpp" -#include "dataBlockHost.hpp" +#include "dataBlock.hpp" #include "gridHost.hpp" #include "output.hpp" +#include "fluid.hpp" #define VTK_RECTILINEAR_GRID 14 #define VTK_STRUCTURED_GRID 35 @@ -26,38 +28,6 @@ #endif #endif -// Whether or not we write the time in the VTK file -#define WRITE_TIME - -void Vtk::WriteHeaderString(const char* header, IdfxFileHandler fvtk) { -#ifdef WITH_MPI - MPI_Status status; - MPI_SAFE_CALL(MPI_File_set_view(fvtk, this->offset, MPI_BYTE, - MPI_CHAR, "native", MPI_INFO_NULL )); - if(idfx::prank==0) { - MPI_SAFE_CALL(MPI_File_write(fvtk, header, strlen(header), MPI_CHAR, &status)); - } - offset=offset+strlen(header); -#else - fprintf (fvtk, "%s", header); -#endif -} - -template -void Vtk::WriteHeaderBinary(T* buffer, int64_t nelem, IdfxFileHandler fvtk) { -#ifdef WITH_MPI - MPI_Status status; - MPI_SAFE_CALL(MPI_File_set_view(fvtk, this->offset, MPI_BYTE, MPI_CHAR, - "native", MPI_INFO_NULL )); - if(idfx::prank==0) { - MPI_SAFE_CALL(MPI_File_write(fvtk, buffer, nelem*sizeof(T), MPI_CHAR, &status)); - } - offset=offset+nelem*sizeof(T); -#else - fwrite(buffer, sizeof(T), nelem, fvtk); -#endif -} - void Vtk::WriteHeaderNodes(IdfxFileHandler fvtk) { int64_t size = node_coord.extent(0) * node_coord.extent(1) * @@ -68,64 +38,92 @@ void Vtk::WriteHeaderNodes(IdfxFileHandler fvtk) { MPI_SAFE_CALL(MPI_File_set_view(fvtk, this->offset, MPI_FLOAT, this->nodeView, "native", MPI_INFO_NULL)); MPI_SAFE_CALL(MPI_File_write_all(fvtk, node_coord.data(), size, MPI_FLOAT, MPI_STATUS_IGNORE)); - this->offset += sizeof(float)*(nx1+IOFFSET)*(nx2+JOFFSET)*(nx3+KOFFSET)*3; + this->offset += sizeof(float)*(nx1+ioffset)*(nx2+joffset)*(nx3+koffset)*3; #else fwrite(node_coord.data(),sizeof(float),size,fvtk); #endif } /*init the object */ -void Vtk::Init(Input &input, DataBlock &datain) { +Vtk::Vtk(Input &input, DataBlock *datain, std::string filebase) { // Initialize the output structure // Create a local datablock as an image of gridin - DataBlockHost data(datain); - data.SyncFromDevice(); + this->data = datain; + + this->filebase = filebase; // Pointer to global grid - GridHost grid(*datain.mygrid); + GridHost grid(*(datain->mygrid)); grid.SyncFromDevice(); + // initialize output path + if(input.CheckEntry("Output","vtk_dir")>=0) { + outputDirectory = input.Get("Output","vtk_dir",0); + } else { + outputDirectory = "./"; + } + + if(idfx::prank==0) { + if(!std::filesystem::is_directory(outputDirectory)) { + try { + if(!std::filesystem::create_directory(outputDirectory)) { + std::stringstream msg; + msg << "Cannot create directory " << outputDirectory << std::endl; + IDEFIX_ERROR(msg); + } + } catch(std::exception &e) { + std::stringstream msg; + msg << "Cannot create directory " << outputDirectory << std::endl; + msg << e.what(); + IDEFIX_ERROR(msg); + } + } + } + /* Note that there are two kinds of dimensions: - nx1, nx2, nx3, derived from the grid, which are the global dimensions - nx1loc,nx2loc,n3loc, which are the local dimensions of the current datablock */ for (int dir=0; dir<3; dir++) { - this->periodicity[dir] = (datain.mygrid->lbound[dir] == periodic); + this->periodicity[dir] = (datain->mygrid->lbound[dir] == periodic); } // Create the coordinate array required in VTK files this->nx1 = grid.np_int[IDIR]; this->nx2 = grid.np_int[JDIR]; this->nx3 = grid.np_int[KDIR]; - this->nx1loc = data.np_int[IDIR]; - this->nx2loc = data.np_int[JDIR]; - this->nx3loc = data.np_int[KDIR]; + this->nx1loc = data->np_int[IDIR]; + this->nx2loc = data->np_int[JDIR]; + this->nx3loc = data->np_int[KDIR]; + + this->ioffset = datain->mygrid->np_tot[IDIR] == 1 ? 0 : 1; + this->joffset = datain->mygrid->np_tot[JDIR] == 1 ? 0 : 1; + this->koffset = datain->mygrid->np_tot[KDIR] == 1 ? 0 : 1; // Temporary storage on host for 3D arrays this->vect3D = new float[nx1loc*nx2loc*nx3loc]; - // Test endianness - int tmp1 = 1; - this->shouldSwapEndian = 0; - unsigned char *tmp2 = (unsigned char *) &tmp1; - if (*tmp2 != 0) - this->shouldSwapEndian = 1; - // Store coordinates for later use - this->xnode = new float[nx1+IOFFSET]; - this->ynode = new float[nx2+JOFFSET]; - this->znode = new float[nx3+KOFFSET]; + this->xnode = new float[nx1+ioffset]; + this->ynode = new float[nx2+joffset]; + this->znode = new float[nx3+koffset]; - for (int32_t i = 0; i < nx1 + IOFFSET; i++) { - xnode[i] = BigEndian(static_cast(grid.xl[IDIR](i + grid.nghost[IDIR]))); + for (int32_t i = 0; i < nx1 + ioffset; i++) { + if(grid.np_tot[IDIR] == 1) // only one dimension in this direction + xnode[i] = BigEndian(static_cast(grid.x[IDIR](i))); + else + xnode[i] = BigEndian(static_cast(grid.xl[IDIR](i + grid.nghost[IDIR]))); } - for (int32_t j = 0; j < nx2 + JOFFSET; j++) { - ynode[j] = BigEndian(static_cast(grid.xl[JDIR](j + grid.nghost[JDIR]))); + for (int32_t j = 0; j < nx2 + joffset; j++) { + if(grid.np_tot[JDIR] == 1) // only one dimension in this direction + ynode[j] = BigEndian(static_cast(grid.x[JDIR](j))); + else + ynode[j] = BigEndian(static_cast(grid.xl[JDIR](j + grid.nghost[JDIR]))); } - for (int32_t k = 0; k < nx3 + KOFFSET; k++) { - if(DIMENSIONS==2) - znode[k] = BigEndian(static_cast(0.0)); + for (int32_t k = 0; k < nx3 + koffset; k++) { + if(grid.np_tot[KDIR] == 1) + znode[k] = BigEndian(static_cast(grid.x[KDIR](k))); else znode[k] = BigEndian(static_cast(grid.xl[KDIR](k + grid.nghost[KDIR]))); } @@ -138,9 +136,9 @@ void Vtk::Init(Input &input, DataBlock &datain) { int nodesubsize[4]; for(int dir = 0; dir < 3 ; dir++) { - nodesize[2-dir] = datain.mygrid->np_int[dir]; - nodestart[2-dir] = datain.gbeg[dir]-datain.nghost[dir]; - nodesubsize[2-dir] = datain.np_int[dir]; + nodesize[2-dir] = datain->mygrid->np_int[dir]; + nodestart[2-dir] = datain->gbeg[dir]-datain->nghost[dir]; + nodesubsize[2-dir] = datain->np_int[dir]; } // In the 4th dimension, we always have the 3 components @@ -151,16 +149,17 @@ void Vtk::Init(Input &input, DataBlock &datain) { // Since we use cell-defined vtk variables, // we add one cell in each direction when we're looking at the last // sub-domain in each direction - nodesize[2] += IOFFSET; - nodesize[1] += JOFFSET; - nodesize[0] += KOFFSET; + nodesize[2] += ioffset; + nodesize[1] += joffset; + nodesize[0] += koffset; - if(datain.mygrid->xproc[0] == datain.mygrid->nproc[0]-1) nodesubsize[2] += IOFFSET; - if(datain.mygrid->xproc[1] == datain.mygrid->nproc[1]-1) nodesubsize[1] += JOFFSET; - if(datain.mygrid->xproc[2] == datain.mygrid->nproc[2]-1) nodesubsize[0] += KOFFSET; + if(datain->mygrid->xproc[0] == datain->mygrid->nproc[0]-1) nodesubsize[2] += ioffset; + if(datain->mygrid->xproc[1] == datain->mygrid->nproc[1]-1) nodesubsize[1] += joffset; + if(datain->mygrid->xproc[2] == datain->mygrid->nproc[2]-1) nodesubsize[0] += koffset; // Build an MPI view if needed #ifdef WITH_MPI + // Keep communicator for later use MPI_SAFE_CALL(MPI_Type_create_subarray(4, nodesize, nodesubsize, nodestart, MPI_ORDER_C, MPI_FLOAT, &this->nodeView)); MPI_SAFE_CALL(MPI_Type_commit(&this->nodeView)); @@ -180,9 +179,9 @@ void Vtk::Init(Input &input, DataBlock &datain) { for (int32_t j = 0; j < nodesubsize[1]; j++) { for (int32_t i = 0; i < nodesubsize[2]; i++) { // BigEndian allows us to get back to little endian when needed - D_EXPAND( x1 = data.xl[IDIR](i + grid.nghost[IDIR] ); , - x2 = data.xl[JDIR](j + grid.nghost[JDIR]); , - x3 = data.xl[KDIR](k + grid.nghost[KDIR]); ) + x1 = grid.xl[IDIR](i + data->gbeg[IDIR]); + x2 = grid.xl[JDIR](j + data->gbeg[JDIR]); + x3 = grid.xl[KDIR](k + data->gbeg[KDIR]); #if (GEOMETRY == CARTESIAN) || (GEOMETRY == CYLINDRICAL) node_coord(k,j,i,0) = BigEndian(x1); @@ -214,7 +213,7 @@ void Vtk::Init(Input &input, DataBlock &datain) { } } -#endif +#endif // VTK_FORMAT == VTK_STRUCTURED_GRID // Create MPI view when using MPI I/O #ifdef WITH_MPI @@ -224,105 +223,77 @@ void Vtk::Init(Input &input, DataBlock &datain) { for(int dir = 0; dir < 3 ; dir++) { // VTK assumes Fortran array ordering, hence arrays dimensions are filled backwards - start[2-dir] = datain.gbeg[dir]-grid.nghost[dir]; + start[2-dir] = data->gbeg[dir]-grid.nghost[dir]; size[2-dir] = grid.np_int[dir]; - subsize[2-dir] = datain.np_int[dir]; + subsize[2-dir] = data->np_int[dir]; } MPI_SAFE_CALL(MPI_Type_create_subarray(3, size, subsize, start, MPI_ORDER_C, MPI_FLOAT, &this->view)); MPI_SAFE_CALL(MPI_Type_commit(&this->view)); + this->comm = datain->mygrid->CartComm; + this->isRoot = (data->mygrid->xproc[0] == 0) + && (data->mygrid->xproc[1] == 0) + && (data->mygrid->xproc[2] == 0); #endif + + // Register variables that are required in restart dumps + if(data->dump.get() != nullptr) + data->dump->RegisterVariable(&vtkFileNumber, "vtkFileNumber"); } -int Vtk::Write(DataBlock &datain, Output &output) { +int Vtk::Write() { idfx::pushRegion("Vtk::Write"); IdfxFileHandler fileHdl; - std::string filename; - - idfx::cout << "Vtk: Write file n " << vtkFileNumber << "..." << std::flush; + std::filesystem::path filename; timer.reset(); - // Create a copy of the dataBlock on Host, and sync it. - DataBlockHost data(datain); - data.SyncFromDevice(); - std::stringstream ssfileName, ssvtkFileNum; ssvtkFileNum << std::setfill('0') << std::setw(4) << vtkFileNumber; - ssfileName << "data." << ssvtkFileNum.str() << ".vtk"; - filename = ssfileName.str(); + ssfileName << filebase << "." << ssvtkFileNum.str() << ".vtk"; + filename = outputDirectory/ssfileName.str(); + + idfx::cout << "Vtk: Write file " << ssfileName.str() << "..." << std::flush; + + // Check if file exists, if yes, delete it + if(this->isRoot) { + if(std::filesystem::exists(filename)) { + std::filesystem::remove(filename); + } + } + // Open file and write header #ifdef WITH_MPI + MPI_Barrier(this->comm); // Open file for creating, return error if file already exists. - int err = MPI_File_open(MPI_COMM_WORLD, filename.c_str(), - MPI_MODE_CREATE | MPI_MODE_RDWR - | MPI_MODE_EXCL | MPI_MODE_UNIQUE_OPEN, - MPI_INFO_NULL, &fileHdl); - if (err != MPI_SUCCESS) { - // File exists, delete it before reopening - if(idfx::prank == 0) { - MPI_File_delete(filename.c_str(),MPI_INFO_NULL); - } - MPI_SAFE_CALL(MPI_File_open(MPI_COMM_WORLD, filename.c_str(), + MPI_SAFE_CALL(MPI_File_open(this->comm, filename.c_str(), MPI_MODE_CREATE | MPI_MODE_RDWR | MPI_MODE_EXCL | MPI_MODE_UNIQUE_OPEN, MPI_INFO_NULL, &fileHdl)); - } this->offset = 0; #else fileHdl = fopen(filename.c_str(),"wb"); #endif - WriteHeader(fileHdl, datain.t); + WriteHeader(fileHdl, this->data->t); // Write field one by one - for(int nv = 0 ; nv < NVAR ; nv++) { - for(int k = data.beg[KDIR]; k < data.end[KDIR] ; k++ ) { - for(int j = data.beg[JDIR]; j < data.end[JDIR] ; j++ ) { - for(int i = data.beg[IDIR]; i < data.end[IDIR] ; i++ ) { - vect3D[i-data.beg[IDIR] + (j-data.beg[JDIR])*nx1loc + (k-data.beg[KDIR])*nx1loc*nx2loc] - = BigEndian(static_cast(data.Vc(nv,k,j,i))); + for(auto const& [name, scalar] : vtkScalarMap) { + auto Vcin = scalar.GetHostField(); + for(int k = data->beg[KDIR]; k < data->end[KDIR] ; k++ ) { + for(int j = data->beg[JDIR]; j < data->end[JDIR] ; j++ ) { + for(int i = data->beg[IDIR]; i < data->end[IDIR] ; i++ ) { + vect3D[i-data->beg[IDIR] + (j-data->beg[JDIR])*nx1loc + (k-data->beg[KDIR])*nx1loc*nx2loc] + = BigEndian(static_cast(Vcin(k,j,i))); } } } - WriteScalar(fileHdl, vect3D, datain.hydro.VcName[nv]); + WriteScalar(fileHdl, vect3D, name); } - // Write user-defined variables (when required by output) - if(output.userDefVariablesEnabled) { - // Walk the map and make an output for each key of the map - // (and we thank c++11 for its cute way of doing this) - for(auto const &variable : output.userDefVariables) { - for(int k = data.beg[KDIR]; k < data.end[KDIR] ; k++ ) { - for(int j = data.beg[JDIR]; j < data.end[JDIR] ; j++ ) { - for(int i = data.beg[IDIR]; i < data.end[IDIR] ; i++ ) { - vect3D[i-data.beg[IDIR] + (j-data.beg[JDIR])*nx1loc + (k-data.beg[KDIR])*nx1loc*nx2loc] - = BigEndian(static_cast(variable.second(k,j,i))); - } - } - } - WriteScalar(fileHdl, vect3D, variable.first); - } - } - - // Write vector potential if we're using this - #ifdef EVOLVE_VECTOR_POTENTIAL - for(int nv = 0 ; nv <= AX3e ; nv++) { - for(int k = data.beg[KDIR]; k < data.end[KDIR] ; k++ ) { - for(int j = data.beg[JDIR]; j < data.end[JDIR] ; j++ ) { - for(int i = data.beg[IDIR]; i < data.end[IDIR] ; i++ ) { - vect3D[i-data.beg[IDIR] + (j-data.beg[JDIR])*nx1loc + (k-data.beg[KDIR])*nx1loc*nx2loc] - = BigEndian(static_cast(data.Ve(nv,k,j,i))); - } - } - } - WriteScalar(fileHdl, vect3D, datain.hydro.VeName[nv]); - } - #endif - #ifdef WITH_MPI MPI_SAFE_CALL(MPI_File_close(&fileHdl)); @@ -364,7 +335,7 @@ void Vtk::WriteHeader(IdfxFileHandler fvtk, real time) { 2. Header ------------------------------------------- */ - ssheader << "Idefix " << GITVERSION << " VTK Data" << std::endl; + ssheader << "Idefix " << IDEFIX_VERSION << " VTK Data" << std::endl; /* ------------------------------------------ 3. File format @@ -381,11 +352,8 @@ void Vtk::WriteHeader(IdfxFileHandler fvtk, real time) { #elif VTK_FORMAT == VTK_STRUCTURED_GRID ssheader << "DATASET STRUCTURED_GRID" << std::endl; #endif - // One field for the geometry, another for periodicity - int nfields = 2; - #ifdef WRITE_TIME - nfields ++; - #endif + // fields: geometry, periodicity, time + int nfields = 3; // Write grid geometry in the VTK file ssheader << "FIELD FieldData " << nfields << std::endl; @@ -421,25 +389,23 @@ void Vtk::WriteHeader(IdfxFileHandler fvtk, real time) { // Done, add cariage return for next ascii write ssheader << std::endl; - #ifdef WRITE_TIME - ssheader << "TIME 1 1 float" << std::endl; - // Flush the ascii header - header = ssheader.str(); - WriteHeaderString(header.c_str(), fvtk); - // reset the string stream - ssheader.str(std::string()); + ssheader << "TIME 1 1 float" << std::endl; + // Flush the ascii header + header = ssheader.str(); + WriteHeaderString(header.c_str(), fvtk); + // reset the string stream + ssheader.str(std::string()); - // convert time to single precision big endian - float timeBE = BigEndian(static_cast(time)); + // convert time to single precision big endian + float timeBE = BigEndian(static_cast(time)); - WriteHeaderBinary(&timeBE, 1, fvtk); - // Done, add cariage return for next ascii write - ssheader << std::endl; - #endif + WriteHeaderBinary(&timeBE, 1, fvtk); + // Done, add cariage return for next ascii write + ssheader << std::endl; - ssheader << "DIMENSIONS " << nx1 + IOFFSET << " " << nx2 + JOFFSET << " " << nx3 + KOFFSET + ssheader << "DIMENSIONS " << nx1 + ioffset << " " << nx2 + joffset << " " << nx3 + koffset << std::endl; header = ssheader.str(); @@ -450,30 +416,30 @@ void Vtk::WriteHeader(IdfxFileHandler fvtk, real time) { /* -- Write rectilinear grid information -- */ std::stringstream coordx, coordy, coordz; - coordx << "X_COORDINATES " << nx1 + IOFFSET << " float" << std::endl; + coordx << "X_COORDINATES " << nx1 + ioffset << " float" << std::endl; header = coordx.str(); WriteHeaderString(header.c_str(), fvtk); - WriteHeaderBinary(xnode, nx1 + IOFFSET, fvtk); + WriteHeaderBinary(xnode, nx1 + ioffset, fvtk); - coordy << std::endl << "Y_COORDINATES " << nx2 + JOFFSET << " float" << std::endl; + coordy << std::endl << "Y_COORDINATES " << nx2 + joffset << " float" << std::endl; header = coordy.str(); WriteHeaderString(header.c_str(), fvtk); - WriteHeaderBinary(ynode, nx2 + JOFFSET, fvtk); + WriteHeaderBinary(ynode, nx2 + joffset, fvtk); - coordz << std::endl << "Z_COORDINATES " << nx3 + KOFFSET << " float" << std::endl; + coordz << std::endl << "Z_COORDINATES " << nx3 + koffset << " float" << std::endl; header = coordz.str(); WriteHeaderString(header.c_str(), fvtk); - WriteHeaderBinary(znode, nx3 + KOFFSET, fvtk); + WriteHeaderBinary(znode, nx3 + koffset, fvtk); #elif VTK_FORMAT == VTK_STRUCTURED_GRID /* -- define node_coord -- */ std::stringstream ssheader2; - ssheader2 << "POINTS " << (nx1 + IOFFSET) * (nx2 + JOFFSET) * (nx3 + KOFFSET) << " float" + ssheader2 << "POINTS " << (nx1 + ioffset) * (nx2 + joffset) * (nx3 + koffset) << " float" << std::endl; header = ssheader2.str(); WriteHeaderString(header.c_str(), fvtk); @@ -516,29 +482,13 @@ void Vtk::WriteScalar(IdfxFileHandler fvtk, float* Vin, const std::string &var_ #ifdef WITH_MPI MPI_SAFE_CALL(MPI_File_set_view(fvtk, this->offset, MPI_FLOAT, this->view, "native", MPI_INFO_NULL)); - MPI_SAFE_CALL(MPI_File_write_all(fvtk, Vin, nx1loc*nx2loc*nx3loc, MPI_FLOAT, MPI_STATUS_IGNORE)); + + int nwrite = nx1loc*nx2loc*nx3loc; + //if(idfx::prank != 0) nwrite = 0; + MPI_SAFE_CALL(MPI_File_write_all(fvtk, Vin, nwrite, MPI_FLOAT, MPI_STATUS_IGNORE)); + this->offset = this->offset + sizeof(float)*nx1*nx2*nx3; #else fwrite(Vin,sizeof(float),nx1loc*nx2loc*nx3loc,fvtk); #endif } - -/* ****************************************************************************/ -/** Determines if the machine is little-endian. If so, - it will force the data to be big-endian. -@param in_number floating point number to be converted in big endian */ -/* *************************************************************************** */ - -template -T Vtk::BigEndian(T in_number) { - if (shouldSwapEndian) { - unsigned char *bytes = (unsigned char*) &in_number; - unsigned char tmp = bytes[0]; - bytes[0] = bytes[3]; - bytes[3] = tmp; - tmp = bytes[1]; - bytes[1] = bytes[2]; - bytes[2] = tmp; - } - return(in_number); -} diff --git a/src/output/vtk.hpp b/src/output/vtk.hpp index 6d36b375..eb6b813f 100644 --- a/src/output/vtk.hpp +++ b/src/output/vtk.hpp @@ -9,21 +9,37 @@ #define OUTPUT_VTK_HPP_ #include #include +#include #include "idefix.hpp" #include "input.hpp" #include "dataBlock.hpp" +#include "scalarField.hpp" // Forward class declaration class Output; +class DataBlock; -class Vtk { - friend class Dump; - - public: - void Init(Input &, DataBlock &); // init VTK object - int Write(DataBlock &, Output &); // Create a VTK from the current DataBlock +class BaseVtk { private: + // Endianness swaping function and variable + bool shouldSwapEndian {true}; + + protected: + BaseVtk() { + // Test endianness + int tmp1 = 1; + unsigned char *tmp2 = (unsigned char *) &tmp1; + if (*tmp2 == 0) + this->shouldSwapEndian = false; + // Initialise the root tag (used for MPI non-collective I/Os) + this->isRoot = idfx::prank == 0; + } + + // File offset +#ifdef WITH_MPI + MPI_Offset offset; +#endif // define a mapping from global geometry flags defined in idefix.hpp // to the ones we write in vtk files std::map VTKGeometryFlags = { @@ -32,10 +48,83 @@ class Vtk { {SPHERICAL, 2}, {CYLINDRICAL, 3}, }; + int vtkFileNumber = 0; int geometry{VTKGeometryFlags[GEOMETRY]}; int periodicity[3]; + bool isRoot; // Whether this process is our root process for the current + // Timer + Kokkos::Timer timer; + + // DataBlock parent + DataBlock *data; + + /* ****************************************************************************/ + /** Determines if the machine is little-endian. If so, + it will force the data to be big-endian. + @param in_number floating point number to be converted in big endian */ + /* *************************************************************************** */ + + template + T BigEndian(T in_number) { + if (shouldSwapEndian) { + unsigned char *bytes = (unsigned char*) &in_number; + unsigned char tmp = bytes[0]; + bytes[0] = bytes[3]; + bytes[3] = tmp; + tmp = bytes[1]; + bytes[1] = bytes[2]; + bytes[2] = tmp; + } + return(in_number); + } + + void WriteHeaderString(const char* header, IdfxFileHandler fvtk) { + #ifdef WITH_MPI + MPI_Status status; + MPI_SAFE_CALL(MPI_File_set_view(fvtk, this->offset, MPI_BYTE, + MPI_CHAR, "native", MPI_INFO_NULL )); + if(this->isRoot) { + MPI_SAFE_CALL(MPI_File_write(fvtk, header, strlen(header), MPI_CHAR, &status)); + } + offset=offset+strlen(header); + #else + fprintf (fvtk, "%s", header); + #endif + } + + template + void WriteHeaderBinary(T* buffer, int64_t nelem, IdfxFileHandler fvtk) { + #ifdef WITH_MPI + MPI_Status status; + MPI_SAFE_CALL(MPI_File_set_view(fvtk, this->offset, MPI_BYTE, MPI_CHAR, + "native", MPI_INFO_NULL )); + if(this->isRoot) { + MPI_SAFE_CALL(MPI_File_write(fvtk, buffer, nelem*sizeof(T), MPI_CHAR, &status)); + } + offset=offset+nelem*sizeof(T); + #else + fwrite(buffer, sizeof(T), nelem, fvtk); + #endif + } +}; + + +class Vtk : public BaseVtk { + friend class Dump; + + public: + explicit Vtk(Input &, DataBlock *, std::string filebase = "data"); // init VTK object + int Write(); // Create a VTK from the current DataBlock + + template + void RegisterVariable(T&, std::string, int var = -1); + + private: + // List of variables to be written to vtk files + std::map vtkScalarMap; + // dimensions int64_t nx1,nx2,nx3; int64_t nx1loc,nx2loc,nx3loc; @@ -43,6 +132,9 @@ class Vtk { // number of ghost zones int64_t ngx1,ngx2,ngx3; + // offset in each direction (used by the vtk grid) + int ioffset, joffset, koffset; + // Coordinates needed by VTK outputs float *xnode, *ynode, *znode; @@ -51,25 +143,33 @@ class Vtk { // Array designed to store the temporary vector array float *vect3D; - // Endianness swaping function and variable - int doneEndianTest, shouldSwapEndian; - - // Timer - Kokkos::Timer timer; + // File name + std::string filebase; // File offset #ifdef WITH_MPI - MPI_Offset offset; MPI_Datatype view; MPI_Datatype nodeView; + MPI_Comm comm; #endif void WriteHeader(IdfxFileHandler, real); void WriteScalar(IdfxFileHandler, float*, const std::string &); - template T BigEndian(T); - void WriteHeaderString(const char* , IdfxFileHandler ); - template void WriteHeaderBinary(T* , int64_t, IdfxFileHandler); void WriteHeaderNodes(IdfxFileHandler); + + // output directory + std::filesystem::path outputDirectory; }; +template +void Vtk::RegisterVariable(T& in, std::string name, int var) { + // if var>0, the caller provided explicitely an index + if constexpr(std::is_same>::value || + std::is_same>::value) { + vtkScalarMap.emplace(name, ScalarField(in) ); + } else { + vtkScalarMap.emplace(name, ScalarField(in, var)); + } +} + #endif // OUTPUT_VTK_HPP_ diff --git a/src/output/xdmf.cpp b/src/output/xdmf.cpp index ccf7e4a6..7f2baf7d 100644 --- a/src/output/xdmf.cpp +++ b/src/output/xdmf.cpp @@ -11,7 +11,7 @@ #include #include #include "xdmf.hpp" -#include "gitversion.hpp" +#include "version.hpp" #include "idefix.hpp" #include "dataBlockHost.hpp" #include "gridHost.hpp" @@ -20,41 +20,66 @@ // Whether or not we write the time in the XDMF file #define WRITE_TIME -/*init the object */ -void Xdmf::Init(Input &input, DataBlock &datain) { + +Xdmf::Xdmf(Input &input, DataBlock *datain) { // Initialize the output structure // Create a local datablock as an image of gridin - DataBlockHost data(datain); - data.SyncFromDevice(); + + this->data = datain; // Pointer to global grid - GridHost grid(*datain.mygrid); + GridHost grid(*(data->mygrid)); grid.SyncFromDevice(); + // initialize output path + if(input.CheckEntry("Output","xdmf_dir")>=0) { + outputDirectory = input.Get("Output","xdmf_dir",0); + } else { + outputDirectory = "./"; + } + + if(idfx::prank==0) { + if(!std::filesystem::is_directory(outputDirectory)) { + try { + if(!std::filesystem::create_directory(outputDirectory)) { + std::stringstream msg; + msg << "Cannot create directory " << outputDirectory << std::endl; + IDEFIX_ERROR(msg); + } + } catch(std::exception &e) { + std::stringstream msg; + msg << "Cannot create directory " << outputDirectory << std::endl; + msg << e.what(); + IDEFIX_ERROR(msg); + } + } + } + /* Note that there are two kinds of dimensions: - nx1, nx2, nx3, derived from the grid, which are the global dimensions - nx1loc,nx2loc,n3loc, which are the local dimensions of the current datablock */ for (int dir=0; dir<3; dir++) { - this->periodicity[dir] = (datain.mygrid->lbound[dir] == periodic); + this->periodicity[dir] = (data->mygrid->lbound[dir] == periodic); } // Create the coordinate array required in XDMF files this->nx1 = grid.np_int[IDIR]; this->nx2 = grid.np_int[JDIR]; this->nx3 = grid.np_int[KDIR]; - this->nx1loc = data.np_int[IDIR]; - this->nx2loc = data.np_int[JDIR]; - this->nx3loc = data.np_int[KDIR]; + this->nx1loc = data->np_int[IDIR]; + this->nx2loc = data->np_int[JDIR]; + this->nx3loc = data->np_int[KDIR]; this->nx1tot = grid.np_tot[IDIR]; this->nx2tot = grid.np_tot[JDIR]; this->nx3tot = grid.np_tot[KDIR]; - this->nx1loctot = data.np_tot[IDIR]; - this->nx2loctot = data.np_tot[JDIR]; - this->nx3loctot = data.np_tot[KDIR]; + this->nx1loctot = data->np_tot[IDIR]; + this->nx2loctot = data->np_tot[JDIR]; + this->nx3loctot = data->np_tot[KDIR]; + this->ngx1 = grid.nghost[IDIR]; this->ngx2 = grid.nghost[JDIR]; @@ -94,13 +119,13 @@ void Xdmf::Init(Input &input, DataBlock &datain) { /* -- Allocate memory for node_coord which is later used -- */ /* -- Data order that is saved is 3D/1D: Z-Y-X and 2D: Y-X-Z -- */ for(int dir = 0; dir < 3 ; dir++) { - this->nodesize[3-dir] = datain.mygrid->np_int[dir]; - this->nodestart[3-dir] = datain.gbeg[dir]-datain.nghost[dir]; - this->nodesubsize[3-dir] = datain.np_int[dir]; + this->nodesize[3-dir] = data->mygrid->np_int[dir]; + this->nodestart[3-dir] = data->gbeg[dir]-data->nghost[dir]; + this->nodesubsize[3-dir] = data->np_int[dir]; - this->cellsize[3-dir] = datain.mygrid->np_int[dir]; - this->cellstart[3-dir] = datain.gbeg[dir]-datain.nghost[dir]; - this->cellsubsize[3-dir] = datain.np_int[dir]; + this->cellsize[3-dir] = data->mygrid->np_int[dir]; + this->cellstart[3-dir] = data->gbeg[dir]-data->nghost[dir]; + this->cellsubsize[3-dir] = data->np_int[dir]; } // In the 0th dimension, we always have the 3 components @@ -120,9 +145,9 @@ void Xdmf::Init(Input &input, DataBlock &datain) { this->nodesize[2] += JOFFSET; this->nodesize[1] += KOFFSET; - if(datain.mygrid->xproc[0] == datain.mygrid->nproc[0]-1) this->nodesubsize[3] += IOFFSET; - if(datain.mygrid->xproc[1] == datain.mygrid->nproc[1]-1) this->nodesubsize[2] += JOFFSET; - if(datain.mygrid->xproc[2] == datain.mygrid->nproc[2]-1) this->nodesubsize[1] += KOFFSET; + if(data->mygrid->xproc[0] == data->mygrid->nproc[0]-1) this->nodesubsize[3] += IOFFSET; + if(data->mygrid->xproc[1] == data->mygrid->nproc[1]-1) this->nodesubsize[2] += JOFFSET; + if(data->mygrid->xproc[2] == data->mygrid->nproc[2]-1) this->nodesubsize[1] += KOFFSET; // Allocate a node and cell views on the host node_coord = IdefixHostArray4D("XdmfNodeCoord", nodesubsize[0], @@ -153,13 +178,13 @@ void Xdmf::Init(Input &input, DataBlock &datain) { for (int32_t k = 0; k < nodesubsize[1]; k++) { for (int32_t j = 0; j < nodesubsize[2]; j++) { for (int32_t i = 0; i < nodesubsize[3]; i++) { - D_EXPAND( x1 = data.xl[IDIR](i + grid.nghost[IDIR] ); , - x2 = data.xl[JDIR](j + grid.nghost[JDIR]); , - x3 = data.xl[KDIR](k + grid.nghost[KDIR]); ) + D_EXPAND( x1 = grid.xl[IDIR](i + data->gbeg[IDIR]); , + x2 = grid.xl[JDIR](j + data->gbeg[JDIR]); , + x3 = grid.xl[KDIR](k + data->gbeg[KDIR]); ) if ( (k<(cellsubsize[1])) && (j<(cellsubsize[2])) && (i<(cellsubsize[3])) ) { - D_EXPAND( x1_cell = data.x[IDIR](i + grid.nghost[IDIR] ); , - x2_cell = data.x[JDIR](j + grid.nghost[JDIR]); , - x3_cell = data.x[KDIR](k + grid.nghost[KDIR]); ) + D_EXPAND( x1_cell = grid.x[IDIR](i + data->gbeg[IDIR]); , + x2_cell = grid.x[JDIR](j + data->gbeg[JDIR]); , + x3_cell = grid.x[KDIR](k + data->gbeg[KDIR]); ) } #if (GEOMETRY == CARTESIAN) || (GEOMETRY == CYLINDRICAL) node_coord(0,k,j,i) = x1; @@ -222,44 +247,40 @@ void Xdmf::Init(Input &input, DataBlock &datain) { // XDMF assumes Fortran array ordering, hence arrays dimensions are filled backwards // So ordering is 3D/1D: Z-Y-X and 2D: Y-X-Z // offset in the destination array - this->mpi_data_start[dir] = datain.gbeg[2-dir]-grid.nghost[2-dir]; + this->mpi_data_start[dir] = data->gbeg[2-dir]-grid.nghost[2-dir]; this->mpi_data_size[dir] = grid.np_int[2-dir]; - this->mpi_data_subsize[dir] = datain.np_int[2-dir]; + this->mpi_data_subsize[dir] = data->np_int[2-dir]; } #elif (DIMENSIONS == 2) for(int dir = 0; dir < DIMENSIONS ; dir++) { // XDMF assumes Fortran array ordering, hence arrays dimensions are filled backwards // So ordering is 3D/1D: Z-Y-X and 2D: Y-X-Z // offset in the destination array - this->mpi_data_start[dir] = datain.gbeg[DIMENSIONS-dir-1]-grid.nghost[DIMENSIONS-dir-1]; + this->mpi_data_start[dir] = data->gbeg[DIMENSIONS-dir-1]-grid.nghost[DIMENSIONS-dir-1]; this->mpi_data_size[dir] = grid.np_int[DIMENSIONS-dir-1]; - this->mpi_data_subsize[dir] = datain.np_int[DIMENSIONS-dir-1]; + this->mpi_data_subsize[dir] = data->np_int[DIMENSIONS-dir-1]; } for(int dir = DIMENSIONS; dir < 3 ; dir++) { // XDMF assumes Fortran array ordering, hence arrays dimensions are filled backwards // So ordering is 3D/1D: Z-Y-X and 2D: Y-X-Z // offset in the destination array - this->mpi_data_start[dir] = datain.gbeg[dir]-grid.nghost[dir]; + this->mpi_data_start[dir] = data->gbeg[dir]-grid.nghost[dir]; this->mpi_data_size[dir] = grid.np_int[dir]; - this->mpi_data_subsize[dir] = datain.np_int[dir]; + this->mpi_data_subsize[dir] = data->np_int[dir]; } #endif #endif } -int Xdmf::Write(DataBlock &datain, Output &output) { +int Xdmf::Write() { idfx::pushRegion("Xdmf::Write"); - std::string filename; - std::string filename_xmf; + std::filesystem::path filename; + std::filesystem::path filename_xmf; hid_t err; - idfx::cout << "Xdmf: Write file n " << xdmfFileNumber << "..." << std::flush; - timer.reset(); // Create a copy of the dataBlock on Host, and sync it. - DataBlockHost data(datain); - data.SyncFromDevice(); #if DIMENSIONS == 1 int tot_dim = 1; @@ -276,10 +297,10 @@ int Xdmf::Write(DataBlock &datain, Output &output) { std::string extension = ".dbl"; #endif ssxdmfFileNum << std::setfill('0') << std::setw(4) << xdmfFileNumber; - ssfileName << "./data." << ssxdmfFileNum.str() << extension << ".h5"; - ssfileNameXmf << "./data." << ssxdmfFileNum.str() << extension << ".xmf"; - filename = ssfileName.str(); - filename_xmf = ssfileNameXmf.str(); + ssfileName << "data." << ssxdmfFileNum.str() << extension << ".h5"; + ssfileNameXmf << "data." << ssxdmfFileNum.str() << extension << ".xmf"; + filename = outputDirectory/ssfileName.str(); + filename_xmf = outputDirectory/ssfileNameXmf.str(); #ifdef WITH_MPI hid_t file_access = H5Pcreate(H5P_FILE_ACCESS); @@ -299,7 +320,7 @@ int Xdmf::Write(DataBlock &datain, Output &output) { ssgroup_name << "/Timestep_" << xdmfFileNumber; hid_t timestep = H5Gcreate(fileHdf, ssgroup_name.str().c_str(), 0); - WriteHeader(fileHdf, filename, filename_xmf, datain.t, timestep, group_fields); + WriteHeader(fileHdf, ssfileName.str(), filename_xmf, data->t, timestep, group_fields); /* ------------------------------------ write cell-centered field data @@ -363,58 +384,20 @@ int Xdmf::Write(DataBlock &datain, Output &output) { err = H5Sselect_hyperslab(memspace, H5S_SELECT_SET, offset, stride, field_data_subsize, NULL); // Write field one by one - for(int nv = 0 ; nv < NVAR ; nv++) { - for(int k = data.beg[KDIR]; k < data.end[KDIR] ; k++ ) { - for(int j = data.beg[JDIR]; j < data.end[JDIR] ; j++ ) { - for(int i = data.beg[IDIR]; i < data.end[IDIR] ; i++ ) { - vect3D[i-data.beg[IDIR] + (j-data.beg[JDIR])*nx1loc + (k-data.beg[KDIR])*nx1loc*nx2loc] - = static_cast(data.Vc(nv,k,j,i)); - /* field_data(i-data.beg[IDIR],j-data.beg[JDIR],k-data.beg[KDIR]) - = static_cast(data.Vc(nv,k,j,i)); */ + for(auto const& [name, scalar] : xdmfScalarMap) { + auto Vcin = scalar.GetHostField(); + for(int k = data->beg[KDIR]; k < data->end[KDIR] ; k++ ) { + for(int j = data->beg[JDIR]; j < data->end[JDIR] ; j++ ) { + for(int i = data->beg[IDIR]; i < data->end[IDIR] ; i++ ) { + vect3D[i-data->beg[IDIR] + (j-data->beg[JDIR])*nx1loc + (k-data->beg[KDIR])*nx1loc*nx2loc] + = static_cast(Vcin(k,j,i)); } } } - WriteScalar(vect3D, datain.hydro.VcName[nv], field_data_size, filename, filename_xmf, + WriteScalar(vect3D, name, field_data_size, ssfileName.str(), filename_xmf, memspace, dataspace, plist_id_mpiio, static_cast(group_fields)); } - // Write user-defined variables (when required by output) - if(output.userDefVariablesEnabled) { - // Walk the map and make an output for each key of the map - // (and we thank c++11 for its cute way of doing this) - for(auto const &variable : output.userDefVariables) { - for(int k = data.beg[KDIR]; k < data.end[KDIR] ; k++ ) { - for(int j = data.beg[JDIR]; j < data.end[JDIR] ; j++ ) { - for(int i = data.beg[IDIR]; i < data.end[IDIR] ; i++ ) { - vect3D[i-data.beg[IDIR] + (j-data.beg[JDIR])*nx1loc + (k-data.beg[KDIR])*nx1loc*nx2loc] - = static_cast(variable.second(k,j,i)); - /* field_data(i-data.beg[IDIR],j-data.beg[JDIR],k-data.beg[KDIR]) - = static_cast(variable.second(k,j,i)); */ - } - } - } - WriteScalar(vect3D, variable.first, field_data_size, filename, filename_xmf, - memspace, dataspace, plist_id_mpiio, static_cast(group_fields)); - } - } - - // Write vector potential if we're using this - #ifdef EVOLVE_VECTOR_POTENTIAL - for(int nv = 0 ; nv <= AX3e ; nv++) { - for(int k = data.beg[KDIR]; k < data.end[KDIR] ; k++ ) { - for(int j = data.beg[JDIR]; j < data.end[JDIR] ; j++ ) { - for(int i = data.beg[IDIR]; i < data.end[IDIR] ; i++ ) { - vect3D[i-data.beg[IDIR] + (j-data.beg[JDIR])*nx1loc + (k-data.beg[KDIR])*nx1loc*nx2loc] - = static_cast(data.Ve(nv,k,j,i)); - /* field_data(i-data.beg[IDIR],j-data.beg[JDIR],k-data.beg[KDIR]) - = static_cast(data.Ve(nv,k,j,i)); */ - } - } - } - WriteScalar(vect3D, datain.hydro.VeName[nv], field_data_size, filename, filename_xmf, - memspace, dataspace, plist_id_mpiio, static_cast(group_fields)); - } - #endif - WriteFooter(filename, filename_xmf); + WriteFooter(ssfileName.str(), filename_xmf); #ifdef WITH_MPI H5Pclose(plist_id_mpiio); @@ -532,7 +515,8 @@ void Xdmf::WriteHeader( H5Sclose(unit_info); dimstr = 1; - ssheader << "Idefix " << GITVERSION << " XDMF Data"; + + ssheader << "Idefix " << IDEFIX_VERSION << " XDMF Data"; strspace = H5Screate_simple(1, &dimstr, NULL); string_type = H5Tcopy(H5T_C_S1); H5Tset_size(string_type, strlen( ssheader.str().c_str() )); diff --git a/src/output/xdmf.hpp b/src/output/xdmf.hpp index 57335219..b2fb6628 100644 --- a/src/output/xdmf.hpp +++ b/src/output/xdmf.hpp @@ -8,10 +8,11 @@ #ifndef OUTPUT_XDMF_HPP_ #define OUTPUT_XDMF_HPP_ #include +#include +#include #include "idefix.hpp" #include "input.hpp" -#include "dataBlock.hpp" -#include "dataBlockHost.hpp" +#include "scalarField.hpp" #define H5_USE_16_API #include "hdf5.h" @@ -26,15 +27,24 @@ // Forward class declaration class Output; +class DataBlock; class Xdmf { friend class Dump; public: - void Init(Input &, DataBlock &); // init XDMF object - int Write(DataBlock &, Output &); // Create a XDMF from the current DataBlock + Xdmf(Input &, DataBlock *); // init XDMF object + int Write(); // Create a XDMF from the current DataBlock + + template + void RegisterVariable(T&, std::string, int var = -1); private: + // Parent datablock + DataBlock *data; + + // List of variables to be written to vtk files + std::map xdmfScalarMap; int xdmfFileNumber = 0; int periodicity[3]; @@ -70,6 +80,8 @@ class Xdmf { int cellsize[4]; int cellsubsize[4]; + // output directory + std::filesystem::path outputDirectory; #ifdef WITH_MPI int mpi_data_start[3]; @@ -98,4 +110,15 @@ class Xdmf { hid_t & , hid_t & ); }; + +template +void Xdmf::RegisterVariable(T& in, std::string name, int var) { + // if var>0, the caller provided explicitely an index + if constexpr(std::is_same>::value || + std::is_same>::value) { + xdmfScalarMap.emplace(name, ScalarField(in) ); + } else { + xdmfScalarMap.emplace(name, ScalarField(in, var)); + } +} #endif // OUTPUT_XDMF_HPP_ diff --git a/src/physics.hpp b/src/physics.hpp new file mode 100644 index 00000000..e9b29c45 --- /dev/null +++ b/src/physics.hpp @@ -0,0 +1,54 @@ +// *********************************************************************************** +// Idefix MHD astrophysical code +// Copyright(C) Geoffroy R. J. Lesur +// and other code contributors +// Licensed under CeCILL 2.1 License, see COPYING for more information +// *********************************************************************************** + +#ifndef PHYSICS_HPP_ +#define PHYSICS_HPP_ + +#include "idefix.hpp" + +// Physics type +// The default Physics class +struct DefaultPhysics { + static constexpr bool dust{false}; + #if HAVE_ENERGY == 1 + static constexpr bool pressure{true}; + #else + static constexpr bool pressure{false}; + #endif + #ifdef ISOTHERMAL + static constexpr bool isothermal{true}; + #else + static constexpr bool isothermal{false}; + #endif + static constexpr bool eos = isothermal || pressure; // whether we have a eos + #if MHD == YES + static constexpr bool mhd{true}; + static constexpr int nvar{1+2*COMPONENTS + (pressure?1:0)}; + #else + static constexpr bool mhd{false}; + static constexpr int nvar{1+COMPONENTS + (pressure?1:0)}; + #endif + + // prefix + static constexpr std::string_view prefix = "Hydro"; +}; + +// Some Dust +struct DustPhysics { + static constexpr bool dust{true}; + static constexpr bool pressure{false}; + static constexpr bool isothermal{false}; + static constexpr bool eos{false}; + + static constexpr bool mhd{false}; + static constexpr int nvar{1+COMPONENTS}; + + // prefix + static constexpr std::string_view prefix = "Dust"; +}; + +#endif // PHYSICS_HPP_ diff --git a/src/profiler.cpp b/src/profiler.cpp index 8dffdb8e..04ff5f5c 100644 --- a/src/profiler.cpp +++ b/src/profiler.cpp @@ -5,11 +5,18 @@ // Licensed under CeCILL 2.1 License, see COPYING for more information // *********************************************************************************** +#include +#include #include // NOLINT [build/c++11] +#include +#include + #include "idefix.hpp" #include "profiler.hpp" -// Kokkos Profiler hooks +///////////////////////////// +// Kokkos Profiler hooks (needed to track memory allocation) +///////////////////////////// extern "C" void kokkosp_allocate_data(const Kokkos_Profiling_SpaceHandle space, const char* label, const void* const ptr, const uint64_t size) { @@ -46,7 +53,9 @@ const char* label, const void* const ptr, const uint64_t size) { idfx::prof.spaceSize[space_i] -= size; } - +/////////////////////////////////// +// Profiler function definitions // +/////////////////////////////////// void idfx::Profiler::Init() { idfx::pushRegion("Profiler::Init"); @@ -60,15 +69,135 @@ void idfx::Profiler::Init() { Kokkos::Tools::Experimental::set_allocate_data_callback(&kokkosp_allocate_data); Kokkos::Tools::Experimental::set_deallocate_data_callback(&kokkosp_deallocate_data); + // Init region + + idfx::popRegion(); } void idfx::Profiler::Show() { - idfx::pushRegion("Profiler::Show"); + double usedMemory{-1}; + + // follow ISO/IEC 80000 + constexpr int nUnits = 5; + const std::array units {"B", "KB", "MB", "GB", "TB"}; for(int i=0; i < this->numSpaces ; i++) { - idfx::cout << "Profiler: maximum memory usage for " << this->spaceName[i] << " memory space: " - << this->spaceMax[i]/(1024.0*1024.0) << " MB." << std::endl; + usedMemory = this->spaceMax[i]; + int count{0}; + while(count < nUnits && usedMemory/1024 >= 1) { + usedMemory /= 1024; + ++count; + } + idfx::cout << "Profiler: maximum memory usage for " << this->spaceName[i]; + idfx::cout << " memory space: " << usedMemory << " " << units[count] << std::endl; + } + + if(perfEnabled) { + // Show performance results + rootRegion.Stop(); + idfx::cout << "Profiler: performance results: " << std::endl; + idfx::cout << "-------------------------------------------------------------------------------"; + idfx::cout << std::endl; + idfx::cout << " <% of total time> <% of self time> "; + idfx::cout << std::endl; + idfx::cout << "-------------------------------------------------------------------------------"; + idfx::cout << std::endl; + rootRegion.Show(rootRegion.GetTimer()); + idfx::cout << "-------------------------------------------------------------------------------"; + idfx::cout << std::endl; + idfx::cout << "Profiler: end of performance profiling report." << std::endl; + } +} + +void idfx::Profiler::EnablePerformanceProfiling() { + currentRegion = &rootRegion; + rootRegion.Start(); + perfEnabled = true; +} + + +/////////////////////////////////// +// Region functions definitions // +/////////////////////////////////// + + +idfx::Region::Region(Region *parent, std::string name, int level) { + this->parent = parent; + this->name = name; + this->level = level; +} + +idfx::Region::Region() { + this->parent = nullptr; + this->name = std::string("Main"); + this->level = 0; +} + +idfx::Region::~Region() { + if(!isLeaf) { + // Delete all the children manually, since these are pointers + for( auto &it : children) { + delete it.second; + } + } +} + +void idfx::Region::Start() { + this->nCalls++; + this->timer.reset(); +} + +double idfx::Region::GetTimer() { + return this->myTime; +} + +void idfx::Region::Stop() { + this->myTime += this->timer.seconds(); +} + +idfx::Region * idfx::Region::GetChild(std::string name) { + isLeaf=false; + if(this->children.find(name) == this->children.end()) { + // No children found + this->children[name] = new Region(this, name, this->level+1); + } + return this->children[name]; +} + +bool idfx::Region::Compare(Region * r1, Region * r2) { + return r1->GetTimer() > r2->GetTimer(); +} + +void idfx::Region::Show(double totTime) { + // Compute time of all the children + double childTime = 0; + + if(!isLeaf) { + for( auto &it : children) { + childTime += it.second->GetTimer(); + } + } + for(int i = 0 ; i < (this->level) ; i++) { + idfx::cout << "| "; + } + + idfx::cout << "|-> " << std::scientific << std::setprecision(2) << this->myTime << " sec " + << std::fixed << std::setprecision(1) + << this->myTime/totTime*100 << "% " + << (this->myTime-childTime)/this->myTime*100 << "% " + << this->nCalls << " " + << this->name << std::endl; + if(!isLeaf) { + // Sort the children + std::vector sorted; + for( auto &it : this->children) { + sorted.push_back(it.second); + } + + std::sort(sorted.begin(), sorted.end(), this->Compare); + for( auto &it : sorted) { + it->Show(totTime); + } } - idfx::popRegion(); } diff --git a/src/profiler.hpp b/src/profiler.hpp index f055cea6..49814dd3 100644 --- a/src/profiler.hpp +++ b/src/profiler.hpp @@ -8,7 +8,9 @@ #ifndef PROFILER_HPP_ #define PROFILER_HPP_ +#include #include // NOLINT [build/c++11] +#include namespace idfx { @@ -16,17 +18,50 @@ struct SpaceHandle { char name[64]; }; +// Region is a helper class to Profiler +// it used to generate a tree of the regions encountered while running +// and produce a performance report. +class Region { + public: + Region(Region *parent, std::string name, int level); + Region(); + ~Region(); + void Start(); + void Stop(); + void Show(double ); + Region* GetChild(std::string name); + double GetTimer(); + static bool Compare(Region *, Region *); + bool isLeaf{true}; + std::string name; + std::map children; + Region *parent; + int level; + private: + Kokkos::Timer timer; + double myTime{0}; + int64_t nCalls{0}; +}; + + class Profiler { public: void Init(); void Show(); + void EnablePerformanceProfiling(); int numSpaces; int64_t spaceSize[16]; int64_t spaceMax[16]; char spaceName[16][64]; std::mutex m; + + bool perfEnabled{false}; + Region rootRegion; + Region *currentRegion; }; + + }// namespace idfx #endif // PROFILER_HPP_ diff --git a/src/reduce.hpp b/src/reduce.hpp index 6134967b..17ba6b04 100644 --- a/src/reduce.hpp +++ b/src/reduce.hpp @@ -23,8 +23,15 @@ inline void idefix_reduce(const std::string & NAME, const int & IB, const int & IE, Function function, Reducer redFunction) { + #ifdef DEBUG + idfx::pushRegion("idefix_reduce("+NAME+")"); + #endif Kokkos::parallel_reduce(NAME, Kokkos::RangePolicy<>(IB,IE), function, redFunction); + #ifdef DEBUG + Kokkos::fence(); + idfx::popRegion(); + #endif } // 2D default loop pattern @@ -34,11 +41,20 @@ inline void idefix_reduce(const std::string & NAME, const int & IB, const int & IE, Function function, Reducer redFunction) { + #ifdef DEBUG + idfx::pushRegion("idefix_reduce("+NAME+")"); + #endif + // We only implement MDRange reductions here since the other implementations are too // complicated to be implemented for any reduction operator on any class Kokkos::parallel_reduce(NAME, Kokkos::MDRangePolicy> ({JB,IB},{JE,IE}), function, redFunction); + + #ifdef DEBUG + Kokkos::fence(); + idfx::popRegion(); + #endif } // 3D default loop pattern @@ -51,9 +67,17 @@ inline void idefix_reduce(const std::string & NAME, Reducer redFunction) { // We only implement MDRange reductions here since the other implementations are too // complicated to be implemented for any reduction operator on any class + #ifdef DEBUG + idfx::pushRegion("idefix_reduce("+NAME+")"); + #endif Kokkos::parallel_reduce(NAME, Kokkos::MDRangePolicy> ({KB,JB,IB},{KE,JE,IE}), function, redFunction); + + #ifdef DEBUG + Kokkos::fence(); + idfx::popRegion(); + #endif } // 4D default loop pattern @@ -67,9 +91,17 @@ inline void idefix_reduce(const std::string & NAME, Reducer redFunction) { // We only implement MDRange reductions here since the other implementations are too // complicated to be implemented for any reduction operator on any class + #ifdef DEBUG + idfx::pushRegion("idefix_reduce("+NAME+")"); + #endif Kokkos::parallel_reduce(NAME, Kokkos::MDRangePolicy> ({NB,KB,JB,IB},{NE,KE,JE,IE}), function, redFunction); + + #ifdef DEBUG + Kokkos::fence(); + idfx::popRegion(); + #endif } #endif // REDUCE_HPP_ diff --git a/src/rkl/CMakeLists.txt b/src/rkl/CMakeLists.txt index f6058500..245fe415 100644 --- a/src/rkl/CMakeLists.txt +++ b/src/rkl/CMakeLists.txt @@ -1,4 +1,3 @@ target_sources(idefix - PUBLIC ${CMAKE_CURRENT_LIST_DIR}/rkl.cpp PUBLIC ${CMAKE_CURRENT_LIST_DIR}/rkl.hpp ) diff --git a/src/rkl/rkl.cpp b/src/rkl/rkl.cpp deleted file mode 100644 index 125efd1f..00000000 --- a/src/rkl/rkl.cpp +++ /dev/null @@ -1,817 +0,0 @@ -// *********************************************************************************** -// Idefix MHD astrophysical code -// Copyright(C) Geoffroy R. J. Lesur -// and other code contributors -// Licensed under CeCILL 2.1 License, see COPYING for more information -// *********************************************************************************** - -#include -#include - -#include "rkl.hpp" -#include "dataBlock.hpp" -#include "hydro.hpp" -#include "calcParabolicFlux.hpp" - -#ifndef RKL_ORDER - #define RKL_ORDER 2 -#endif - - -void RKLegendre::AddVariable(int var, std::vector &varListHost ) { - bool haveit{false}; - - // Check whether we have this variable in the list - for(int i = 0 ; i < varListHost.size() ; i++) { - if(varListHost[i] == var) haveit=true; - } - - // We don't have it, then add it to the list - if(!haveit) { - varListHost.push_back(var); - } -} - -// Copy just the variables required by the RK scheme -void RKLegendre::Copy(IdefixArray4D &out, IdefixArray4D &in) { - IdefixArray1D vars = this->varList; - - idefix_for("RKL_Copy", - 0, nvarRKL, - 0, data->np_tot[KDIR], - 0, data->np_tot[JDIR], - 0, data->np_tot[IDIR], - KOKKOS_LAMBDA(int n, int k, int j, int i) { - const int var = vars(n); - out(var,k,j,i) = in(var,k,j,i); - }); -} - - -void RKLegendre::Init(Input &input, DataBlock &datain) { - idfx::pushRegion("RKLegendre::Init"); - - // Save the datablock to which we are attached from now on - this->data = &datain; - - cfl_rkl = input.GetOrSet ("RKL","cfl",0, 0.5); - - // By default check nans in debug mode - #ifdef DEBUG - this->checkNan = true; - #endif - - this->checkNan = input.GetOrSet("RKL","check_nan",0, this->checkNan); - - rmax_par = 100.0; - - // Make a list of variables - - std::vector varListHost; - // Create a list of variables - // Viscosity - if(data->hydro.viscosityStatus.isRKL) { - haveVc = true; - EXPAND( AddVariable(MX1, varListHost); , - AddVariable(MX2, varListHost); , - AddVariable(MX3, varListHost); ) - - #if HAVE_ENERGY - AddVariable(ENG, varListHost); - #endif - } - // Thermal diffusion - #if HAVE_ENERGY - if(data->hydro.thermalDiffusionStatus.isRKL) { - haveVc = true; - AddVariable(ENG, varListHost); - } - #endif - // Ambipolar diffusion - if(data->hydro.ambipolarStatus.isRKL || data->hydro.resistivityStatus.isRKL) { - #if COMPONENTS == 3 && DIMENSIONS < 3 - haveVc = true; - AddVariable(BX3, varListHost); - #endif - #if COMPONENTS >= 2 && DIMENSIONS < 2 - haveVc = true; - AddVariable(BX2, varListHost); - #endif - #if HAVE_ENERGY - haveVc = true; - AddVariable(ENG, varListHost); - #endif - haveVs = true; - } - - // Copy the list on the device - varList = idfx::ConvertVectorToIdefixArray(varListHost); - nvarRKL = varListHost.size(); - - #ifdef WITH_MPI - mpi.Init(datain.mygrid, varListHost, datain.nghost.data(), datain.np_int.data(), haveVs); - #endif - - - // Variable allocation - - dU = IdefixArray4D("RKL_dU", NVAR, - data->np_tot[KDIR], data->np_tot[JDIR], data->np_tot[IDIR]); - dU0 = IdefixArray4D("RKL_dU0", NVAR, - data->np_tot[KDIR], data->np_tot[JDIR], data->np_tot[IDIR]); - Uc0 = IdefixArray4D("RKL_Uc0", NVAR, - data->np_tot[KDIR], data->np_tot[JDIR], data->np_tot[IDIR]); - Uc1 = IdefixArray4D("RKL_Uc1", NVAR, - data->np_tot[KDIR], data->np_tot[JDIR], data->np_tot[IDIR]); - - if(haveVs) { - #ifdef EVOLVE_VECTOR_POTENTIAL - dA = IdefixArray4D("RKL_dA", AX3e+1, - data->np_tot[KDIR]+KOFFSET, - data->np_tot[JDIR]+JOFFSET, - data->np_tot[IDIR]+IOFFSET); - dA0 = IdefixArray4D("RKL_dA0", AX3e+1, - data->np_tot[KDIR]+KOFFSET, - data->np_tot[JDIR]+JOFFSET, - data->np_tot[IDIR]+IOFFSET); - Ve0 = IdefixArray4D("RKL_Ve0", AX3e+1, - data->np_tot[KDIR]+KOFFSET, - data->np_tot[JDIR]+JOFFSET, - data->np_tot[IDIR]+IOFFSET); - Ve1 = IdefixArray4D("RKL_Ve1", AX3e+1, - data->np_tot[KDIR]+KOFFSET, - data->np_tot[JDIR]+JOFFSET, - data->np_tot[IDIR]+IOFFSET); - #else - dB = IdefixArray4D("RKL_dB", DIMENSIONS, - data->np_tot[KDIR]+KOFFSET, - data->np_tot[JDIR]+JOFFSET, - data->np_tot[IDIR]+IOFFSET); - dB0 = IdefixArray4D("RKL_dB0", DIMENSIONS, - data->np_tot[KDIR]+KOFFSET, - data->np_tot[JDIR]+JOFFSET, - data->np_tot[IDIR]+IOFFSET); - Vs0 = IdefixArray4D("RKL_Vs0", DIMENSIONS, - data->np_tot[KDIR]+KOFFSET, - data->np_tot[JDIR]+JOFFSET, - data->np_tot[IDIR]+IOFFSET); - Vs1 = IdefixArray4D("RKL_Vs1", DIMENSIONS, - data->np_tot[KDIR]+KOFFSET, - data->np_tot[JDIR]+JOFFSET, - data->np_tot[IDIR]+IOFFSET); - #endif - } - - idfx::popRegion(); -} - -void RKLegendre::ShowConfig() { - #if RKL_ORDER == 1 - idfx::cout << "RKLegendre: 1st order scheme ENABLED." << std::endl; - #elif RKL_ORDER == 2 - idfx::cout << "RKLegendre: 2nd order scheme ENABLED." << std::endl; - #else - IDEFIX_ERROR("Unknown RKL scheme order"); - #endif - idfx::cout << "RKLegendre: RKL cfl set to " << cfl_rkl << "." << std::endl; - if(haveVc) { - idfx::cout << "RKLegendre: will evolve cell-centered fields Vc." << std::endl; - } - if(haveVs) { - idfx::cout << "RKLegendre: will evolve face-centered fields Vs." << std::endl; - } - if(checkNan) { - idfx::cout << "RKLegendre: will check consistency of solution in the integrator (slow!)." - << std::endl; - } -} - -void RKLegendre::Cycle() { - idfx::pushRegion("RKLegendre::Cycle"); - - IdefixArray4D dU = this->dU; - IdefixArray4D dU0 = this->dU0; - IdefixArray4D Uc = data->hydro.Uc; - IdefixArray4D Uc0 = this->Uc0; - IdefixArray4D Uc1 = this->Uc1; - - IdefixArray4D dB = this->dB; - IdefixArray4D dB0 = this->dB0; - IdefixArray4D Vs = data->hydro.Vs; - IdefixArray4D Vs0 = this->Vs0; - IdefixArray4D Vs1 = this->Vs1; - - #ifdef EVOLVE_VECTOR_POTENTIAL - IdefixArray4D dA = this->dA; - IdefixArray4D dA0 = this->dA0; - IdefixArray4D Ve = data->hydro.Ve; - IdefixArray4D Ve0 = this->Ve0; - IdefixArray4D Ve1 = this->Ve1; - #endif - - IdefixArray1D varList = this->varList; - real time = data->t; - - real dt_hyp = data->dt; - - // Tell the datablock that we're performing the RKL cycle - data->rklCycle = true; - - // first RKL stage - stage = 1; - - // Apply Boundary conditions on the full set of variables - data->hydro.boundary.SetBoundaries(time); - - // Convert current state into conservative variable - data->hydro.ConvertPrimToCons(); - - // Coarsen the conservative variables if needed - if(data->haveGridCoarsening) { - data->Coarsen(); - } - - // Store the result in Uc0 - Copy(Uc0,Uc); - if(haveVs) { - #ifdef EVOLVE_VECTOR_POTENTIAL - Kokkos::deep_copy(Ve0,Ve); - #else - Kokkos::deep_copy(Vs0,Vs); - #endif - } - - // evolve RKL stage - EvolveStage(time); - - ComputeDt(); - - Copy(dU0,dU); - - if(haveVs) { - #ifdef EVOLVE_VECTOR_POTENTIAL - Kokkos::deep_copy(dA0,dA); - #else - Kokkos::deep_copy(dB0,dB); - #endif - } - - // Compute number of RKL steps - real nrkl; - real scrh = dt_hyp/dt; -#if RKL_ORDER == 1 - // Solution of quadratic Eq. - // 2*dt_hyp/dt_exp = s^2 + s - nrkl = 4.0*scrh / (1.0 + std::sqrt(1.0 + 8.0*scrh)); -#elif RKL_ORDER == 2 - // Solution of quadratic Eq. - // 4*dt_hyp/dt_exp = s^2 + s - 2 - nrkl = 4.0*(1.0 + 2.0*scrh) - / (1.0 + sqrt(9.0 + 16.0*scrh)); -#else - //#error Invalid RKL_ORDER -#endif - int rklstages = 1 + floor(nrkl); - - // Compute coefficients - real w1, mu_tilde_j; -#if RKL_ORDER == 1 - w1 = 2.0/(rklstages*rklstages + rklstages); - mu_tilde_j = w1; -#elif RKL_ORDER == 2 - real b_j, b_jm1, b_jm2, a_jm1; - w1 = 4.0/(rklstages*rklstages + rklstages - 2.0); - mu_tilde_j = w1/3.0; - - b_j = b_jm1 = b_jm2 = 1.0/3.0; - a_jm1 = 1.0 - b_jm1; -#endif - -#if RKL_ORDER == 1 - time = data->t + 0.5*dt_hyp*(stage*stage+stage)*w1; -#elif RKL_ORDER == 2 - time = data->t + 0.25*dt_hyp*(stage*stage+stage-2)*w1; -#endif - if(haveVc) { - idefix_for("RKL_Cycle_InitUc1", - 0, nvarRKL, - data->beg[KDIR],data->end[KDIR], - data->beg[JDIR],data->end[JDIR], - data->beg[IDIR],data->end[IDIR], - KOKKOS_LAMBDA (int n, int k, int j, int i) { - int nv = varList(n); - Uc1(nv,k,j,i) = Uc(nv,k,j,i); - Uc(nv,k,j,i) = Uc1(nv,k,j,i) + mu_tilde_j*dt_hyp*dU0(nv,k,j,i); - } - ); - } - if(haveVs) { - #ifdef EVOLVE_VECTOR_POTENTIAL - idefix_for("RKL_Cycle_InitVe1", - 0, AX3e+1, - data->beg[KDIR],data->end[KDIR]+KOFFSET, - data->beg[JDIR],data->end[JDIR]+JOFFSET, - data->beg[IDIR],data->end[IDIR]+IOFFSET, - KOKKOS_LAMBDA (int n, int k, int j, int i) { - Ve1(n,k,j,i) = Ve(n,k,j,i); - Ve(n,k,j,i) = Ve1(n,k,j,i) + mu_tilde_j*dt_hyp*dA0(n,k,j,i); - }); - data->hydro.emf.ComputeMagFieldFromA(Ve,Vs); - #else - idefix_for("RKL_Cycle_InitVs1", - 0, DIMENSIONS, - data->beg[KDIR],data->end[KDIR]+KOFFSET, - data->beg[JDIR],data->end[JDIR]+JOFFSET, - data->beg[IDIR],data->end[IDIR]+IOFFSET, - KOKKOS_LAMBDA (int n, int k, int j, int i) { - Vs1(n,k,j,i) = Vs(n,k,j,i); - Vs(n,k,j,i) = Vs1(n,k,j,i) + mu_tilde_j*dt_hyp*dB0(n,k,j,i); - }); - #endif - data->hydro.boundary.ReconstructVcField(Uc); - } - - // Coarsen conservative variables once they have been evolved - if(data->haveGridCoarsening) { - data->Coarsen(); - } - - // Convert current state into primitive variable - data->hydro.ConvertConsToPrim(); - if(checkNan) { - if(data->CheckNan()>0) { - throw std::runtime_error(std::string("Nan found during RKL stage 1")); - } - } - - real mu_j, nu_j, gamma_j; - // subStages loop - for(stage=2; stage <= rklstages ; stage++) { - //idfx::cout << "RKL: looping stages" << std::endl; - // compute RKL coefficients -#if RKL_ORDER == 1 - mu_j = (2.0*stage -1.0)/stage; - mu_tilde_j = w1*mu_j; - nu_j = -(stage -1.0)/stage; -#elif RKL_ORDER == 2 - mu_j = (2.0*stage -1.0)/stage * b_j/b_jm1; - mu_tilde_j = w1*mu_j; - gamma_j = -a_jm1*mu_tilde_j; - nu_j = -(stage -1.0)*b_j/(stage*b_jm2); - - b_jm2 = b_jm1; - b_jm1 = b_j; - a_jm1 = 1.0 - b_jm1; - b_j = 0.5*(stage*stage+3.0*stage)/(stage*stage+3.0*stage+2.0); -#endif - - // Apply Boundary conditions - this->SetBoundaries(time); - - // evolve RKL stage - EvolveStage(time); - if(haveVc) { - // update Uc - idefix_for("RKL_Cycle_UpdateUc", - 0, nvarRKL, - data->beg[KDIR],data->end[KDIR], - data->beg[JDIR],data->end[JDIR], - data->beg[IDIR],data->end[IDIR], - KOKKOS_LAMBDA (int n, int k, int j, int i) { - const int nv = varList(n); - real Y = mu_j*Uc(nv,k,j,i) + nu_j*Uc1(nv,k,j,i); - Uc1(nv,k,j,i) = Uc(nv,k,j,i); - #if RKL_ORDER == 1 - Uc(nv,k,j,i) = Y + dt_hyp*mu_tilde_j*dU(nv,k,j,i); - #elif RKL_ORDER == 2 - Uc(nv,k,j,i) = Y + (1.0 - mu_j - nu_j)*Uc0(nv,k,j,i) - + dt_hyp*mu_tilde_j*dU(nv,k,j,i) - + gamma_j*dt_hyp*dU0(nv,k,j,i); - #endif - }); - } - if(haveVs) { - #ifdef EVOLVE_VECTOR_POTENTIAL - // update Ve - idefix_for("RKL_Cycle_UpdateVe", - 0, AX3e+1, - data->beg[KDIR],data->end[KDIR]+KOFFSET, - data->beg[JDIR],data->end[JDIR]+JOFFSET, - data->beg[IDIR],data->end[IDIR]+IOFFSET, - KOKKOS_LAMBDA (int n, int k, int j, int i) { - real Y = mu_j*Ve(n,k,j,i) + nu_j*Ve1(n,k,j,i); - Ve1(n,k,j,i) = Ve(n,k,j,i); - #if RKL_ORDER == 1 - Ve(n,k,j,i) = Y + dt_hyp*mu_tilde_j*dA(n,k,j,i); - #elif RKL_ORDER == 2 - Ve(n,k,j,i) = Y + (1.0 - mu_j - nu_j)*Ve0(n,k,j,i) - + dt_hyp*mu_tilde_j*dA(n,k,j,i) - + gamma_j*dt_hyp*dA0(n,k,j,i); - #endif - }); - data->hydro.emf.ComputeMagFieldFromA(Ve,Vs); - #else - // update Vs - idefix_for("RKL_Cycle_UpdateVs", - 0, DIMENSIONS, - data->beg[KDIR],data->end[KDIR]+KOFFSET, - data->beg[JDIR],data->end[JDIR]+JOFFSET, - data->beg[IDIR],data->end[IDIR]+IOFFSET, - KOKKOS_LAMBDA (int n, int k, int j, int i) { - real Y = mu_j*Vs(n,k,j,i) + nu_j*Vs1(n,k,j,i); - Vs1(n,k,j,i) = Vs(n,k,j,i); - #if RKL_ORDER == 1 - Vs(n,k,j,i) = Y + dt_hyp*mu_tilde_j*dB(n,k,j,i); - #elif RKL_ORDER == 2 - Vs(n,k,j,i) = Y + (1.0 - mu_j - nu_j)*Vs0(n,k,j,i) - + dt_hyp*mu_tilde_j*dB(n,k,j,i) - + gamma_j*dt_hyp*dB0(n,k,j,i); - #endif - }); - #endif // EVOLVE_VECTOR_POTENTIAL - - data->hydro.boundary.ReconstructVcField(Uc); - } - - // Coarsen the flow if needed - if(data->haveGridCoarsening) { - data->Coarsen(); - } - // Convert current state into primitive variable - data->hydro.ConvertConsToPrim(); - - if(checkNan) { - if(data->CheckNan()>0) { - throw std::runtime_error(std::string("Nan found during RKL stage ")+std::to_string(stage)); - } - } - - // increment time -#if RKL_ORDER == 1 - time = data->t + 0.5*dt_hyp*(stage*stage+stage)*w1; -#elif RKL_ORDER == 2 - time = data->t + 0.25*dt_hyp*(stage*stage+stage-2)*w1; -#endif - } - - // Tell the datablock that we're done - data->rklCycle = false; - idfx::popRegion(); -} - - -void RKLegendre::ResetFlux() { - idfx::pushRegion("RKLegendre::ResetFlux"); - IdefixArray4D Flux = data->hydro.FluxRiemann; - IdefixArray1D vars = this->varList; - idefix_for("RKL_ResetFlux", - 0,nvarRKL, - 0,data->np_tot[KDIR], - 0,data->np_tot[JDIR], - 0,data->np_tot[IDIR], - KOKKOS_LAMBDA (int n, int k, int j, int i) { - const int nv = vars(n); - Flux(nv,k,j,i) = ZERO_F; - } - ); - idfx::popRegion(); -} - -void RKLegendre::ResetStage() { - idfx::pushRegion("RKLegendre::ResetStage"); - - IdefixArray4D dU = this->dU; - IdefixArray4D Flux = data->hydro.FluxRiemann; - #ifdef EVOLVE_VECTOR_POTENTIAL - IdefixArray4D dA = this->dA; - #else - IdefixArray4D dB = this->dB; - #endif - IdefixArray3D ex = data->hydro.emf.ex; - IdefixArray3D ey = data->hydro.emf.ey; - IdefixArray3D ez = data->hydro.emf.ez; - IdefixArray1D vars = this->varList; - IdefixArray3D invDt = data->hydro.InvDt; - int stage = this->stage; - int nvar = this->nvarRKL; - bool haveVs=this->haveVs; - - bool haveVc = this->haveVc || (this->stage ==1 ); - - - idefix_for("RKL_ResetStage", - 0,data->np_tot[KDIR], - 0,data->np_tot[JDIR], - 0,data->np_tot[IDIR], - KOKKOS_LAMBDA (int k, int j, int i) { - if(haveVc) { - for(int n = 0 ; n < nvar ; n++) { - const int nv = vars(n); - dU(nv,k,j,i) = ZERO_F; - } - } - if(stage == 1) - invDt(k,j,i) = ZERO_F; - - if(haveVs) { - #ifdef EVOLVE_VECTOR_POTENTIAL - for(int n=0; n < AX3e+1; n++) { - dA(n,k,j,i) = ZERO_F; - } - #else - for(int n=0; n < DIMENSIONS; n++) { - dB(n,k,j,i) = ZERO_F; - } - #endif - D_EXPAND( ez(k,j,i) = 0.0; , - , - ex(k,j,i) = 0.0; - ey(k,j,i) = 0.0; ) - } - }); - - idfx::popRegion(); -} - - -void RKLegendre::ComputeDt() { - idfx::pushRegion("RKLegendre::ComputeDt"); - - IdefixArray3D invDt = data->hydro.InvDt; - - real newinvdt = ZERO_F; - Kokkos::parallel_reduce("RKL_Timestep_reduction", - Kokkos::MDRangePolicy> - ({0,0,0},{data->end[KDIR],data->end[JDIR],data->end[IDIR]}), - KOKKOS_LAMBDA (int k, int j, int i, real &invdt) { - invdt = std::fmax(invDt(k,j,i), invdt); - }, - Kokkos::Max(newinvdt) - ); - -#ifdef WITH_MPI - if(idfx::psize>1) { - MPI_SAFE_CALL(MPI_Allreduce(MPI_IN_PLACE, &newinvdt, 1, realMPI, MPI_MAX, MPI_COMM_WORLD)); - } -#endif - - dt = 1.0/newinvdt; - dt = (cfl_rkl*dt)/2.0; // parabolic time step - - idfx::popRegion(); -} - -template void RKLegendre::LoopDir(real t) { - ResetFlux(); - - // CalcParabolicFlux - data->hydro.CalcParabolicFlux(t); - - // Calc Right Hand Side - CalcParabolicRHS(t); - - // Recursive: do next dimension - LoopDir(t); -} - -template<> void RKLegendre::LoopDir(real t) { - // Do nothing -} - -void RKLegendre::EvolveStage(real t) { - idfx::pushRegion("RKLegendre::EvolveStage"); - - ResetStage(); - - if(haveVs && data->hydro.needRKLCurrent) data->hydro.CalcCurrent(); - - // Loop on dimensions for the parabolic fluxes and RHS, starting from IDIR - if(haveVc || stage == 1) LoopDir(t); - - if(haveVs) { - data->hydro.emf.CalcNonidealEMF(t); - data->hydro.emf.EnforceEMFBoundary(); - real dt=1.0; - #ifdef EVOLVE_VECTOR_POTENTIAL - data->hydro.emf.EvolveVectorPotential(dt, this->dA); - #else - data->hydro.emf.EvolveMagField(t, dt, this->dB); - #endif - } - idfx::popRegion(); -} - -template -void RKLegendre::CalcParabolicRHS(real t) { - idfx::pushRegion("RKLegendre::CalcParabolicRHS"); - - IdefixArray4D Flux = data->hydro.FluxRiemann; - IdefixArray3D A = data->A[dir]; - IdefixArray3D dV = data->dV; - IdefixArray1D x1m = data->xl[IDIR]; - IdefixArray1D x1 = data->x[IDIR]; - IdefixArray1D sm = data->sinx2m; - IdefixArray1D rt = data->rt; - IdefixArray1D dmu = data->dmu; - IdefixArray1D s = data->sinx2; - IdefixArray1D dx = data->dx[dir]; - IdefixArray1D dx2 = data->dx[JDIR]; - IdefixArray3D invDt = data->hydro.InvDt; - IdefixArray3D dMax = data->hydro.dMax; - IdefixArray4D viscSrc = data->hydro.viscosity.viscSrc; - IdefixArray4D dU = this->dU; - IdefixArray1D varList = this->varList; - - bool haveViscosity = data->hydro.viscosityStatus.isRKL; - - int ioffset,joffset,koffset; - ioffset=joffset=koffset=0; - // Determine the offset along which we do the extrapolation - if(dir==IDIR) ioffset=1; - if(dir==JDIR) joffset=1; - if(dir==KDIR) koffset=1; - - - idefix_for("CalcTotalFlux", - 0, this->nvarRKL, - data->beg[KDIR],data->end[KDIR]+koffset, - data->beg[JDIR],data->end[JDIR]+joffset, - data->beg[IDIR],data->end[IDIR]+ioffset, - KOKKOS_LAMBDA (int n, int k, int j, int i) { - real Ax = A(k,j,i); - -#if GEOMETRY != CARTESIAN - if(Ax= 2) \ - || (GEOMETRY == CYLINDRICAL && COMPONENTS == 3) - if(dir==IDIR && nv==iMPHI) { - // Conserve angular momentum, hence flux is R*Vphi - Flux(iMPHI,k,j,i) = Flux(iMPHI,k,j,i) * FABS(x1m(i)); - } -#endif // GEOMETRY==POLAR OR CYLINDRICAL - -#if GEOMETRY == SPHERICAL && COMPONENTS == 3 - if(dir==IDIR && nv==iMPHI) { - Flux(iMPHI,k,j,i) = Flux(iMPHI,k,j,i) * FABS(x1m(i)); - } else if(dir==JDIR && nv==iMPHI) { - Flux(iMPHI,k,j,i) = Flux(iMPHI,k,j,i) * FABS(sm(j)); - } -#endif // GEOMETRY == SPHERICAL && COMPONENTS == 3 - } - ); - - - idefix_for("CalcRightHandSide", - 0, this->nvarRKL, - data->beg[KDIR],data->end[KDIR], - data->beg[JDIR],data->end[JDIR], - data->beg[IDIR],data->end[IDIR], - KOKKOS_LAMBDA (int n, int k, int j, int i) { - constexpr const int ioffset = (dir==IDIR) ? 1 : 0; - constexpr const int joffset = (dir==JDIR) ? 1 : 0; - constexpr const int koffset = (dir==KDIR) ? 1 : 0; - real rhs; - - const int nv = varList(n); - - rhs = - ( Flux(nv, k+koffset, j+joffset, i+ioffset) - - Flux(nv, k, j, i))/dV(k,j,i); - - // Viscosity source terms - if( haveViscosity && (nv-VX1 < COMPONENTS) && (nv-VX1>=0)) { - rhs += viscSrc(nv-VX1,k,j,i); - } - -#if GEOMETRY != CARTESIAN - #ifdef iMPHI - if((dir==IDIR) && (nv == iMPHI)) { - rhs /= x1(i); - } - #if (GEOMETRY == SPHERICAL) && (COMPONENTS == 3) - if((dir==JDIR) && (nv == iMPHI)) { - rhs /= FABS(s(j)); - } - #endif // GEOMETRY - // Nothing for KDIR - #endif // iMPHI -#endif // GEOMETRY != CARTESIAN - - // store the field components - dU(nv,k,j,i) += rhs; - }); - - // Compute hyperbolic timestep only if we're in the first stage of the RKL loop - if(stage==1) { - // Grid coarsening - bool haveGridCoarsening = false; - IdefixArray2D coarseningLevel; - - if(data->haveGridCoarsening) { - haveGridCoarsening = data->coarseningDirection[dir]; - coarseningLevel = data->coarseningLevel[dir]; - } - - idefix_for("CalcDt", - data->beg[KDIR],data->end[KDIR], - data->beg[JDIR],data->end[JDIR], - data->beg[IDIR],data->end[IDIR], - KOKKOS_LAMBDA (int k, int j, int i) { - constexpr const int ioffset = (dir==IDIR) ? 1 : 0; - constexpr const int joffset = (dir==JDIR) ? 1 : 0; - constexpr const int koffset = (dir==KDIR) ? 1 : 0; - // Compute dt from max signal speed - const int ig = ioffset*i + joffset*j + koffset*k; - real dl = dx(ig); - - if(haveGridCoarsening) { - int factor; - //factor = 2^(coarsening-1) - if(dir==IDIR) { - factor = 1 << (coarseningLevel(k,j) - 1); - } - if(dir==JDIR) { - factor = 1 << (coarseningLevel(k,i) - 1); - } - if(dir==KDIR) { - factor = 1 << (coarseningLevel(j,i) - 1); - } - dl = dl * factor; - } - #if GEOMETRY == POLAR - if (dir==JDIR) - dl = dl*x1(i); - - #elif GEOMETRY == SPHERICAL - if (dir==JDIR) - dl = dl*rt(i); - else - if (dir==KDIR) - dl = dl*rt(i)*dmu(j)/dx2(j); - #endif - - invDt(k,j,i) += 0.5 * std::fmax(dMax(k+koffset,j+joffset,i+ioffset), - dMax(k,j,i)) / (dl*dl); - }); - } - - idfx::popRegion(); -} - -void RKLegendre::SetBoundaries(real t) { - idfx::pushRegion("RKLegendre::SetBoundaries"); - if(data->haveGridCoarsening) { - data->hydro.CoarsenFlow(data->hydro.Vc); - #if MHD==YES - data->hydro.CoarsenMagField(data->hydro.Vs); - #endif - } - - // set internal boundary conditions - // Disabled since this might affect fields that are NOT updated - // by the MPI instance of RKLegendre - //if(data->hydro.boundary.haveInternalBoundary) - // data->hydro.boundary.internalBoundaryFunc(*data, t); - for(int dir=0 ; dir < DIMENSIONS ; dir++ ) { - // MPI Exchange data when needed - // We use the RKL instance MPI object to ensure that we only exchange the data - // solved by RKL - #ifdef WITH_MPI - if(data->mygrid->nproc[dir]>1) { - switch(dir) { - case 0: - this->mpi.ExchangeX1(data->hydro.Vc, data->hydro.Vs); - break; - case 1: - this->mpi.ExchangeX2(data->hydro.Vc, data->hydro.Vs); - break; - case 2: - this->mpi.ExchangeX3(data->hydro.Vc, data->hydro.Vs); - break; - } - } - #endif - data->hydro.boundary.EnforceBoundaryDir(t, dir); - #if MHD == YES - // Reconstruct the normal field component when using CT - if(haveVs) { - data->hydro.boundary.ReconstructNormalField(dir); - } - #endif - } // Loop on dimension ends - -#if MHD == YES - // Remake the cell-centered field. - if(haveVs) { - data->hydro.boundary.ReconstructVcField(data->hydro.Vc); - } -#endif - idfx::popRegion(); -} diff --git a/src/rkl/rkl.hpp b/src/rkl/rkl.hpp index 318b0de8..a2f6734f 100644 --- a/src/rkl/rkl.hpp +++ b/src/rkl/rkl.hpp @@ -8,18 +8,26 @@ #ifndef RKL_RKL_HPP_ #define RKL_RKL_HPP_ +#include #include + #include "idefix.hpp" #include "input.hpp" #include "dataBlock.hpp" +#include "viscosity.hpp" +#include "bragViscosity.hpp" #ifdef WITH_MPI #include "mpi.hpp" #endif +// Functors being used by RKL +template +struct RKLegendre_ResetStageFunctor; +template class RKLegendre { public: - void Init(Input &, DataBlock &); + RKLegendre(Input &, Fluid*); void Cycle(); void ResetStage(); void ResetFlux(); @@ -53,9 +61,11 @@ class RKLegendre { int stage{0}; private: + friend struct RKLegendre_ResetStageFunctor; void SetBoundaries(real); // Enforce boundary conditions on the variables solved by RKL DataBlock *data; + Fluid *hydro; #ifdef WITH_MPI Mpi mpi; // RKL-specific MPI layer @@ -71,4 +81,868 @@ class RKLegendre { template void LoopDir(real); // Dimensional loop }; +#include "fluid.hpp" +#include "calcParabolicFlux.hpp" + +#ifndef RKL_ORDER + #define RKL_ORDER 2 +#endif + + +template +void RKLegendre::AddVariable(int var, std::vector &varListHost ) { + bool haveit{false}; + + // Check whether we have this variable in the list + for(int i = 0 ; i < varListHost.size() ; i++) { + if(varListHost[i] == var) haveit=true; + } + + // We don't have it, then add it to the list + if(!haveit) { + varListHost.push_back(var); + } +} + +// Copy just the variables required by the RK scheme +template +void RKLegendre::Copy(IdefixArray4D &out, IdefixArray4D &in) { + IdefixArray1D vars = this->varList; + + idefix_for("RKL_Copy", + 0, nvarRKL, + 0, data->np_tot[KDIR], + 0, data->np_tot[JDIR], + 0, data->np_tot[IDIR], + KOKKOS_LAMBDA(int n, int k, int j, int i) { + const int var = vars(n); + out(var,k,j,i) = in(var,k,j,i); + }); +} + +template +RKLegendre::RKLegendre(Input &input, Fluid* hydroin) { + idfx::pushRegion("RKLegendre::Init"); + + // Save the datablock to which we are attached from now on + this->data = hydroin->data; + this->hydro = hydroin; + + cfl_rkl = input.GetOrSet ("RKL","cfl",0, 0.5); + rmax_par = input.GetOrSet ("RKL","rmax_par",0, 100.0); + + // By default check nans in debug mode + #ifdef DEBUG + this->checkNan = true; + #endif + + this->checkNan = input.GetOrSet("RKL","check_nan",0, this->checkNan); + + // Make a list of variables + + std::vector varListHost; + // Create a list of variables + // Viscosity + if(hydro->viscosityStatus.isRKL) { + haveVc = true; + EXPAND( AddVariable(MX1, varListHost); , + AddVariable(MX2, varListHost); , + AddVariable(MX3, varListHost); ) + + #if HAVE_ENERGY + AddVariable(ENG, varListHost); + #endif + } + // BragViscosity + if(hydro->bragViscosityStatus.isRKL) { + haveVc = true; + EXPAND( AddVariable(MX1, varListHost); , + AddVariable(MX2, varListHost); , + AddVariable(MX3, varListHost); ) + #if HAVE_ENERGY + AddVariable(ENG, varListHost); + #endif + } + + // Thermal diffusion + #if HAVE_ENERGY + if(hydro->thermalDiffusionStatus.isRKL) { + haveVc = true; + AddVariable(ENG, varListHost); + } + // Braginskii Thermal diffusion + if(hydro->bragThermalDiffusionStatus.isRKL) { + haveVc = true; + AddVariable(ENG, varListHost); + } + #endif + // Ambipolar diffusion + if(hydro->ambipolarStatus.isRKL || hydro->resistivityStatus.isRKL) { + #if COMPONENTS == 3 && DIMENSIONS < 3 + haveVc = true; + AddVariable(BX3, varListHost); + #endif + #if COMPONENTS >= 2 && DIMENSIONS < 2 + haveVc = true; + AddVariable(BX2, varListHost); + #endif + #if HAVE_ENERGY + haveVc = true; + AddVariable(ENG, varListHost); + #endif + haveVs = true; + } + + // Copy the list on the device + varList = idfx::ConvertVectorToIdefixArray(varListHost); + nvarRKL = varListHost.size(); + + #ifdef WITH_MPI + mpi.Init(data->mygrid, varListHost, data->nghost.data(), data->np_int.data(), haveVs); + #endif + + + // Variable allocation + + dU = IdefixArray4D("RKL_dU", NVAR, + data->np_tot[KDIR], data->np_tot[JDIR], data->np_tot[IDIR]); + dU0 = IdefixArray4D("RKL_dU0", NVAR, + data->np_tot[KDIR], data->np_tot[JDIR], data->np_tot[IDIR]); + Uc0 = IdefixArray4D("RKL_Uc0", NVAR, + data->np_tot[KDIR], data->np_tot[JDIR], data->np_tot[IDIR]); + Uc1 = IdefixArray4D("RKL_Uc1", NVAR, + data->np_tot[KDIR], data->np_tot[JDIR], data->np_tot[IDIR]); + + if(haveVs) { + #ifdef EVOLVE_VECTOR_POTENTIAL + dA = IdefixArray4D("RKL_dA", AX3e+1, + data->np_tot[KDIR]+KOFFSET, + data->np_tot[JDIR]+JOFFSET, + data->np_tot[IDIR]+IOFFSET); + dA0 = IdefixArray4D("RKL_dA0", AX3e+1, + data->np_tot[KDIR]+KOFFSET, + data->np_tot[JDIR]+JOFFSET, + data->np_tot[IDIR]+IOFFSET); + Ve0 = IdefixArray4D("RKL_Ve0", AX3e+1, + data->np_tot[KDIR]+KOFFSET, + data->np_tot[JDIR]+JOFFSET, + data->np_tot[IDIR]+IOFFSET); + Ve1 = IdefixArray4D("RKL_Ve1", AX3e+1, + data->np_tot[KDIR]+KOFFSET, + data->np_tot[JDIR]+JOFFSET, + data->np_tot[IDIR]+IOFFSET); + #else + dB = IdefixArray4D("RKL_dB", DIMENSIONS, + data->np_tot[KDIR]+KOFFSET, + data->np_tot[JDIR]+JOFFSET, + data->np_tot[IDIR]+IOFFSET); + dB0 = IdefixArray4D("RKL_dB0", DIMENSIONS, + data->np_tot[KDIR]+KOFFSET, + data->np_tot[JDIR]+JOFFSET, + data->np_tot[IDIR]+IOFFSET); + Vs0 = IdefixArray4D("RKL_Vs0", DIMENSIONS, + data->np_tot[KDIR]+KOFFSET, + data->np_tot[JDIR]+JOFFSET, + data->np_tot[IDIR]+IOFFSET); + Vs1 = IdefixArray4D("RKL_Vs1", DIMENSIONS, + data->np_tot[KDIR]+KOFFSET, + data->np_tot[JDIR]+JOFFSET, + data->np_tot[IDIR]+IOFFSET); + #endif + } + + idfx::popRegion(); +} + +template +void RKLegendre::ShowConfig() { + #if RKL_ORDER == 1 + idfx::cout << "RKLegendre: 1st order scheme ENABLED." << std::endl; + #elif RKL_ORDER == 2 + idfx::cout << "RKLegendre: 2nd order scheme ENABLED." << std::endl; + #else + IDEFIX_ERROR("Unknown RKL scheme order"); + #endif + idfx::cout << "RKLegendre: RKL cfl set to " << cfl_rkl << "." << std::endl; + idfx::cout << "RKLegendre: maximum ratio hyperbolic/parabolic timestep " + << rmax_par << "." << std::endl; + if(haveVc) { + idfx::cout << "RKLegendre: will evolve cell-centered fields Vc." << std::endl; + } + if(haveVs) { + idfx::cout << "RKLegendre: will evolve face-centered fields Vs." << std::endl; + } + if(checkNan) { + idfx::cout << "RKLegendre: will check consistency of solution in the integrator (slow!)." + << std::endl; + } +} + +template +void RKLegendre::Cycle() { + idfx::pushRegion("RKLegendre::Cycle"); + + IdefixArray4D dU = this->dU; + IdefixArray4D dU0 = this->dU0; + IdefixArray4D Uc = hydro->Uc; + IdefixArray4D Uc0 = this->Uc0; + IdefixArray4D Uc1 = this->Uc1; + + IdefixArray4D dB = this->dB; + IdefixArray4D dB0 = this->dB0; + IdefixArray4D Vs = hydro->Vs; + IdefixArray4D Vs0 = this->Vs0; + IdefixArray4D Vs1 = this->Vs1; + + #ifdef EVOLVE_VECTOR_POTENTIAL + IdefixArray4D dA = this->dA; + IdefixArray4D dA0 = this->dA0; + IdefixArray4D Ve = hydro->Ve; + IdefixArray4D Ve0 = this->Ve0; + IdefixArray4D Ve1 = this->Ve1; + #endif + + IdefixArray1D varList = this->varList; + real time = data->t; + + real dt_hyp = data->dt; + + // Tell the datablock that we're performing the RKL cycle + data->rklCycle = true; + + // first RKL stage + stage = 1; + + // Apply Boundary conditions on the full set of variables + hydro->boundary->SetBoundaries(time); + + // Convert current state into conservative variable + hydro->ConvertPrimToCons(); + + // Coarsen the conservative variables if needed + if(data->haveGridCoarsening) { + data->Coarsen(); + } + + // Store the result in Uc0 + Copy(Uc0,Uc); + if(haveVs) { + #ifdef EVOLVE_VECTOR_POTENTIAL + Kokkos::deep_copy(Ve0,Ve); + #else + Kokkos::deep_copy(Vs0,Vs); + #endif + } + + // evolve RKL stage + EvolveStage(time); + + ComputeDt(); + + Copy(dU0,dU); + + if(haveVs) { + #ifdef EVOLVE_VECTOR_POTENTIAL + Kokkos::deep_copy(dA0,dA); + #else + Kokkos::deep_copy(dB0,dB); + #endif + } + + // Compute number of RKL steps + real nrkl; + real scrh = dt_hyp/dt; +#if RKL_ORDER == 1 + // Solution of quadratic Eq. + // 2*dt_hyp/dt_exp = s^2 + s + nrkl = 4.0*scrh / (1.0 + std::sqrt(1.0 + 8.0*scrh)); +#elif RKL_ORDER == 2 + // Solution of quadratic Eq. + // 4*dt_hyp/dt_exp = s^2 + s - 2 + nrkl = 4.0*(1.0 + 2.0*scrh) + / (1.0 + sqrt(9.0 + 16.0*scrh)); +#else + //#error Invalid RKL_ORDER +#endif + int rklstages = 1 + floor(nrkl); + + // Compute coefficients + real w1, mu_tilde_j; +#if RKL_ORDER == 1 + w1 = 2.0/(rklstages*rklstages + rklstages); + mu_tilde_j = w1; +#elif RKL_ORDER == 2 + real b_j, b_jm1, b_jm2, a_jm1; + w1 = 4.0/(rklstages*rklstages + rklstages - 2.0); + mu_tilde_j = w1/3.0; + + b_j = b_jm1 = b_jm2 = 1.0/3.0; + a_jm1 = 1.0 - b_jm1; +#endif + +#if RKL_ORDER == 1 + time = data->t + 0.5*dt_hyp*(stage*stage+stage)*w1; +#elif RKL_ORDER == 2 + time = data->t + 0.25*dt_hyp*(stage*stage+stage-2)*w1; +#endif + if(haveVc) { + idefix_for("RKL_Cycle_InitUc1", + 0, nvarRKL, + data->beg[KDIR],data->end[KDIR], + data->beg[JDIR],data->end[JDIR], + data->beg[IDIR],data->end[IDIR], + KOKKOS_LAMBDA (int n, int k, int j, int i) { + int nv = varList(n); + Uc1(nv,k,j,i) = Uc(nv,k,j,i); + Uc(nv,k,j,i) = Uc1(nv,k,j,i) + mu_tilde_j*dt_hyp*dU0(nv,k,j,i); + } + ); + } + if(haveVs) { + #ifdef EVOLVE_VECTOR_POTENTIAL + idefix_for("RKL_Cycle_InitVe1", + 0, AX3e+1, + data->beg[KDIR],data->end[KDIR]+KOFFSET, + data->beg[JDIR],data->end[JDIR]+JOFFSET, + data->beg[IDIR],data->end[IDIR]+IOFFSET, + KOKKOS_LAMBDA (int n, int k, int j, int i) { + Ve1(n,k,j,i) = Ve(n,k,j,i); + Ve(n,k,j,i) = Ve1(n,k,j,i) + mu_tilde_j*dt_hyp*dA0(n,k,j,i); + }); + hydro->emf->ComputeMagFieldFromA(Ve,Vs); + #else + idefix_for("RKL_Cycle_InitVs1", + 0, DIMENSIONS, + data->beg[KDIR],data->end[KDIR]+KOFFSET, + data->beg[JDIR],data->end[JDIR]+JOFFSET, + data->beg[IDIR],data->end[IDIR]+IOFFSET, + KOKKOS_LAMBDA (int n, int k, int j, int i) { + Vs1(n,k,j,i) = Vs(n,k,j,i); + Vs(n,k,j,i) = Vs1(n,k,j,i) + mu_tilde_j*dt_hyp*dB0(n,k,j,i); + }); + #endif + hydro->boundary->ReconstructVcField(Uc); + } + + // Coarsen conservative variables once they have been evolved + if(data->haveGridCoarsening) { + data->Coarsen(); + } + + // Convert current state into primitive variable + hydro->ConvertConsToPrim(); + if(checkNan) { + if(data->CheckNan()>0) { + throw std::runtime_error(std::string("Nan found during RKL stage 1")); + } + } + + real mu_j, nu_j, gamma_j; + // subStages loop + for(stage=2; stage <= rklstages ; stage++) { + //idfx::cout << "RKL: looping stages" << std::endl; + // compute RKL coefficients +#if RKL_ORDER == 1 + mu_j = (2.0*stage -1.0)/stage; + mu_tilde_j = w1*mu_j; + nu_j = -(stage -1.0)/stage; +#elif RKL_ORDER == 2 + mu_j = (2.0*stage -1.0)/stage * b_j/b_jm1; + mu_tilde_j = w1*mu_j; + gamma_j = -a_jm1*mu_tilde_j; + nu_j = -(stage -1.0)*b_j/(stage*b_jm2); + + b_jm2 = b_jm1; + b_jm1 = b_j; + a_jm1 = 1.0 - b_jm1; + b_j = 0.5*(stage*stage+3.0*stage)/(stage*stage+3.0*stage+2.0); +#endif + + // Apply Boundary conditions + this->SetBoundaries(time); + + // evolve RKL stage + EvolveStage(time); + if(haveVc) { + // update Uc + idefix_for("RKL_Cycle_UpdateUc", + 0, nvarRKL, + data->beg[KDIR],data->end[KDIR], + data->beg[JDIR],data->end[JDIR], + data->beg[IDIR],data->end[IDIR], + KOKKOS_LAMBDA (int n, int k, int j, int i) { + const int nv = varList(n); + real Y = mu_j*Uc(nv,k,j,i) + nu_j*Uc1(nv,k,j,i); + Uc1(nv,k,j,i) = Uc(nv,k,j,i); + #if RKL_ORDER == 1 + Uc(nv,k,j,i) = Y + dt_hyp*mu_tilde_j*dU(nv,k,j,i); + #elif RKL_ORDER == 2 + Uc(nv,k,j,i) = Y + (1.0 - mu_j - nu_j)*Uc0(nv,k,j,i) + + dt_hyp*mu_tilde_j*dU(nv,k,j,i) + + gamma_j*dt_hyp*dU0(nv,k,j,i); + #endif + }); + } + if(haveVs) { + #ifdef EVOLVE_VECTOR_POTENTIAL + // update Ve + idefix_for("RKL_Cycle_UpdateVe", + 0, AX3e+1, + data->beg[KDIR],data->end[KDIR]+KOFFSET, + data->beg[JDIR],data->end[JDIR]+JOFFSET, + data->beg[IDIR],data->end[IDIR]+IOFFSET, + KOKKOS_LAMBDA (int n, int k, int j, int i) { + real Y = mu_j*Ve(n,k,j,i) + nu_j*Ve1(n,k,j,i); + Ve1(n,k,j,i) = Ve(n,k,j,i); + #if RKL_ORDER == 1 + Ve(n,k,j,i) = Y + dt_hyp*mu_tilde_j*dA(n,k,j,i); + #elif RKL_ORDER == 2 + Ve(n,k,j,i) = Y + (1.0 - mu_j - nu_j)*Ve0(n,k,j,i) + + dt_hyp*mu_tilde_j*dA(n,k,j,i) + + gamma_j*dt_hyp*dA0(n,k,j,i); + #endif + }); + hydro->emf->ComputeMagFieldFromA(Ve,Vs); + #else + // update Vs + idefix_for("RKL_Cycle_UpdateVs", + 0, DIMENSIONS, + data->beg[KDIR],data->end[KDIR]+KOFFSET, + data->beg[JDIR],data->end[JDIR]+JOFFSET, + data->beg[IDIR],data->end[IDIR]+IOFFSET, + KOKKOS_LAMBDA (int n, int k, int j, int i) { + real Y = mu_j*Vs(n,k,j,i) + nu_j*Vs1(n,k,j,i); + Vs1(n,k,j,i) = Vs(n,k,j,i); + #if RKL_ORDER == 1 + Vs(n,k,j,i) = Y + dt_hyp*mu_tilde_j*dB(n,k,j,i); + #elif RKL_ORDER == 2 + Vs(n,k,j,i) = Y + (1.0 - mu_j - nu_j)*Vs0(n,k,j,i) + + dt_hyp*mu_tilde_j*dB(n,k,j,i) + + gamma_j*dt_hyp*dB0(n,k,j,i); + #endif + }); + #endif // EVOLVE_VECTOR_POTENTIAL + + hydro->boundary->ReconstructVcField(Uc); + } + + // Coarsen the flow if needed + if(data->haveGridCoarsening) { + data->Coarsen(); + } + // Convert current state into primitive variable + hydro->ConvertConsToPrim(); + + if(checkNan) { + if(data->CheckNan()>0) { + throw std::runtime_error(std::string("Nan found during RKL stage ")+std::to_string(stage)); + } + } + + // increment time +#if RKL_ORDER == 1 + time = data->t + 0.5*dt_hyp*(stage*stage+stage)*w1; +#elif RKL_ORDER == 2 + time = data->t + 0.25*dt_hyp*(stage*stage+stage-2)*w1; +#endif + } + + // Tell the datablock that we're done + data->rklCycle = false; + idfx::popRegion(); +} + + + +template +void RKLegendre::ResetFlux() { + idfx::pushRegion("RKLegendre::ResetFlux"); + IdefixArray4D Flux = hydro->FluxRiemann; + IdefixArray1D vars = this->varList; + idefix_for("RKL_ResetFlux", + 0,nvarRKL, + 0,data->np_tot[KDIR], + 0,data->np_tot[JDIR], + 0,data->np_tot[IDIR], + KOKKOS_LAMBDA (int n, int k, int j, int i) { + const int nv = vars(n); + Flux(nv,k,j,i) = ZERO_F; + } + ); + idfx::popRegion(); +} + +template +struct RKLegendre_ResetStageFunctor { + explicit RKLegendre_ResetStageFunctor(RKLegendre *rkl) { + dU = rkl->dU; + Flux = rkl->hydro->FluxRiemann; + vars = rkl->varList; + stage = rkl->stage; + nvar = rkl->nvarRKL; + haveVc = rkl->haveVc || (rkl->stage ==1 ); + haveVs = rkl->haveVs; + invDt = rkl->hydro->InvDt; + if constexpr(Phys::mhd) { + #ifdef EVOLVE_VECTOR_POTENTIAL + dA = rkl->dA; + #else + dB = rkl->dB; + #endif + ex = rkl->hydro->emf->ex; + ey = rkl->hydro->emf->ey; + ez = rkl->hydro->emf->ez; + } + } + + IdefixArray4D dU; + IdefixArray4D Flux; + IdefixArray1D vars; + IdefixArray4D dA, dB; + IdefixArray3D ex,ey,ez; + IdefixArray3D invDt; + int stage, nvar; + bool haveVs, haveVc; + + KOKKOS_INLINE_FUNCTION void operator() (const int k, const int j, const int i) const { + if(haveVc) { + for(int n = 0 ; n < nvar ; n++) { + const int nv = vars(n); + dU(nv,k,j,i) = ZERO_F; + } + } + if(stage == 1) invDt(k,j,i) = ZERO_F; + if constexpr(Phys::mhd) { + if(haveVs) { + #ifdef EVOLVE_VECTOR_POTENTIAL + for(int n=0; n < AX3e+1; n++) { + dA(n,k,j,i) = ZERO_F; + } + #else + for(int n=0; n < DIMENSIONS; n++) { + dB(n,k,j,i) = ZERO_F; + } + #endif + D_EXPAND( ez(k,j,i) = 0.0; , + , + ex(k,j,i) = 0.0; + ey(k,j,i) = 0.0; ) + } + } + } +}; + +template +void RKLegendre::ResetStage() { + idfx::pushRegion("RKLegendre::ResetStage"); + + auto func = RKLegendre_ResetStageFunctor(this); + + + idefix_for("RKL_ResetStage", + 0,data->np_tot[KDIR], + 0,data->np_tot[JDIR], + 0,data->np_tot[IDIR], + func); + + idfx::popRegion(); +} + + +template +void RKLegendre::ComputeDt() { + idfx::pushRegion("RKLegendre::ComputeDt"); + + IdefixArray3D invDt = hydro->InvDt; + + real newinvdt = ZERO_F; + Kokkos::parallel_reduce("RKL_Timestep_reduction", + Kokkos::MDRangePolicy> + ({0,0,0},{data->end[KDIR],data->end[JDIR],data->end[IDIR]}), + KOKKOS_LAMBDA (int k, int j, int i, real &invdt) { + invdt = std::fmax(invDt(k,j,i), invdt); + }, + Kokkos::Max(newinvdt) + ); + +#ifdef WITH_MPI + if(idfx::psize>1) { + MPI_SAFE_CALL(MPI_Allreduce(MPI_IN_PLACE, &newinvdt, 1, realMPI, MPI_MAX, MPI_COMM_WORLD)); + } +#endif + + dt = 1.0/newinvdt; + dt = (cfl_rkl*dt)/2.0; // parabolic time step + + idfx::popRegion(); +} + +template +template +void RKLegendre::LoopDir(real t) { + ResetFlux(); + + // CalcParabolicFlux + hydro->template CalcParabolicFlux(t); + + // Calc Right Hand Side + CalcParabolicRHS(t); + + // Recursive: do next dimension + if constexpr(dir+1(t); + } +} + + +template +void RKLegendre::EvolveStage(real t) { + idfx::pushRegion("RKLegendre::EvolveStage"); + + ResetStage(); + + if(haveVs && hydro->needRKLCurrent) hydro->CalcCurrent(); + + // Loop on dimensions for the parabolic fluxes and RHS, starting from IDIR + if(haveVc || stage == 1) LoopDir(t); + + if(haveVs) { + hydro->emf->CalcNonidealEMF(t); + hydro->emf->EnforceEMFBoundary(); + real dt=1.0; + #ifdef EVOLVE_VECTOR_POTENTIAL + hydro->emf->EvolveVectorPotential(dt, this->dA); + #else + hydro->emf->EvolveMagField(t, dt, this->dB); + #endif + } + idfx::popRegion(); +} + +template +template +void RKLegendre::CalcParabolicRHS(real t) { + idfx::pushRegion("RKLegendre::CalcParabolicRHS"); + + IdefixArray4D Flux = hydro->FluxRiemann; + IdefixArray3D A = data->A[dir]; + IdefixArray3D dV = data->dV; + IdefixArray1D x1m = data->xl[IDIR]; + IdefixArray1D x1 = data->x[IDIR]; + IdefixArray1D sm = data->sinx2m; + IdefixArray1D rt = data->rt; + IdefixArray1D dmu = data->dmu; + IdefixArray1D s = data->sinx2; + IdefixArray1D dx = data->dx[dir]; + IdefixArray1D dx2 = data->dx[JDIR]; + IdefixArray3D invDt = hydro->InvDt; + IdefixArray3D dMax = hydro->dMax; + IdefixArray4D viscSrc; + IdefixArray4D dU = this->dU; + IdefixArray1D varList = this->varList; + + bool haveViscosity = hydro->viscosityStatus.isRKL; + if(haveViscosity) viscSrc = hydro->viscosity->viscSrc; + IdefixArray4D bragViscSrc; + bool haveBragViscosity = hydro->bragViscosityStatus.isRKL; + if(haveBragViscosity) bragViscSrc = hydro->bragViscosity->bragViscSrc; + + int ioffset,joffset,koffset; + ioffset=joffset=koffset=0; + // Determine the offset along which we do the extrapolation + if(dir==IDIR) ioffset=1; + if(dir==JDIR) joffset=1; + if(dir==KDIR) koffset=1; + + + idefix_for("CalcTotalFlux", + 0, this->nvarRKL, + data->beg[KDIR],data->end[KDIR]+koffset, + data->beg[JDIR],data->end[JDIR]+joffset, + data->beg[IDIR],data->end[IDIR]+ioffset, + KOKKOS_LAMBDA (int n, int k, int j, int i) { + real Ax = A(k,j,i); + +#if GEOMETRY != CARTESIAN + if(Ax= 2) \ + || (GEOMETRY == CYLINDRICAL && COMPONENTS == 3) + if(dir==IDIR && nv==iMPHI) { + // Conserve angular momentum, hence flux is R*Vphi + Flux(iMPHI,k,j,i) = Flux(iMPHI,k,j,i) * FABS(x1m(i)); + } +#endif // GEOMETRY==POLAR OR CYLINDRICAL + +#if GEOMETRY == SPHERICAL && COMPONENTS == 3 + if(dir==IDIR && nv==iMPHI) { + Flux(iMPHI,k,j,i) = Flux(iMPHI,k,j,i) * FABS(x1m(i)); + } else if(dir==JDIR && nv==iMPHI) { + Flux(iMPHI,k,j,i) = Flux(iMPHI,k,j,i) * FABS(sm(j)); + } +#endif // GEOMETRY == SPHERICAL && COMPONENTS == 3 + } + ); + + + idefix_for("CalcRightHandSide", + 0, this->nvarRKL, + data->beg[KDIR],data->end[KDIR], + data->beg[JDIR],data->end[JDIR], + data->beg[IDIR],data->end[IDIR], + KOKKOS_LAMBDA (int n, int k, int j, int i) { + constexpr const int ioffset = (dir==IDIR) ? 1 : 0; + constexpr const int joffset = (dir==JDIR) ? 1 : 0; + constexpr const int koffset = (dir==KDIR) ? 1 : 0; + real rhs; + + const int nv = varList(n); + + rhs = - ( Flux(nv, k+koffset, j+joffset, i+ioffset) + - Flux(nv, k, j, i))/dV(k,j,i); + + // Viscosity source terms + if( haveViscosity && (nv-VX1 < COMPONENTS) && (nv-VX1>=0)) { + rhs += viscSrc(nv-VX1,k,j,i); + } + // Braginskii Viscosity source terms + if (haveBragViscosity && (nv-VX1 < COMPONENTS) && (nv-VX1>=0)) { + rhs += bragViscSrc(nv-VX1,k,j,i); + } + +#if GEOMETRY != CARTESIAN + #ifdef iMPHI + if((dir==IDIR) && (nv == iMPHI)) { + rhs /= x1(i); + } + #if (GEOMETRY == SPHERICAL) && (COMPONENTS == 3) + if((dir==JDIR) && (nv == iMPHI)) { + rhs /= FABS(s(j)); + } + #endif // GEOMETRY + // Nothing for KDIR + #endif // iMPHI +#endif // GEOMETRY != CARTESIAN + + // store the field components + dU(nv,k,j,i) += rhs; + }); + + // Compute hyperbolic timestep only if we're in the first stage of the RKL loop + if(stage==1) { + // Grid coarsening + bool haveGridCoarsening = false; + IdefixArray2D coarseningLevel; + + if(data->haveGridCoarsening) { + haveGridCoarsening = data->coarseningDirection[dir]; + coarseningLevel = data->coarseningLevel[dir]; + } + + idefix_for("CalcDt", + data->beg[KDIR],data->end[KDIR], + data->beg[JDIR],data->end[JDIR], + data->beg[IDIR],data->end[IDIR], + KOKKOS_LAMBDA (int k, int j, int i) { + constexpr const int ioffset = (dir==IDIR) ? 1 : 0; + constexpr const int joffset = (dir==JDIR) ? 1 : 0; + constexpr const int koffset = (dir==KDIR) ? 1 : 0; + // Compute dt from max signal speed + const int ig = ioffset*i + joffset*j + koffset*k; + real dl = dx(ig); + + if(haveGridCoarsening) { + int factor; + //factor = 2^(coarsening-1) + if(dir==IDIR) { + factor = 1 << (coarseningLevel(k,j) - 1); + } + if(dir==JDIR) { + factor = 1 << (coarseningLevel(k,i) - 1); + } + if(dir==KDIR) { + factor = 1 << (coarseningLevel(j,i) - 1); + } + dl = dl * factor; + } + #if GEOMETRY == POLAR + if (dir==JDIR) + dl = dl*x1(i); + + #elif GEOMETRY == SPHERICAL + if (dir==JDIR) + dl = dl*rt(i); + else + if (dir==KDIR) + dl = dl*rt(i)*dmu(j)/dx2(j); + #endif + + invDt(k,j,i) += 0.5 * std::fmax(dMax(k+koffset,j+joffset,i+ioffset), + dMax(k,j,i)) / (dl*dl); + }); + } + + idfx::popRegion(); +} + +template +void RKLegendre::SetBoundaries(real t) { + idfx::pushRegion("RKLegendre::SetBoundaries"); + if(data->haveGridCoarsening) { + hydro->CoarsenFlow(hydro->Vc); + if constexpr(Phys::mhd) { + hydro->CoarsenMagField(hydro->Vs); + } + } + + // set internal boundary conditions + // Disabled since this might affect fields that are NOT updated + // by the MPI instance of RKLegendre + //if(hydro->boundary->haveInternalBoundary) + // hydro->boundary->internalBoundaryFunc(*data, t); + for(int dir=0 ; dir < DIMENSIONS ; dir++ ) { + // MPI Exchange data when needed + // We use the RKL instance MPI object to ensure that we only exchange the data + // solved by RKL + #ifdef WITH_MPI + if(data->mygrid->nproc[dir]>1) { + switch(dir) { + case 0: + this->mpi.ExchangeX1(hydro->Vc, hydro->Vs); + break; + case 1: + this->mpi.ExchangeX2(hydro->Vc, hydro->Vs); + break; + case 2: + this->mpi.ExchangeX3(hydro->Vc, hydro->Vs); + break; + } + } + #endif + hydro->boundary->EnforceBoundaryDir(t, dir); + if constexpr(Phys::mhd) { + // Reconstruct the normal field component when using CT + if(haveVs) { + hydro->boundary->ReconstructNormalField(dir); + } + } + } // Loop on dimension ends + + if constexpr(Phys::mhd) { + // Remake the cell-centered field. + if(haveVs) { + hydro->boundary->ReconstructVcField(hydro->Vc); + } + } + idfx::popRegion(); +} + + #endif // RKL_RKL_HPP_ diff --git a/src/setup.hpp b/src/setup.hpp index 8d5aa6b3..8e1aa0ed 100644 --- a/src/setup.hpp +++ b/src/setup.hpp @@ -12,12 +12,13 @@ #include "input.hpp" #include "grid.hpp" #include "dataBlock.hpp" +#include "fluid.hpp" #include "output.hpp" // These two will likely be used in the code, so we include them here #include "gridHost.hpp" #include "dataBlockHost.hpp" -#include "boundaryloop.hpp" +#include "boundary.hpp" class Setup { public: diff --git a/src/timeIntegrator.cpp b/src/timeIntegrator.cpp index b8e58ced..6b2328d7 100644 --- a/src/timeIntegrator.cpp +++ b/src/timeIntegrator.cpp @@ -11,7 +11,10 @@ #include "idefix.hpp" #include "timeIntegrator.hpp" #include "input.hpp" +#include "dataBlock.hpp" #include "stateContainer.hpp" +#include "fluid.hpp" +#include "planetarySystem.hpp" TimeIntegrator::TimeIntegrator(Input & input, DataBlock & data) { @@ -65,8 +68,7 @@ TimeIntegrator::TimeIntegrator(Input & input, DataBlock & data) { } // Init the RKL scheme if it's needed - if(data.hydro.haveRKLParabolicTerms) { - rkl.Init(input,data); + if(data.hydro->haveRKLParabolicTerms) { haveRKL = true; } @@ -91,6 +93,13 @@ void TimeIntegrator::ShowLog(DataBlock &data) { double mpiOverhead = 100.0 * mpiCycleTime / (timer.seconds() - lastLog); lastMpiLog = idfx::mpiCallsTimer; #endif + double sgOverhead; + if(data.haveGravity && data.gravity->haveSelfGravityPotential) { + double sgCycleTime = data.gravity->selfGravity.elapsedTime - lastSGLog; + sgOverhead = 100.0 * sgCycleTime / (timer.seconds() - lastLog); + lastSGLog = data.gravity->selfGravity.elapsedTime; + } + lastLog = timer.seconds(); @@ -110,6 +119,11 @@ void TimeIntegrator::ShowLog(DataBlock &data) { if(haveRKL) { idfx::cout << " | " << std::setw(col_width) << "RKL stages"; } + if(data.haveGravity && data.gravity->haveSelfGravityPotential) { + idfx::cout << " | " << std::setw(col_width) << "SG iterations"; + idfx::cout << " | " << std::setw(col_width) << "SG error"; + idfx::cout << " | " << std::setw(col_width) << "SG overhead (%)"; + } idfx::cout << std::endl; } @@ -134,7 +148,7 @@ void TimeIntegrator::ShowLog(DataBlock &data) { #if MHD == YES // Check divB - real divB = data.hydro.CheckDivB(); + real divB = data.hydro->CheckDivB(); idfx::cout << std::scientific; idfx::cout << " | " << std::setw(col_width) << divB; @@ -145,7 +159,20 @@ void TimeIntegrator::ShowLog(DataBlock &data) { } #endif if(haveRKL) { - idfx::cout << " | " << std::setw(col_width) << rkl.stage; + idfx::cout << " | " << std::setw(col_width) << data.hydro->rkl->stage; + } + if(data.haveGravity && data.gravity->haveSelfGravityPotential) { + if(ncycles>=cyclePeriod) { + idfx::cout << " | " << std::setw(col_width) << data.gravity->selfGravity.nsteps; + idfx::cout << std::scientific; + idfx::cout << " | " << std::setw(col_width) << data.gravity->selfGravity.currentError; + idfx::cout << std::fixed; + idfx::cout << " | " << std::setw(col_width) << sgOverhead; + } else { + idfx::cout << " | " << std::setw(col_width) << "N/A"; + idfx::cout << " | " << std::setw(col_width) << "N/A"; + idfx::cout << " | " << std::setw(col_width) << "N/A"; + } } idfx::cout << std::endl; } @@ -154,15 +181,18 @@ void TimeIntegrator::ShowLog(DataBlock &data) { // Compute one full cycle of the time Integrator void TimeIntegrator::Cycle(DataBlock &data) { // Do one cycle - IdefixArray3D InvDt = data.hydro.InvDt; + IdefixArray3D InvDt = data.hydro->InvDt; real newdt; idfx::pushRegion("TimeIntegrator::Cycle"); if(ncycles%cyclePeriod==0) ShowLog(data); + // Launch user step before everything + data.LaunchUserStepFirst(); + if(haveRKL && (ncycles%2)==1) { // Runge-Kutta-Legendre cycle - rkl.Cycle(); + data.EvolveRKLStage(); } // save t at the begining of the cycle @@ -183,17 +213,19 @@ void TimeIntegrator::Cycle(DataBlock &data) { data.SetBoundaries(); // Remove Fargo velocity so that the integrator works on the residual - if(data.haveFargo) data.fargo.SubstractVelocity(data.t); + if(data.haveFargo) data.fargo->SubstractVelocity(data.t); // Convert current state into conservative variable and save it - data.hydro.ConvertPrimToCons(); + data.PrimToCons(); // Store (deep copy) initial stage for multi-stage time integrators if(nstages>1 && stage==0) { data.states["begin"].CopyFrom(data.states["current"]); } // If gravity is needed, update it - if(data.haveGravity) data.gravity.ComputeGravity(); + if(data.haveGravity) { + if(ncycles % data.gravity->skipGravity == 0) data.gravity->ComputeGravity(ncycles); + } // Update Uc & Vs data.EvolveStage(); @@ -234,7 +266,7 @@ void TimeIntegrator::Cycle(DataBlock &data) { } // Shift solution according to fargo if this is our last stage if(data.haveFargo && stage==nstages-1) { - data.fargo.ShiftSolution(t0,data.dt); + data.fargo->ShiftSolution(t0,data.dt); } // Coarsen conservative variables once they have been evolved @@ -243,10 +275,10 @@ void TimeIntegrator::Cycle(DataBlock &data) { } // Back to using Vc - data.hydro.ConvertConsToPrim(); + data.ConsToPrim(); // Add back fargo velocity so that boundary conditions are applied on the total V - if(data.haveFargo) data.fargo.AddVelocity(data.t); + if(data.haveFargo) data.fargo->AddVelocity(data.t); } ///////////////////////////////////////////////// // END STAGES LOOP // @@ -260,20 +292,29 @@ void TimeIntegrator::Cycle(DataBlock &data) { #endif if(haveRKL && (ncycles%2)==0) { // Runge-Kutta-Legendre cycle - rkl.Cycle(); + data.EvolveRKLStage(); + } + + // Update planet position + if(data.haveplanetarySystem) { + data.planetarySystem->EvolveSystem(data, data.dt); } // Coarsen the grid if(data.haveGridCoarsening) { data.Coarsen(); } + + // Launch user step last + data.LaunchUserStepLast(); + // Update current time (should have already been done, but this gets rid of roundoff errors) data.t=t0+data.dt; if(haveRKL) { // update next time step - real tt = newdt/rkl.dt; - newdt *= std::fmin(ONE_F, rkl.rmax_par/(tt)); + real tt = newdt/data.hydro->rkl->dt; + newdt *= std::fmin(ONE_F, data.hydro->rkl->rmax_par/(tt)); } // Next time step @@ -304,8 +345,6 @@ void TimeIntegrator::Cycle(DataBlock &data) { idfx::popRegion(); } - - int64_t TimeIntegrator::GetNCycles() { return(ncycles); } @@ -357,8 +396,4 @@ void TimeIntegrator::ShowConfig() { if(maxRuntime>0) { idfx::cout << "TimeIntegrator: will stop after " << maxRuntime/3600 << " hours." << std::endl; } - - if(haveRKL) { - rkl.ShowConfig(); - } } diff --git a/src/timeIntegrator.hpp b/src/timeIntegrator.hpp index 722bc9cf..9f41ccbf 100644 --- a/src/timeIntegrator.hpp +++ b/src/timeIntegrator.hpp @@ -33,8 +33,7 @@ class TimeIntegrator { bool isSilent{false}; // Whether the integration should proceed silently private: - // The RKL object attached to this datablock - RKLegendre rkl; + // Whether we have RKL bool haveRKL{false}; int nstages; @@ -53,6 +52,7 @@ class TimeIntegrator { int64_t ncycles; // # of cycles double lastLog; // time for the last log (s) double lastMpiLog; // time for the last MPI log (s) + double lastSGLog; // time for the last SelfGravity log (s) double maxRuntime; // Maximum runtime requested (disabled when negative) int64_t cyclePeriod; // # of cycles between two logs Kokkos::Timer timer; // Internal timer of the integrator diff --git a/src/utils/CMakeLists.txt b/src/utils/CMakeLists.txt index 65591f78..ff971569 100644 --- a/src/utils/CMakeLists.txt +++ b/src/utils/CMakeLists.txt @@ -1,3 +1,5 @@ +add_subdirectory(iterativesolver) + target_sources(idefix PUBLIC ${CMAKE_CURRENT_LIST_DIR}/dumpImage.cpp PUBLIC ${CMAKE_CURRENT_LIST_DIR}/dumpImage.hpp diff --git a/src/utils/dumpImage.cpp b/src/utils/dumpImage.cpp index 5adf6f06..74e0a319 100644 --- a/src/utils/dumpImage.cpp +++ b/src/utils/dumpImage.cpp @@ -6,13 +6,13 @@ // *********************************************************************************** #include "dumpImage.hpp" -#include "output.hpp" +#include "dataBlock.hpp" #include "idefix.hpp" #define HEADERSIZE 128 -DumpImage::DumpImage(std::string filename, Output &output) { +DumpImage::DumpImage(std::string filename, DataBlock *data) { idfx::pushRegion("DumpImage::DumpImage"); int nx[3]; @@ -21,7 +21,7 @@ DumpImage::DumpImage(std::string filename, Output &output) { DataType type; int ndim; IdfxFileHandler fileHdl; - Dump &dump = output.dump; + Dump dump(data); idfx::cout << "DumpImage: loading restart file " << filename << "..." << std::flush; @@ -82,6 +82,8 @@ DumpImage::DumpImage(std::string filename, Output &output) { dump.ReadSerial(fileHdl, ndim, nx, type, &this->time); } else if(fieldName.compare("geometry")==0) { dump.ReadSerial(fileHdl, ndim, nx, type, &this->geometry); + } else if(fieldName.compare("centralMass")==0) { + dump.ReadSerial(fileHdl, ndim, nx, type, &this->centralMass); } else { int size=sizeof(double); for(int dim =0 ; dim x[3]; // geometrical central points IdefixHostArray1D xr[3]; // cell right interface IdefixHostArray1D xl[3]; // cell left interface diff --git a/src/utils/iterativesolver/CMakeLists.txt b/src/utils/iterativesolver/CMakeLists.txt new file mode 100644 index 00000000..c8367b3f --- /dev/null +++ b/src/utils/iterativesolver/CMakeLists.txt @@ -0,0 +1,6 @@ +target_sources(idefix + PUBLIC ${CMAKE_CURRENT_LIST_DIR}/cg.hpp + PUBLIC ${CMAKE_CURRENT_LIST_DIR}/minres.hpp + PUBLIC ${CMAKE_CURRENT_LIST_DIR}/bicgstab.hpp + PUBLIC ${CMAKE_CURRENT_LIST_DIR}/jacobi.hpp + ) diff --git a/src/utils/iterativesolver/bicgstab.hpp b/src/utils/iterativesolver/bicgstab.hpp new file mode 100644 index 00000000..0b3dbb1b --- /dev/null +++ b/src/utils/iterativesolver/bicgstab.hpp @@ -0,0 +1,319 @@ +// *********************************************************************************** +// Idefix MHD astrophysical code +// Copyright(C) Geoffroy R. J. Lesur +// and other code contributors +// Licensed under CeCILL 2.1 License, see COPYING for more information +// *********************************************************************************** + +#ifndef UTILS_ITERATIVESOLVER_BICGSTAB_HPP_ +#define UTILS_ITERATIVESOLVER_BICGSTAB_HPP_ +#include +#include "idefix.hpp" +#include "vector.hpp" +#include "iterativesolver.hpp" + +// The bicgstab derives from the iterativesolver class +template +class Bicgstab : public IterativeSolver { + public: + Bicgstab(T &op, real error, int maxIter, + std::array ntot, std::array beg, std::array end); + + int Solve(IdefixArray3D &guess, IdefixArray3D &rhs); + + void PerformIter(); + void InitSolver(); + void ShowConfig(); + + private: + real rho; // BICSTAB parameter + real alpha; // BICGSTAB parameter + real omega; // BICGSTAB parameter + + + IdefixArray3D res0; // Reference (initial) residual + IdefixArray3D dir; // Search direction for gradient descent + IdefixArray3D work1; // work array + IdefixArray3D work2; // work array + IdefixArray3D work3; // work array +}; + +template +Bicgstab::Bicgstab(T &op, real error, int maxiter, + std::array ntot, std::array beg, std::array end) : + IterativeSolver(op, error, maxiter, ntot, beg, end) { + // BICGSTAB scalars initialisation + this->rho = 1.0; + this->alpha = 1.0; + this->omega = 1.0; + + + + this->dir = IdefixArray3D ("Direction", this->ntot[KDIR], + this->ntot[JDIR], + this->ntot[IDIR]); + + this->res0 = IdefixArray3D ("InitialResidual", this->ntot[KDIR], + this->ntot[JDIR], + this->ntot[IDIR]); + + this->work1 = IdefixArray3D ("WorkingArray1", this->ntot[KDIR], + this->ntot[JDIR], + this->ntot[IDIR]); + + this->work2 = IdefixArray3D ("WorkingArray2", this->ntot[KDIR], + this->ntot[JDIR], + this->ntot[IDIR]); + + this->work3 = IdefixArray3D ("WorkingArray3", this->ntot[KDIR], + this->ntot[JDIR], + this->ntot[IDIR]); +} + +template +int Bicgstab::Solve(IdefixArray3D &guess, IdefixArray3D &rhs) { + idfx::pushRegion("Bicgstab::Solve"); + this->solution = guess; + this->rhs = rhs; + + // Re-initialise convStatus + this->convStatus = false; + + this->InitSolver(); + + int n = 0; + while(this->convStatus != true && n < this->maxiter) { + this->PerformIter(); + if(this->restart) { + this->restart=false; + // Resetting parameters + this->rho = 1.0; + this->alpha = 1.0; + this->omega = 1.0; + n = -1; + idfx::popRegion(); + return(n); + } + n++; + } + + if(n == this->maxiter) { + idfx::cout << "Bicgstab:: Reached max iter." << std::endl; + IDEFIX_WARNING("Bicgstab:: Failed to converge before reaching max iter." + "You should consider to use a preconditionner."); + } + + idfx::popRegion(); + return(n); +} + +template +void Bicgstab::InitSolver() { + idfx::pushRegion("Bicgstab::InitSolver"); + // Residual initialisation + this->SetRes(); + + Kokkos::deep_copy(this->res0, this->res); // (Re)setting reference residual + Kokkos::deep_copy(this->dir, this->res); // (Re)setting initial searching direction + this->linearOperator(this->dir, this->work1); // (Re)setting associated laplacian + + // // Resetting parameters + // this->rho = 1.0; + // this->alpha = 1.0; + // this->omega = 1.0; + + idfx::popRegion(); +} + +template +void Bicgstab::PerformIter() { + idfx::pushRegion("Bicgstab::PerformIter"); + + // Loading needed attributes + IdefixArray3D solution = this->solution; + IdefixArray3D res = this->res; + IdefixArray3D dir = this->dir; + IdefixArray3D res0 = this->res0; // Reference residual, do not evolve through the loop + + // The following variables are named following wikipedia's page nomenclature of BICGSTAB algorithm + IdefixArray3D v = this->work1; // Working array, for laplacian dir calculation + IdefixArray3D s = this->work2; // Working array, for intermediate dir calculation + IdefixArray3D t = this->work3; // Working array, for laplacian intermediate dir calculation + real omega; + real &alpha = this->alpha; + real &rhoOld = this->rho; + real &omegaOld = this->omega; + + int ibeg, iend, jbeg, jend, kbeg, kend; + ibeg = this->beg[IDIR]; + iend = this->end[IDIR]; + jbeg = this->beg[JDIR]; + jend = this->end[JDIR]; + kbeg = this->beg[KDIR]; + kend = this->end[KDIR]; + + // ***** Step 1. + real rho = this->ComputeDotProduct(res0, res); + #ifdef DEBUG_BICGSTAB + idfx::cout << "rho= " << rho; + #endif + + // Checking for Nans + if(std::isnan(rho)) { + idfx::cout << "Bicgstab:: rho is nan in step 1." << std::endl; + this->restart = true; + idfx::popRegion(); + return; + // IDEFIX_ERROR("rho is nan in step 1"); + } + + // ***** Step 2. + real beta = rho / rhoOld * alpha / omegaOld; + #ifdef DEBUG_BICGSTAB + idfx::cout << " ; beta= " << beta; + #endif + + // Checking for Nans + if(std::isnan(beta)) { + idfx::cout << "Bicgstab:: beta is nan in step 2." << std::endl; + // IDEFIX_ERROR("beta is nan in step 2"); + this->restart = true; + idfx::popRegion(); + return; + } + + // ***** Step 3. + idefix_for("UpdateDir", kbeg, kend, jbeg, jend, ibeg, iend, + KOKKOS_LAMBDA (int k, int j, int i) { + dir(k,j,i) = res(k,j,i) + beta * (dir(k,j,i) - omegaOld * v(k,j,i)); + }); + + // From now dir is updated + + // ***** Step 4. + this->linearOperator(dir, v); + + // from now v is updated (laplacian of dir) + + // ******* Step 5. + + alpha = rho / this->ComputeDotProduct(res0, v); + #ifdef DEBUG_BICGSTAB + idfx::cout << " ; alpha= " << alpha; + #endif + + // Checking Nans + if(std::isnan(alpha)) { + idfx::cout << "Bicgstab:: alpha is nan in step 5." << std::endl; + // IDEFIX_ERROR("alpha is nan in step 5"); + this->restart = true; + idfx::popRegion(); + return; + } + + // *********** Step 6. + // Assumes solution = x_i-1 + idefix_for("FirstUpdatePot", kbeg, kend, jbeg, jend, ibeg, iend, + KOKKOS_LAMBDA (int k, int j, int i) { + solution(k,j,i) = solution(k,j,i) + alpha * dir(k,j,i); + }); + + // From here solution = h_i + + // ********** Step.7. + // Store current residual + Kokkos::deep_copy(s, res); // s is momentarily oldRes to recycle arrays + + // Update residual + this->SetRes(); + + // Test intermediate guess h_i + this->TestErrorL2(); + + // The loop continues if no convergence + if(this->convStatus == false) { + // ***************** Step. 8. + idefix_for("FillIntermediateDir", kbeg, kend, jbeg, jend, ibeg, iend, + KOKKOS_LAMBDA (int k, int j, int i) { + s(k,j,i) = s(k,j,i) - alpha * v(k,j,i); // s in RHS is oldRes before update + }); + + // From here s is updated + + // ************** Step 9. + this->linearOperator(s, t); + + // From here t is updated + + // ************* Step 10. + omega = this->ComputeDotProduct(t, s) / this->ComputeDotProduct(t, t); + + #ifdef DEBUG_BICGSTAB + idfx::cout << " ; omega= " << omega; + #endif + + // Checking Nans + if(std::isnan(omega)) { + idfx::cout << "Bicgstab:: omega is nan in step 10." << std::endl; + // IDEFIX_ERROR("omega is nan in step 10"); + this->restart = true; + idfx::popRegion(); + return; + } + + // ************ Step 11. + // solution is h_i from step 6. + idefix_for("SecondUpdatePot", kbeg, kend, jbeg, jend, ibeg, iend, + KOKKOS_LAMBDA (int k, int j, int i) { + solution(k,j,i) = solution(k,j,i) + omega * s(k,j,i); + }); + + // From here, solution = x_i + + // *********** Step 12. + // Update residual + this->SetRes(); + + // Test final guess x_i + this->TestErrorL2(); + + // Last task if no convergence : update res + if(this->convStatus == false) { + idefix_for("UpdateRes", kbeg, kend, jbeg, jend, ibeg, iend, + KOKKOS_LAMBDA (int k, int j, int i) { + res(k,j,i) = s(k,j,i) - omega * t(k,j,i); + }); + + // Saving rho and omega for next iteration + rhoOld = rho; + omegaOld = omega; + } else { + #ifdef DEBUG_BICGSTAB + idfx::cout << "Bicgstab:: BICGSTAB converged in step 12." << std::endl; + #endif + } + } else { + #ifdef DEBUG_BICGSTAB + idfx::cout << "Bicgstab:: BICGSTAB converged in step 7." << std::endl; + #endif + } + #ifdef DEBUG_BICGSTAB + idfx::cout << std::endl; + #endif + + idfx::popRegion(); +} + + + +template +void Bicgstab::ShowConfig() { + idfx::pushRegion("Bicgstab::ShowConfig"); + idfx::cout << "Bicgstab: TargetError: " << this->targetError << std::endl; + idfx::cout << "Bicgstab: Maximum iterations: " << this->maxiter << std::endl; + idfx::popRegion(); + return; +} + + +#endif // UTILS_ITERATIVESOLVER_BICGSTAB_HPP_ diff --git a/src/utils/iterativesolver/cg.hpp b/src/utils/iterativesolver/cg.hpp new file mode 100644 index 00000000..de75b738 --- /dev/null +++ b/src/utils/iterativesolver/cg.hpp @@ -0,0 +1,161 @@ +// *********************************************************************************** +// Idefix MHD astrophysical code +// Copyright(C) Geoffroy R. J. Lesur +// and other code contributors +// Licensed under CeCILL 2.1 License, see COPYING for more information +// *********************************************************************************** + +#ifndef UTILS_ITERATIVESOLVER_CG_HPP_ +#define UTILS_ITERATIVESOLVER_CG_HPP_ +#include +#include "idefix.hpp" +#include "vector.hpp" +#include "iterativesolver.hpp" + +// The conjugate gradient derives from the iterativesolver class +template +class Cg : public IterativeSolver { + public: + Cg(T &op, real error, int maxIter, + std::array ntot, std::array beg, std::array end); + + int Solve(IdefixArray3D &guess, IdefixArray3D &rhs); + + void PerformIter(); + void InitSolver(); + void ShowConfig(); + + private: + IdefixArray3D p1; // Search direction for gradient descent + IdefixArray3D s1; // Search direction for gradient descent +}; + +template +Cg::Cg(T &op, real error, int maxiter, + std::array ntot, std::array beg, std::array end) : + IterativeSolver(op, error, maxiter, ntot, beg, end) { + // CG scalars initialisation + + this->p1 = IdefixArray3D ("p1", this->ntot[KDIR], + this->ntot[JDIR], + this->ntot[IDIR]); + + + this->s1 = IdefixArray3D ("s1", this->ntot[KDIR], + this->ntot[JDIR], + this->ntot[IDIR]); +} + +template +int Cg::Solve(IdefixArray3D &guess, IdefixArray3D &rhs) { + idfx::pushRegion("Cg::Solve"); + this->solution = guess; + this->rhs = rhs; + + // Re-initialise convStatus + this->convStatus = false; + this->InitSolver(); + int n = 0; + + while(this->convStatus != true && n < this->maxiter) { + this->PerformIter(); + + + n++; + } + + if(n == this->maxiter) { + idfx::cout << "Cg:: Reached max iter." << std::endl; + IDEFIX_ERROR("Cg:: Failed to converge before reaching max iter." + "You should consider to use a preconditionner."); + } + + idfx::popRegion(); + return(n); +} + +template +void Cg::InitSolver() { + idfx::pushRegion("Cg::InitSolver"); + // Residual initialisation + this->SetRes(); + + Kokkos::deep_copy(this->p1, this->res); // (Re)setting reference residual + + idfx::popRegion(); +} + +template +void Cg::PerformIter() { + idfx::pushRegion("Cg::PerformIter"); + + // Loading needed attributes + auto x = this->solution; + auto r = this->res; + auto p1 = this->p1; + auto s1 = this->s1; + + int ibeg, iend, jbeg, jend, kbeg, kend; + ibeg = this->beg[IDIR]; + iend = this->end[IDIR]; + jbeg = this->beg[JDIR]; + jend = this->end[JDIR]; + kbeg = this->beg[KDIR]; + kend = this->end[KDIR]; + + // ***** Step 1. + this->linearOperator(p1, s1); + + real rr = this->ComputeDotProduct(r,r); + //idfx::cout << "rr=" << rr << std::endl; + real alpha = rr / (this->ComputeDotProduct(p1,s1)); + + // Checking for Nans + if(std::isnan(alpha)) { + idfx::cout << "Cg:: alpha is nan in step 1." << std::endl; + this->restart = true; + idfx::popRegion(); + return; + // IDEFIX_ERROR("rho is nan in step 1"); + } + + // ******* Step 2 + idefix_for("UpdateDir", kbeg, kend, jbeg, jend, ibeg, iend, + KOKKOS_LAMBDA (int k, int j, int i) { + x(k,j,i) = x(k,j,i) + alpha * p1(k,j,i); + r(k,j,i) = r(k,j,i) - alpha * s1(k,j,i); + }); + + this->TestErrorL2(); + + real beta = this->ComputeDotProduct(r,r) / rr; + + // Checking for Nans + if(std::isnan(beta)) { + idfx::cout << "Cg:: beta is nan in step 2." << std::endl; + idfx::popRegion(); + return; + // IDEFIX_ERROR("rho is nan in step 1"); + } + + idefix_for("UpdateDir", kbeg, kend, jbeg, jend, ibeg, iend, + KOKKOS_LAMBDA (int k, int j, int i) { + p1(k,j,i) = r(k,j,i) + beta * p1(k,j,i); + }); + + idfx::popRegion(); +} + + + +template +void Cg::ShowConfig() { + idfx::pushRegion("Cg::ShowConfig"); + idfx::cout << "Cg: TargetError: " << this->targetError << std::endl; + idfx::cout << "Cg: Maximum iterations: " << this->maxiter << std::endl; + idfx::popRegion(); + return; +} + + +#endif // UTILS_ITERATIVESOLVER_CG_HPP_ diff --git a/src/utils/iterativesolver/iterativesolver.hpp b/src/utils/iterativesolver/iterativesolver.hpp new file mode 100644 index 00000000..d103eba6 --- /dev/null +++ b/src/utils/iterativesolver/iterativesolver.hpp @@ -0,0 +1,315 @@ +// *********************************************************************************** +// Idefix MHD astrophysical code +// Copyright(C) Geoffroy R. J. Lesur +// and other code contributors +// Licensed under CeCILL 2.1 License, see COPYING for more information +// *********************************************************************************** + +#ifndef UTILS_ITERATIVESOLVER_ITERATIVESOLVER_HPP_ +#define UTILS_ITERATIVESOLVER_ITERATIVESOLVER_HPP_ + +#include +#include "idefix.hpp" +#include "vector.hpp" + +template +class IterativeSolver { + public: + IterativeSolver(T &op, real error, int maxIter, + std::array ntot, std::array beg, std::array end); + + real GetError(); // return the current error of the solver + + virtual int Solve(IdefixArray3D &guess, IdefixArray3D &rhs) = 0; + virtual void ShowConfig() = 0; + + // Internal functions (left public for Lambda capture) + void SetRes(); // Set residual from current guess + void TestErrorL1(); // Test the convergence status of the current iteration with L1 norm + void TestErrorL2(); // Test the convergence status of the current iteration with L2 norm + void TestErrorLINF(); // Test the convergence status of the current iteration with LINF norm + real ComputeDotProduct(IdefixArray3D mat1, IdefixArray3D mat2); + + protected: + T & linearOperator; + real currentError; + real targetError; + int maxiter; // Maximum iteration allowed to achieve convergence + bool convStatus; // Convergence status + bool restart{false}; + + std::array beg; + std::array end; + std::array ntot; + + IdefixArray3D solution; + IdefixArray3D rhs; + IdefixArray3D res; // Residual +}; + +template +IterativeSolver::IterativeSolver(T &op, real error, int maxiter, + std::array ntot, std::array beg, std::array end) + : linearOperator(op) { + this->targetError = error; + this->maxiter = maxiter; + this->beg = beg; + this->end = end; + this->ntot = ntot; + this->restart = false; + this->currentError = 0; + this->res = IdefixArray3D ("Residual", this->ntot[KDIR], + this->ntot[JDIR], + this->ntot[IDIR]); +} + +template +void IterativeSolver::TestErrorL1() { + idfx::pushRegion("IterativeSolver::TestErrorL1"); + + // Loading needed attributes + IdefixArray3D res = this->res; + IdefixArray3D rhs = this->rhs; + + int ibeg, iend, jbeg, jend, kbeg, kend; + ibeg = this->beg[IDIR]; + iend = this->end[IDIR]; + jbeg = this->beg[JDIR]; + jend = this->end[JDIR]; + kbeg = this->beg[KDIR]; + kend = this->end[KDIR]; + + // Do the reduction on a vector + MyVector normL1Vector; + + // Sum of absolute residuals over the grid and sum of squared rhs + // both stored in a 2D reduction vector + idefix_reduce("absRes", + kbeg, kend, + jbeg, jend, + ibeg, iend, + KOKKOS_LAMBDA (int k, int j, int i, MyVector &localVector) { + localVector.v[0] += FABS(res(k,j,i)); + localVector.v[1] += rhs(k,j,i) * rhs(k,j,i); + }, + Kokkos::Sum(normL1Vector)); + + // Reduction on the whole grid + #ifdef WITH_MPI + MPI_Allreduce(MPI_IN_PLACE, &normL1Vector.v, 2, realMPI, MPI_SUM, MPI_COMM_WORLD); + #endif + + // Squared error + this->currentError = sqrt(normL1Vector.v[0] * normL1Vector.v[0] / normL1Vector.v[1]); + + // Checking Nans + if(std::isnan(currentError)) { + std::stringstream msg; + msg << "IterativeSolver:: Current error is Nan." << std::endl; + throw std::runtime_error(msg.str()); + } + + // Checking convergence + if(currentError <= this->targetError) { + this->convStatus = true; + #ifdef DEBUG_BICGSTAB + idfx::cout << "IterativeSolver:: Squared, normalized norm L1 is " << currentError + << " at convergence." << std::endl; + #endif + } + + idfx::popRegion(); +} + +template +void IterativeSolver::TestErrorL2() { + idfx::pushRegion("IterativeSolver::TestErrorL2"); + + // Loading needed attributes + IdefixArray3D res = this->res; + IdefixArray3D rhs = this->rhs; + + int ibeg, iend, jbeg, jend, kbeg, kend; + ibeg = this->beg[IDIR]; + iend = this->end[IDIR]; + jbeg = this->beg[JDIR]; + jend = this->end[JDIR]; + kbeg = this->beg[KDIR]; + kend = this->end[KDIR]; + + // Do the reduction on a vector + MyVector normL2Vector; + + // Sum of squared residuals over the grid and sum of squared rhs + // both stored in a 2D reduction vector + idefix_reduce("SumRes2", + kbeg, kend, + jbeg, jend, + ibeg, iend, + KOKKOS_LAMBDA (int k, int j, int i, MyVector &localVector) { + localVector.v[0] += res(k,j,i) * res(k,j,i); + localVector.v[1] += rhs(k,j,i) * rhs(k,j,i); + }, + Kokkos::Sum(normL2Vector)); + + // Reduction on the whole grid + #ifdef WITH_MPI + MPI_Allreduce(MPI_IN_PLACE, &normL2Vector.v, 2, realMPI, MPI_SUM, MPI_COMM_WORLD); + #endif + + // Squared error + this->currentError = sqrt(normL2Vector.v[0] / normL2Vector.v[1]); + //idfx::cout << "Error=" << this->currentError << std::endl; + + // Checking Nans + if(std::isnan(currentError)) { + std::stringstream msg; + msg << "IterativeSolver:: Current error is Nan." << std::endl; + throw std::runtime_error(msg.str()); + } + // Checking convergence + if(currentError <= this->targetError) { + this->convStatus = true; + #ifdef DEBUG_BICGSTAB + idfx::cout << "IterativeSolver:: Squared, normalized norm L2 is " << currentError + << " at convergence." << std::endl; + #endif + } + + idfx::popRegion(); +} + +template +void IterativeSolver::TestErrorLINF() { + idfx::pushRegion("IterativeSolver::TestErrorLINF"); + + // Loading needed attributes + IdefixArray3D res = this->res; + IdefixArray3D rhs = this->rhs; + + int ibeg, iend, jbeg, jend, kbeg, kend; + ibeg = this->beg[IDIR]; + iend = this->end[IDIR]; + jbeg = this->beg[JDIR]; + jend = this->end[JDIR]; + kbeg = this->beg[KDIR]; + kend = this->end[KDIR]; + + real maxRes2, rho2; + + // Searching for the maximum residual over the grid + idefix_reduce("MaxRes2", + kbeg, kend, + jbeg, jend, + ibeg, iend, + KOKKOS_LAMBDA (int k, int j, int i, real &localMax) { + localMax = std::fmax(res(k,j,i) * res(k,j,i), localMax); + }, + Kokkos::Max(maxRes2)); + + // Sum of squared rhs over the grid + idefix_reduce("SumDensity2", + kbeg, kend, + jbeg, jend, + ibeg, iend, + KOKKOS_LAMBDA (int k, int j, int i, real &localSum) { + localSum += rhs(k,j,i) * rhs(k,j,i); + }, + Kokkos::Sum(rho2)); + + // Reduction on the whole grid + #ifdef WITH_MPI + MPI_Allreduce(MPI_IN_PLACE, &maxRes2, 1, realMPI, MPI_MAX, MPI_COMM_WORLD); + MPI_Allreduce(MPI_IN_PLACE, &rho2, 1, realMPI, MPI_SUM, MPI_COMM_WORLD); + #endif + + // Squared error + currentError = sqrt(maxRes2/rho2); + + // Checking Nans + if(std::isnan(currentError)) { + std::stringstream msg; + msg << "IterativeSolver:: Current error is Nan." << std::endl; + throw std::runtime_error(msg.str()); + } + // Checking convergence + if(currentError <= this->targetError) { + this->convStatus = true; + #ifdef DEBUG_BICGSTAB + idfx::cout << "IterativeSolver:: Squared, normalized norm LINF is " << currentError + << " at convergence." << std::endl; + #endif + } + + idfx::popRegion(); +} + +template +void IterativeSolver::SetRes() { + idfx::pushRegion("IterativeSolver::SetRes"); + + // Loading needed attributes + IdefixArray3D rhs = this->rhs; + IdefixArray3D solution = this->solution; + IdefixArray3D res = this->res; + + // Computing operator + this->linearOperator(solution, res); // We store function output in res to spare workRes array + + int ibeg, iend, jbeg, jend, kbeg, kend; + ibeg = this->beg[IDIR]; + iend = this->end[IDIR]; + jbeg = this->beg[JDIR]; + jend = this->end[JDIR]; + kbeg = this->beg[KDIR]; + kend = this->end[KDIR]; + + // Computing residual + idefix_for("SetRes", kbeg, kend, jbeg, jend, ibeg, iend, + KOKKOS_LAMBDA (int k, int j, int i) { + res(k,j,i) = rhs(k,j,i) - res(k,j,i); + }); + + idfx::popRegion(); +} + +template +real IterativeSolver::ComputeDotProduct(IdefixArray3D mat1, IdefixArray3D mat2) { + idfx::pushRegion("IterativeSolver::ComputeDotProduct"); + + int ibeg, iend, jbeg, jend, kbeg, kend; + ibeg = this->beg[IDIR]; + iend = this->end[IDIR]; + jbeg = this->beg[JDIR]; + jend = this->end[JDIR]; + kbeg = this->beg[KDIR]; + kend = this->end[KDIR]; + + real sum; + + idefix_reduce("DotProduct", + kbeg, kend, + jbeg, jend, + ibeg, iend, + KOKKOS_LAMBDA (int k, int j, int i, real &localSum) { + localSum += mat1(k,j,i) * mat2(k,j,i); + }, + Kokkos::Sum(sum)); + + // Reduction on the whole grid + #ifdef WITH_MPI + MPI_Allreduce(MPI_IN_PLACE, &sum, 1, realMPI, MPI_SUM, MPI_COMM_WORLD); + #endif + + idfx::popRegion(); + + return sum; +} + + +template +real IterativeSolver::GetError() { + return(currentError); +} + +#endif //UTILS_ITERATIVESOLVER_ITERATIVESOLVER_HPP_ diff --git a/src/utils/iterativesolver/jacobi.hpp b/src/utils/iterativesolver/jacobi.hpp new file mode 100644 index 00000000..941415e4 --- /dev/null +++ b/src/utils/iterativesolver/jacobi.hpp @@ -0,0 +1,102 @@ +// *********************************************************************************** +// Idefix MHD astrophysical code +// Copyright(C) Geoffroy R. J. Lesur +// and other code contributors +// Licensed under CeCILL 2.1 License, see COPYING for more information +// *********************************************************************************** + +#ifndef UTILS_ITERATIVESOLVER_JACOBI_HPP_ +#define UTILS_ITERATIVESOLVER_JACOBI_HPP_ +#include +#include "idefix.hpp" +#include "vector.hpp" +#include "iterativesolver.hpp" + +// The jacobi derives from the iterativesolver class +template +class Jacobi : public IterativeSolver { + public: + Jacobi(T &op, real error, int maxIter, real step, + std::array ntot, std::array beg, std::array end); + + int Solve(IdefixArray3D &guess, IdefixArray3D &rhs); + void ShowConfig(); + void PerformIter(); + + private: + real dt; +}; + +template +Jacobi::Jacobi(T &op, real error, int maxIter, real step, + std::array ntot, std::array beg, std::array end) : + IterativeSolver(op, error, maxIter, ntot, beg, end), dt(step) { + // do nothing + } + +template +int Jacobi::Solve(IdefixArray3D &guess, IdefixArray3D &rhs) { + idfx::pushRegion("Jacobi::Solve"); + this->solution = guess; + this->rhs = rhs; + // Re-initialise convStatus + this->convStatus = false; + // Residual initialisation + this->SetRes(); + + int n = 0; + while(this->convStatus != true && n < this->maxiter) { + this->PerformIter(); + n++; + } + + if(n == this->maxiter) { + IDEFIX_WARNING("Jacobi:: Failed to converge before reaching max iter." + "You should consider to use (P)BICGSTAB."); + } + + idfx::popRegion(); + return(n); +} + +template +void Jacobi::PerformIter() { + idfx::pushRegion("Jacobi::PerformIter"); + + // Loading needed attributes + IdefixArray3D solution = this->solution; + IdefixArray3D res = this->res; + real dt = this->dt; + + int ibeg, iend, jbeg, jend, kbeg, kend; + ibeg = this->beg[IDIR]; + iend = this->end[IDIR]; + jbeg = this->beg[JDIR]; + jend = this->end[JDIR]; + kbeg = this->beg[KDIR]; + kend = this->end[KDIR]; + + // Perform one iteration of the Jacobi method + idefix_for("JacIter", kbeg, kend, jbeg, jend, ibeg, iend, + KOKKOS_LAMBDA (int k, int j, int i) { + solution(k, j, i) = solution(k, j, i) - dt * res(k,j,i); + }); + + // Update residual + this->SetRes(); + + // Test convergence + this->TestErrorL2(); + + idfx::popRegion(); +} + +template +void Jacobi::ShowConfig() { + idfx::cout << "Jacobi: TargetError: " << this->targetError << std::endl; + idfx::cout << "Jacobi: Maximum iterations: " << this->maxiter << std::endl; + idfx::cout << "Jacobi: step: " << this->dt << std::endl; + return; +} + +#endif // UTILS_ITERATIVESOLVER_JACOBI_HPP_ diff --git a/src/utils/iterativesolver/minres.hpp b/src/utils/iterativesolver/minres.hpp new file mode 100644 index 00000000..148457e5 --- /dev/null +++ b/src/utils/iterativesolver/minres.hpp @@ -0,0 +1,226 @@ +// *********************************************************************************** +// Idefix MHD astrophysical code +// Copyright(C) Geoffroy R. J. Lesur +// and other code contributors +// Licensed under CeCILL 2.1 License, see COPYING for more information +// *********************************************************************************** + +#ifndef UTILS_ITERATIVESOLVER_MINRES_HPP_ +#define UTILS_ITERATIVESOLVER_MINRES_HPP_ +#include +#include "idefix.hpp" +#include "vector.hpp" +#include "iterativesolver.hpp" + +// The conjugate gradient derives from the iterativesolver class +template +class Minres : public IterativeSolver { + public: + Minres(T &op, real error, int maxIter, + std::array ntot, std::array beg, std::array end); + + int Solve(IdefixArray3D &guess, IdefixArray3D &rhs); + + void PerformIter(); + void InitSolver(); + void ShowConfig(); + + private: + real alpha; // MINRES parameter + real beta; // MINRES parameter + real previousError; + bool shouldReset{false}; + bool firstStep{false}; + + IdefixArray3D p0; // Search direction for gradient descent + IdefixArray3D p1; // Search direction for gradient descent + IdefixArray3D p2; // Search direction for gradient descent + IdefixArray3D s0; // Search direction for gradient descent + IdefixArray3D s1; // Search direction for gradient descent + IdefixArray3D s2; // Search direction for gradient descent + IdefixArray3D work1; // work array + IdefixArray3D work2; // work array + IdefixArray3D work3; // work array +}; + +template +Minres::Minres(T &op, real error, int maxiter, + std::array ntot, std::array beg, std::array end) : + IterativeSolver(op, error, maxiter, ntot, beg, end) { + // MINRES scalars initialisation + this->alpha = 1.0; + this->beta = 1.0; + + + + this->p0 = IdefixArray3D ("p0", this->ntot[KDIR], + this->ntot[JDIR], + this->ntot[IDIR]); + this->p1 = IdefixArray3D ("p0", this->ntot[KDIR], + this->ntot[JDIR], + this->ntot[IDIR]); + this->p2 = IdefixArray3D ("p0", this->ntot[KDIR], + this->ntot[JDIR], + this->ntot[IDIR]); + + this->s0 = IdefixArray3D ("p0", this->ntot[KDIR], + this->ntot[JDIR], + this->ntot[IDIR]); + this->s1 = IdefixArray3D ("p0", this->ntot[KDIR], + this->ntot[JDIR], + this->ntot[IDIR]); + this->s2 = IdefixArray3D ("p0", this->ntot[KDIR], + this->ntot[JDIR], + this->ntot[IDIR]); +} + +template +int Minres::Solve(IdefixArray3D &guess, IdefixArray3D &rhs) { + idfx::pushRegion("Minres::Solve"); + this->solution = guess; + this->rhs = rhs; + + // Re-initialise convStatus + this->convStatus = false; + + int n = 0; + + while(this->convStatus != true && n < this->maxiter) { + if(n%20==0) { + // Reset Minres every 20 step to avoid breakdown + this->firstStep = true; + this->InitSolver(); + } + this->PerformIter(); + n++; + } + + if(n == this->maxiter) { + idfx::cout << "Minres:: Reached max iter." << std::endl; + IDEFIX_ERROR("Minres:: Failed to converge before reaching max iter." + "You should consider to use a preconditionner."); + } + + idfx::popRegion(); + return(n); +} + +template +void Minres::InitSolver() { + idfx::pushRegion("Minres::InitSolver"); + // Residual initialisation + this->SetRes(); + + Kokkos::deep_copy(this->p0, this->res); // (Re)setting reference residual + this->linearOperator(this->p0, this->s0); // (Re)setting associated laplacian + //Kokkos::deep_copy(this->p1, this->p0); + //Kokkos::deep_copy(this->s1, this->s0); + + idfx::popRegion(); +} + +template +void Minres::PerformIter() { + idfx::pushRegion("Minres::PerformIter"); + + // Loading needed attributes + auto x = this->solution; + auto r = this->res; + auto p0 = this->p0; + auto p1 = this->p1; + auto p2 = this->p2; + auto s0 = this->s0; + auto s1 = this->s1; + auto s2 = this->s2; + + Kokkos::deep_copy(p2,p1); + Kokkos::deep_copy(p1,p0); + Kokkos::deep_copy(s2,s1); + Kokkos::deep_copy(s1,s0); + + int ibeg, iend, jbeg, jend, kbeg, kend; + ibeg = this->beg[IDIR]; + iend = this->end[IDIR]; + jbeg = this->beg[JDIR]; + jend = this->end[JDIR]; + kbeg = this->beg[KDIR]; + kend = this->end[KDIR]; + + // ***** Step 1. + + real alpha = this->ComputeDotProduct(r,s1) / (this->ComputeDotProduct(s1,s1)); + + // Checking for Nans + if(std::isnan(alpha)) { + idfx::cout << "Minres:: alpha is nan in step 1." << std::endl; + this->shouldReset = true; + idfx::popRegion(); + return; + // IDEFIX_ERROR("rho is nan in step 1"); + } + + // ******* Step 2 + idefix_for("UpdateDir", kbeg, kend, jbeg, jend, ibeg, iend, + KOKKOS_LAMBDA (int k, int j, int i) { + x(k,j,i) = x(k,j,i) + alpha * p1(k,j,i); + r(k,j,i) = r(k,j,i) - alpha * s1(k,j,i); + }); + + this->TestErrorL2(); + /* + if(this->currentError/this->previousError>0.999) { + this->firstStep = true; + Kokkos::deep_copy(p0,r); + this->linearOperator(this->p0, this->s0); + idfx::cout << "Reset" << std::endl; + idfx::popRegion(); + return; + }*/ + + this->previousError = this->currentError; + Kokkos::deep_copy(p0, s1); + + this->linearOperator(s1, s0); + + real beta1 = this->ComputeDotProduct(s0,s1) / this->ComputeDotProduct(s1,s1); + + // Checking for Nans + if(std::isnan(beta1)) { + idfx::cout << "Minres:: beta1 is nan in step 2." << std::endl; + idfx::popRegion(); + return; + // IDEFIX_ERROR("rho is nan in step 1"); + } + + idefix_for("UpdateDir", kbeg, kend, jbeg, jend, ibeg, iend, + KOKKOS_LAMBDA (int k, int j, int i) { + p0(k,j,i) -= beta1 * p1(k,j,i); + s0(k,j,i) -= beta1 * s1(k,j,i); + }); + if(!this->firstStep) { + real beta2 = this->ComputeDotProduct(s0,s2) / this->ComputeDotProduct(s2,s2); + idefix_for("UpdateDir", kbeg, kend, jbeg, jend, ibeg, iend, + KOKKOS_LAMBDA (int k, int j, int i) { + p0(k,j,i) -= beta2 * p2(k,j,i); + s0(k,j,i) -= beta2 * s2(k,j,i); + }); + } else { + this->firstStep = false; + } + + idfx::popRegion(); +} + + + +template +void Minres::ShowConfig() { + idfx::pushRegion("Minres::ShowConfig"); + idfx::cout << "Minres: TargetError: " << this->targetError << std::endl; + idfx::cout << "Minres: Maximum iterations: " << this->maxiter << std::endl; + idfx::popRegion(); + return; +} + + +#endif // UTILS_ITERATIVESOLVER_MINRES_HPP_ diff --git a/src/utils/vector.hpp b/src/utils/vector.hpp new file mode 100644 index 00000000..68c49eea --- /dev/null +++ b/src/utils/vector.hpp @@ -0,0 +1,46 @@ +// *********************************************************************************** +// Idefix MHD astrophysical code +// Copyright(C) Geoffroy R. J. Lesur +// and other code contributors +// Licensed under CeCILL 2.1 License, see COPYING for more information +// *********************************************************************************** + + +#ifndef UTILS_VECTOR_HPP_ +#define UTILS_VECTOR_HPP_ + +#include "idefix.hpp" +// a vector structure of size N for reduction in sums +// This struct(=class) defines the += operator on its elements +// Which is used by the reduction function. Other operators +// Can be implemented as well. +template +struct Vector { + T v[N]; + KOKKOS_FUNCTION Vector() { + for (int i = 0; i < N; ++i) { + v[i] = 0; + } + } + + KOKKOS_FUNCTION void operator+=(Vector const volatile& vec) volatile { + for (int i = 0; i < N; ++i) { + v[i] = v[i] + vec.v[i]; + } + } +}; + +// Shortcut for what follows: a vector of 2 reals +typedef Vector MyVector; + +// Define the reduction operator in Kokkos space +namespace Kokkos { +template<> +struct reduction_identity< MyVector > { + KOKKOS_FORCEINLINE_FUNCTION static MyVector sum() { + return MyVector(); + } +}; +} + +#endif// UTILS_VECTOR_HPP_ diff --git a/test/Dust/DustEnergy/definitions.hpp b/test/Dust/DustEnergy/definitions.hpp new file mode 100644 index 00000000..498be97a --- /dev/null +++ b/test/Dust/DustEnergy/definitions.hpp @@ -0,0 +1,4 @@ +#define COMPONENTS 1 +#define DIMENSIONS 1 + +#define GEOMETRY CARTESIAN diff --git a/test/Dust/DustEnergy/idefix.ini b/test/Dust/DustEnergy/idefix.ini new file mode 100644 index 00000000..7236b68a --- /dev/null +++ b/test/Dust/DustEnergy/idefix.ini @@ -0,0 +1,35 @@ +# This test checks that the total energy (thermal+dust kinetic+gas kinetic) +# is effectively conserved when drag is present + +[Grid] +X1-grid 1 0.0 500 u 1.0 +X2-grid 1 0.0 1 u 1.0 +X3-grid 1 0.0 1 u 1.0 + +[TimeIntegrator] +CFL 0.8 +tstop 1.0 +first_dt 1.e-4 +nstages 2 + +[Hydro] +solver hllc +gamma 1.4 + +[Dust] +nSpecies 1 +drag tau 1.0 +drag_feedback yes + +[Boundary] +X1-beg periodic +X1-end periodic +X2-beg outflow +X2-end outflow +X3-beg outflow +X3-end outflow + +[Output] +dmp 1.0 +analysis 0.01 +log 1000 diff --git a/test/Dust/DustEnergy/python/testidefix.py b/test/Dust/DustEnergy/python/testidefix.py new file mode 100755 index 00000000..d7d766a1 --- /dev/null +++ b/test/Dust/DustEnergy/python/testidefix.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +Created on Mon Mar 13 14:45:58 2023 + +@author: lesurg +""" +import sys +import numpy as np +import argparse +import matplotlib.pyplot as plt + +parser = argparse.ArgumentParser() +parser.add_argument("-noplot", + default=False, + help="disable plotting", + action="store_true") + + +args, unknown=parser.parse_known_args() + + +# solve eq. A6 in Riols & Lesur (2018) + + + + + +# load the dat file produced by the setup +raw=np.loadtxt('../timevol.dat',skiprows=1) +t=raw[:,0] +Ekg=raw[:,1] +Ekd=raw[:,2] +Eth=raw[:,3] + +etot=Ekg+Ekd+Eth + +if not(args.noplot): + plt.figure() + plt.plot(t,Ekg,label="Ek_g") + plt.plot(t,Ekd,label="Ek_d") + plt.plot(t,Eth,label="Eth") + plt.plot(t,etot,'--',label="Etot") + plt.legend() + plt.xlabel("t") + plt.ylabel("Flow energy") + plt.show() + +# Compute relative evolution of total energy +error=abs((etot[-1]-etot[0])/etot[0]) + +print("error=%f"%error) +if(error<1e-4): + print("Success!") +else: + print("Failure!") + sys.exit(1) diff --git a/test/Dust/DustEnergy/setup.cpp b/test/Dust/DustEnergy/setup.cpp new file mode 100644 index 00000000..cba900a0 --- /dev/null +++ b/test/Dust/DustEnergy/setup.cpp @@ -0,0 +1,90 @@ +#include "idefix.hpp" +#include "setup.hpp" + +#define FILENAME "timevol.dat" + + +// Analyse data to produce an output +void Analysis(DataBlock & data) { + + auto Vc=data.hydro->Vc; + auto Vcd=data.dust[0]->Vc; + real Ekg,Ekd, Etherm; + real gamma = data.hydro->eos->GetGamma(); + + idefix_reduce("Ek",data.beg[KDIR],data.end[KDIR],data.beg[JDIR],data.end[JDIR],data.beg[IDIR],data.end[IDIR], + KOKKOS_LAMBDA (int k, int j, int i, real &vx2) { + vx2 += 0.5*Vc(RHO,k,j,i)*Vc(VX1,k,j,i) * Vc(VX1,k,j,i); + }, Kokkos::Sum(Ekg) ); + + idefix_reduce("Ek",data.beg[KDIR],data.end[KDIR],data.beg[JDIR],data.end[JDIR],data.beg[IDIR],data.end[IDIR], + KOKKOS_LAMBDA (int k, int j, int i, real &vx2) { + vx2 += 0.5*Vcd(RHO,k,j,i)*Vcd(VX1,k,j,i) * Vcd(VX1,k,j,i); + }, Kokkos::Sum(Ekd) ); + + idefix_reduce("RHO2",data.beg[KDIR],data.end[KDIR],data.beg[JDIR],data.end[JDIR],data.beg[IDIR],data.end[IDIR], + KOKKOS_LAMBDA (int k, int j, int i, real &e) { + e += Vc(PRS,k,j,i)/(gamma-1); + }, Kokkos::Sum(Etherm) ); + + Ekg /= (data.np_int[KDIR]*data.np_int[JDIR]*data.np_int[IDIR]); + Ekd /= (data.np_int[KDIR]*data.np_int[JDIR]*data.np_int[IDIR]); + Etherm /= (data.np_int[KDIR]*data.np_int[JDIR]*data.np_int[IDIR]); + + if(idfx::prank == 0) { + std::ofstream f; + f.open(FILENAME,std::ios::app); + f.precision(10); + f << std::scientific << data.t << "\t" << Ekg << "\t" << Ekd << "\t" << Etherm << std::endl; + f.close(); + } + +} + +// Initialisation routine. Can be used to allocate +// Arrays or variables which are used later on +Setup::Setup(Input &input, Grid &grid, DataBlock &data, Output &output) { + // stopping time + + output.EnrollAnalysis(&Analysis); + if(!input.restartRequested) { + // Initialise the output file + std::ofstream f; + f.open(FILENAME,std::ios::trunc); + f << "t\t\t Ekg \t\t Ekd << Etherm" << std::endl; + f.close(); + } +} + +// This routine initialize the flow +// Note that data is on the device. +// One can therefore define locally +// a datahost and sync it, if needed +void Setup::InitFlow(DataBlock &data) { + // Create a host copy + DataBlockHost d(data); + + + for(int k = 0; k < d.np_tot[KDIR] ; k++) { + for(int j = 0; j < d.np_tot[JDIR] ; j++) { + for(int i = 0; i < d.np_tot[IDIR] ; i++) { + + d.Vc(RHO,k,j,i) = 1.0; + d.dustVc[0](RHO,k,j,i) = 0.5; + + d.Vc(VX1,k,j,i) = 2.0; + d.dustVc[0](VX1,k,j,i) = -2.0; + + d.Vc(PRS,k,j,i) = 1.0; + } + } + } + + // Send it all, if needed + d.SyncToDevice(); +} + +// Analyse data to produce an output +void MakeAnalysis(DataBlock & data) { + +} diff --git a/test/Dust/DustEnergy/testme.py b/test/Dust/DustEnergy/testme.py new file mode 100755 index 00000000..f3b413e8 --- /dev/null +++ b/test/Dust/DustEnergy/testme.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python3 + +""" + +@author: glesur +""" +import os +import sys +sys.path.append(os.getenv("IDEFIX_DIR")) + +import pytools.idfx_test as tst + +name="dump.0001.dmp" + +def testMe(test): + test.configure() + test.compile() + inifiles=["idefix.ini"] + + # loop on all the ini files for this test + for ini in inifiles: + test.run(inputFile=ini) + if test.init: + test.makeReference(filename=name) + test.standardTest() + test.nonRegressionTest(filename=name) + + +test=tst.idfxTest() + +if not test.all: + if(test.check): + test.checkOnly(filename=name) + else: + testMe(test) +else: + test.noplot = True + test.reconstruction=2 + testMe(test) diff --git a/test/Dust/DustyWave/definitions.hpp b/test/Dust/DustyWave/definitions.hpp new file mode 100644 index 00000000..cdbe23d6 --- /dev/null +++ b/test/Dust/DustyWave/definitions.hpp @@ -0,0 +1,5 @@ +#define COMPONENTS 1 +#define DIMENSIONS 1 +#define ISOTHERMAL + +#define GEOMETRY CARTESIAN diff --git a/test/Dust/DustyWave/idefix.ini b/test/Dust/DustyWave/idefix.ini new file mode 100644 index 00000000..3b65e065 --- /dev/null +++ b/test/Dust/DustyWave/idefix.ini @@ -0,0 +1,35 @@ +# This test checks the dissipation of a sound wave by a dust grains +# partially coupled to the gas (Riols & Lesur 2018, appendix A) + +[Grid] +X1-grid 1 0.0 500 u 1.0 +X2-grid 1 0.0 1 u 1.0 +X3-grid 1 0.0 1 u 1.0 + +[TimeIntegrator] +CFL 0.8 +tstop 10.0 +first_dt 1.e-4 +nstages 2 + +[Hydro] +solver hllc +csiso constant 1.0 + +[Dust] +nSpecies 1 +drag tau 1.0 +drag_feedback yes + +[Boundary] +X1-beg periodic +X1-end periodic +X2-beg outflow +X2-end outflow +X3-beg outflow +X3-end outflow + +[Output] +dmp 10.0 +analysis 0.01 +log 1000 diff --git a/test/Dust/DustyWave/python/testidefix.py b/test/Dust/DustyWave/python/testidefix.py new file mode 100755 index 00000000..c98bc821 --- /dev/null +++ b/test/Dust/DustyWave/python/testidefix.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +Created on Mon Mar 13 14:45:58 2023 + +@author: lesurg +""" +import sys +import numpy as np +import argparse +import matplotlib.pyplot as plt + +parser = argparse.ArgumentParser() +parser.add_argument("-noplot", + default=False, + help="disable plotting", + action="store_true") + + +args, unknown=parser.parse_known_args() + + +# solve eq. A6 in Riols & Lesur (2018) +cs=1.0 +chi=1.0 +kx=2.0*np.pi +taus=1.0 +poly=np.zeros(4,dtype=complex) +poly[0]=1 +poly[1]=1j/taus*(1+chi) +poly[2]=-kx**2*cs**2 +poly[3]=-1j*kx**2*cs**2/taus +sol=np.roots(poly) + +# Get the minimal decay rate (this should be the one that pops up) +tau=np.amax(np.imag(sol)) + + + + +# load the dat file produced by the setup +raw=np.loadtxt('../timevol.dat',skiprows=1) +t=raw[:,0] +vx2=raw[:,1] +rho2=raw[:,2] + +etot=vx2+rho2 + +if not(args.noplot): + plt.figure() + plt.semilogy(t,etot,label="idefix") + plt.semilogy(t,np.exp(2*tau*t)*etot[10],'--',label="theoretical decay rate") + plt.legend() + plt.xlabel("t") + plt.ylabel("Wave energy") + plt.show() + +# Compute decay rate: +tau_measured=t[-1]/(2*np.log(etot[-1]/etot[0])) +# error on the decay rate: +error=(tau_measured-tau)/tau + +print("error=%f"%error) +if(error<0.065): + print("Success!") +else: + print("Failure!") + sys.exit(1) diff --git a/test/Dust/DustyWave/setup.cpp b/test/Dust/DustyWave/setup.cpp new file mode 100644 index 00000000..9cd5c66b --- /dev/null +++ b/test/Dust/DustyWave/setup.cpp @@ -0,0 +1,82 @@ +#include "idefix.hpp" +#include "setup.hpp" + +#define FILENAME "timevol.dat" + +// Analyse data to produce an output +void Analysis(DataBlock & data) { + + auto Vc=data.hydro->Vc; + real Ek, Erho; + idefix_reduce("Vx2",data.beg[KDIR],data.end[KDIR],data.beg[JDIR],data.end[JDIR],data.beg[IDIR],data.end[IDIR], + KOKKOS_LAMBDA (int k, int j, int i, real &vx2) { + vx2 += Vc(VX1,k,j,i) * Vc(VX1,k,j,i); + }, Kokkos::Sum(Ek) ); + + idefix_reduce("RHO2",data.beg[KDIR],data.end[KDIR],data.beg[JDIR],data.end[JDIR],data.beg[IDIR],data.end[IDIR], + KOKKOS_LAMBDA (int k, int j, int i, real &rho2) { + rho2 += (Vc(RHO,k,j,i)-1)* (Vc(RHO,k,j,i)-1); + }, Kokkos::Sum(Erho) ); + + Ek /= (data.np_int[KDIR]*data.np_int[JDIR]*data.np_int[IDIR]); + Erho /= (data.np_int[KDIR]*data.np_int[JDIR]*data.np_int[IDIR]); + + if(idfx::prank == 0) { + std::ofstream f; + f.open(FILENAME,std::ios::app); + f.precision(10); + f << std::scientific << data.t << "\t" << Ek << "\t" << Erho << std::endl; + f.close(); + } + +} + +// Initialisation routine. Can be used to allocate +// Arrays or variables which are used later on +Setup::Setup(Input &input, Grid &grid, DataBlock &data, Output &output) { + + output.EnrollAnalysis(&Analysis); + if(!input.restartRequested) { + // Initialise the output file + std::ofstream f; + f.open(FILENAME,std::ios::trunc); + f << "t\t\t vx2 \t\t rho2" << std::endl; + f.close(); + } +} + +// This routine initialize the flow +// Note that data is on the device. +// One can therefore define locally +// a datahost and sync it, if needed +void Setup::InitFlow(DataBlock &data) { + // Create a host copy + DataBlockHost d(data); + + + for(int k = 0; k < d.np_tot[KDIR] ; k++) { + for(int j = 0; j < d.np_tot[JDIR] ; j++) { + for(int i = 0; i < d.np_tot[IDIR] ; i++) { + + d.Vc(RHO,k,j,i) = 1.0; + d.dustVc[0](RHO,k,j,i) = 1.0; + + d.Vc(VX1,k,j,i) = 0.01*sin(2.0*M_PI*d.x[IDIR](i)); + d.dustVc[0](VX1,k,j,i) = 0.0; + +#if HAVE_ENERGY + d.Vc(PRS,k,j,i) = 1.0; +#endif + + } + } + } + + // Send it all, if needed + d.SyncToDevice(); +} + +// Analyse data to produce an output +void MakeAnalysis(DataBlock & data) { + +} diff --git a/test/Dust/DustyWave/testme.py b/test/Dust/DustyWave/testme.py new file mode 100755 index 00000000..b6a75ce3 --- /dev/null +++ b/test/Dust/DustyWave/testme.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python3 + +""" + +@author: glesur +""" +import os +import sys +sys.path.append(os.getenv("IDEFIX_DIR")) + +import pytools.idfx_test as tst + +name="dump.0001.dmp" + +def testMe(test): + test.configure() + test.compile() + inifiles=["idefix.ini"] + + # loop on all the ini files for this test + for ini in inifiles: + test.run(inputFile=ini) + if test.init: + test.makeReference(filename=name) + test.standardTest() + test.nonRegressionTest(filename=name,tolerance=1e-14) + + +test=tst.idfxTest() + +if not test.all: + if(test.check): + test.checkOnly(filename=name) + else: + testMe(test) +else: + test.noplot = True + test.reconstruction=2 + testMe(test) diff --git a/test/Dust/FargoPlanet/definitions.hpp b/test/Dust/FargoPlanet/definitions.hpp new file mode 100644 index 00000000..deac509a --- /dev/null +++ b/test/Dust/FargoPlanet/definitions.hpp @@ -0,0 +1,6 @@ +#define COMPONENTS 3 +#define DIMENSIONS 2 + +#define ISOTHERMAL + +#define GEOMETRY POLAR diff --git a/test/Dust/FargoPlanet/idefix.ini b/test/Dust/FargoPlanet/idefix.ini new file mode 100644 index 00000000..b5c55b8c --- /dev/null +++ b/test/Dust/FargoPlanet/idefix.ini @@ -0,0 +1,54 @@ +[Grid] +X1-grid 1 0.4 128 l 2.5 +X2-grid 1 0.0 256 u 6.283185307179586 +X3-grid 1 -0.0125 1 u 0.0125 + +[TimeIntegrator] +CFL 0.5 +tstop 100.0 +first_dt 1.e-3 +nstages 2 + +[Hydro] +solver hllc +csiso userdef +# viscosity explicit userdef + +[Fargo] +velocity userdef + +[Dust] +nSpecies 3 +drag userdef 1.0 0.2 0.04 # St=1, 0.2, 0.04 +drag_feedback yes + +[Gravity] +potential central planet +Mcentral 1.0 + +[Boundary] +X1-beg userdef +X1-end userdef +X2-beg periodic +X2-end periodic +X3-beg outflow +X3-end outflow + +[Setup] +sigma0 0.125 +sigmaSlope 0.5 +h0 0.05 +alpha 1.0e-4 + +[Planet] +integrator analytical +planetToPrimary 1.0e-3 +initialDistance 1.0 +feelDisk false +feelPlanets false +smoothing plummer 0.03 0.0 + +[Output] +vtk 6.283185307179586 +dmp 10.0 +log 100 diff --git a/test/Dust/FargoPlanet/setup.cpp b/test/Dust/FargoPlanet/setup.cpp new file mode 100644 index 00000000..af611911 --- /dev/null +++ b/test/Dust/FargoPlanet/setup.cpp @@ -0,0 +1,219 @@ +#include +#include "idefix.hpp" +#include "setup.hpp" + +real sigmaSlopeGlob; +real sigma0Glob; +real h0Glob; +real HidealGlob; +real gammaGlob; +real densityFloorGlob; +real alphaGlob; + + +void MySoundSpeed(DataBlock &data, const real t, IdefixArray3D &cs) { + IdefixArray1D x1=data.x[IDIR]; + real h0 = h0Glob; + idefix_for("MySoundSpeed",0,data.np_tot[KDIR],0,data.np_tot[JDIR],0,data.np_tot[IDIR], + KOKKOS_LAMBDA (int k, int j, int i) { + real R = x1(i); + cs(k,j,i) = h0/sqrt(R); + }); +} + +// drag coefficient assuming vertical hydrostatic equilirbium +// And using the fact that Vc(RHO) is the surface density Sigma + +// Here, we start from the definition of the stopping time (see doc): +// gamma_i = 1 / (tau_s * Sigma) = Omega / (St * Sigma) +// by definition of the Stokes number (assuming Epstein) +// St = Omega * taus = Omega * rho_s * a / (rho_mid * c_s) +// Using the vertical equilibrium: rho_mid = Sigma/(sqrt(2pi)*h) +// St = sqrt(2*pi) * rho_s a / Sigma +// Hence the product St*Sigma is a constant for each specie. +// We are free to define beta = St*Sigma/sigma0 so that gamma_i = Omega/(beta_i*sigma0) + +// In this setup, we assume Sigma = sigma0 @ R=1, so that St(R=1) = beta +// Note that in this setup, sigma0 is entirely scale-free because there is no +// gravitational interraction due to the gas. + +// Assuming Epstein drag and a disk with physical surface density Sigma_phys, +// the particle size is related to beta through: +// a = 20 cm * beta * (Sigma_phys/100 g.cm^-2) / (rho_s / 2 g.cm^-3) +// NB: checked against A. Johansen+ (2007) + +void MyDrag(DataBlock *data, real beta, IdefixArray3D &gamma) { + // Compute the drag coefficient gamma from the input beta + auto x1 = data->x[IDIR]; + real sigma0 = sigma0Glob; + + idefix_for("MyDrag",0,data->np_tot[KDIR],0,data->np_tot[JDIR],0,data->np_tot[IDIR], + KOKKOS_LAMBDA (int k, int j, int i) { + real Omega = pow(x1(i),-1.5); + gamma(k,j,i) = Omega / (beta*sigma0); + }); +} + +// User-defined boundaries +void UserdefBoundary(Hydro *hydro, int dir, BoundarySide side, real t) { + IdefixArray4D Vc = hydro->Vc; + auto *data = hydro->data; + IdefixArray1D x1 = data->x[IDIR]; + IdefixArray1D x3 = data->x[KDIR]; + if(dir==IDIR) { + int ighost,ibeg,iend; + if(side == left) { + ighost = data->beg[IDIR]; + ibeg = 0; + iend = data->beg[IDIR]; + idefix_for("UserDefBoundary",0,data->np_tot[KDIR],0,data->np_tot[JDIR],ibeg,iend, + KOKKOS_LAMBDA (int k, int j, int i) { + real R=x1(i); + real z=x3(k); + real Vk = 1.0/sqrt(R); + + Vc(RHO,k,j,i) = Vc(RHO,k,j,2*ighost - i +1); + Vc(VX1,k,j,i) = - Vc(VX1,k,j,2*ighost - i +1); + Vc(VX2,k,j,i) = Vk; + Vc(VX3,k,j,i) = Vc(VX3,k,j,2*ighost - i +1); + }); + } + else if(side==right) { + ighost = data->end[IDIR]-1; + ibeg=data->end[IDIR]; + iend=data->np_tot[IDIR]; + idefix_for("UserDefBoundary",0,data->np_tot[KDIR],0,data->np_tot[JDIR],ibeg,iend, + KOKKOS_LAMBDA (int k, int j, int i) { + real R=x1(i); + real z=x3(k); + real Vk = 1.0/sqrt(R); + + Vc(RHO,k,j,i) = Vc(RHO,k,j,ighost); + Vc(VX1,k,j,i) = Vc(VX1,k,j,ighost); + Vc(VX2,k,j,i) = Vk; + Vc(VX3,k,j,i) = Vc(VX3,k,j,ighost); + }); + } + } +} + +void UserdefBoundaryDust(Fluid *dust, int dir, BoundarySide side, real t) { + IdefixArray4D Vc = dust->Vc; + auto data = dust->data; + IdefixArray1D x1 = data->x[IDIR]; + IdefixArray1D x3 = data->x[KDIR]; + if(dir==IDIR) { + int ighost,ibeg,iend; + if(side == left) { + ighost = data->beg[IDIR]; + ibeg = 0; + iend = data->beg[IDIR]; + idefix_for("UserDefBoundary",0,data->np_tot[KDIR],0,data->np_tot[JDIR],ibeg,iend, + KOKKOS_LAMBDA (int k, int j, int i) { + real R=x1(i); + real z=x3(k); + real Vk = 1.0/sqrt(R); + + Vc(RHO,k,j,i) = Vc(RHO,k,j,2*ighost - i +1); + Vc(VX1,k,j,i) = - Vc(VX1,k,j,2*ighost - i +1); + Vc(VX2,k,j,i) = Vk; + Vc(VX3,k,j,i) = Vc(VX3,k,j,2*ighost - i +1); + }); + } + else if(side==right) { + ighost = data->end[IDIR]-1; + ibeg=data->end[IDIR]; + iend=data->np_tot[IDIR]; + idefix_for("UserDefBoundary",0,data->np_tot[KDIR],0,data->np_tot[JDIR],ibeg,iend, + KOKKOS_LAMBDA (int k, int j, int i) { + real R=x1(i); + real z=x3(k); + real Vk = 1.0/sqrt(R); + + Vc(RHO,k,j,i) = Vc(RHO,k,j,ighost); + Vc(VX1,k,j,i) = Vc(VX1,k,j,ighost); + Vc(VX2,k,j,i) = Vk; + Vc(VX3,k,j,i) = Vc(VX3,k,j,ighost); + }); + } + } +} + + + +void FargoVelocity(DataBlock &data, IdefixArray2D &Vphi) { + IdefixArray1D x1 = data.x[IDIR]; + + idefix_for("FargoVphi",0,data.np_tot[KDIR], 0, data.np_tot[IDIR], + KOKKOS_LAMBDA (int k, int i) { + Vphi(k,i) = 1.0/sqrt(x1(i)); + }); +} + + +// Default constructor +// Initialisation routine. Can be used to allocate +// Arrays or variables which are used later on +Setup::Setup(Input &input, Grid &grid, DataBlock &data, Output &output)// : m_planet(0)//, Planet &planet) +{ + // Set the function for userdefboundary + data.hydro->EnrollUserDefBoundary(&UserdefBoundary); + if(data.haveDust) { + int nSpecies = data.dust.size(); + for(int n = 0 ; n < nSpecies ; n++) { + data.dust[n]->EnrollUserDefBoundary(&UserdefBoundaryDust); + data.dust[n]->drag->EnrollUserDrag(&MyDrag); + } + } + + data.hydro->EnrollIsoSoundSpeed(&MySoundSpeed); + if(data.haveFargo) + data.fargo->EnrollVelocity(&FargoVelocity); + sigmaSlopeGlob = input.Get("Setup","sigmaSlope",0); + sigma0Glob = input.Get("Setup","sigma0",0); + h0Glob = input.Get("Setup","h0",0); +} + +// This routine initialize the flow +// Note that data is on the device. +// One can therefore define locally +// a datahost and sync it, if needed +void Setup::InitFlow(DataBlock &data) { + // Create a host copy + DataBlockHost d(data); + real h0=h0Glob; + real sigmaSlope = sigmaSlopeGlob; + real sigma0 = sigma0Glob; + + for(int k = 0; k < d.np_tot[KDIR] ; k++) { + for(int j = 0; j < d.np_tot[JDIR] ; j++) { + for(int i = 0; i < d.np_tot[IDIR] ; i++) { + real R=d.x[IDIR](i); + real z=d.x[KDIR](k); + real Vk=1.0/sqrt(R); + + real cs2=(h0*Vk)*(h0*Vk); + + d.Vc(RHO,k,j,i) = sigma0*pow(R,-sigmaSlope-1) * exp(1.0/ (cs2) * (1.0/sqrt(R*R+z*z)-1.0/R)) ; + d.Vc(VX1,k,j,i) = 0.0;//+sin(8*d.x[JDIR](j)); + d.Vc(VX3,k,j,i) = 0.0; + d.Vc(VX2,k,j,i) = Vk*sqrt( R / sqrt(R*R + z*z) -(2.0+sigmaSlope)*h0*h0); + + for(int n = 0 ; n < data.dust.size() ; n++) { + d.dustVc[n](RHO,k,j,i) = 1e-2*d.Vc(RHO,k,j,i); + d.dustVc[n](VX1,k,j,i) = 0.0; + d.dustVc[n](VX2,k,j,i) = Vk; + d.dustVc[n](VX3,k,j,i) = 0.0; + } + } + } + } + + // Send it all, if needed + d.SyncToDevice(); +} + +// Analyse data to produce an output +void MakeAnalysis(DataBlock & data) { + +} diff --git a/test/Dust/StreamingInstability/definitions.hpp b/test/Dust/StreamingInstability/definitions.hpp new file mode 100644 index 00000000..486a4ea3 --- /dev/null +++ b/test/Dust/StreamingInstability/definitions.hpp @@ -0,0 +1,4 @@ +#define COMPONENTS 3 +#define DIMENSIONS 3 +#define ISOTHERMAL +#define GEOMETRY CARTESIAN diff --git a/test/Dust/StreamingInstability/idefix.ini b/test/Dust/StreamingInstability/idefix.ini new file mode 100644 index 00000000..c04bdbf5 --- /dev/null +++ b/test/Dust/StreamingInstability/idefix.ini @@ -0,0 +1,41 @@ +[Grid] +X1-grid 1 -0.5 64 u 0.5 +X2-grid 1 -0.5 1 u 0.5 +X3-grid 1 -0.5 64 u 0.5 + +[TimeIntegrator] +CFL 0.8 +tstop 3000.0 +first_dt 1.e-4 +nstages 2 + +[Hydro] +solver hllc +rotation 1.0 +shearingBox -1.5 +csiso constant 1.0 + +[Dust] +nSpecies 1 +drag tau 1.0 # constant stopping time +drag_feedback yes + +[Gravity] +bodyForce userdef + +[Boundary] +X1-beg shearingbox +X1-end shearingbox +X2-beg periodic +X2-end periodic +X3-beg periodic +X3-end periodic + +[Setup] +epsilon 0.1 # Pressure gradient (=2*eta*R/H from JY07) +chi 0.2 # Dust to gas ratio + +[Output] +vtk 1.0 +# dmp 2.0 +# analysis 0.01 diff --git a/test/Dust/StreamingInstability/setup.cpp b/test/Dust/StreamingInstability/setup.cpp new file mode 100644 index 00000000..410ab134 --- /dev/null +++ b/test/Dust/StreamingInstability/setup.cpp @@ -0,0 +1,136 @@ +#include "idefix.hpp" +#include "setup.hpp" + +static real omega; +static real shear; +real epsilon; +real chi; +real tauGlob; + + +#define FILENAME "timevol.dat" + +//#define STRATIFIED +void PressureGradient(Hydro *hydro, const real t, const real dt) { + auto Uc = hydro->Uc; + auto Vc = hydro->Vc; + DataBlock *data = hydro->data; + real eps = epsilon; + idefix_for("MySourceTerm",0,data->np_tot[KDIR],0,data->np_tot[JDIR],0,data->np_tot[IDIR], + KOKKOS_LAMBDA (int k, int j, int i) { + // Radial pressure gradient + Uc(MX1,k,j,i) += eps*Vc(RHO,k,j,i)*dt; + }); +} + +void BodyForce(DataBlock &data, const real t, IdefixArray4D &force) { + idfx::pushRegion("BodyForce"); + IdefixArray1D x = data.x[IDIR]; + IdefixArray1D z = data.x[KDIR]; + + // GPUS cannot capture static variables + real omegaLocal=omega; + real shearLocal =shear; + + idefix_for("BodyForce", + data.beg[KDIR] , data.end[KDIR], + data.beg[JDIR] , data.end[JDIR], + data.beg[IDIR] , data.end[IDIR], + KOKKOS_LAMBDA (int k, int j, int i) { + + force(IDIR,k,j,i) = -2.0*omegaLocal*shearLocal*x(i); + force(JDIR,k,j,i) = ZERO_F; + force(KDIR,k,j,i) = ZERO_F; + + }); + + + idfx::popRegion(); +} + +// Analyse data to produce an output +void Analysis(DataBlock & data) { + + + + if(idfx::prank == 0) { + std::ofstream f; + f.open(FILENAME,std::ios::app); + f.precision(10); + int k=18; + int j=18; + int i=18; + + real vx = data.hydro->Vc(VX1,k,j,i); + real vy = data.hydro->Vc(VX2,k,j,i); + real ux = data.dust[0]->Vc(VX1,k,j,i); + real uy = data.dust[0]->Vc(VX2,k,j,i); + f << std::scientific << data.t << "\t" << vx << "\t" << vy << "\t" << ux << "\t" << uy << std::endl; + f.close(); + } + +} + +// Initialisation routine. Can be used to allocate +// Arrays or variables which are used later on +Setup::Setup(Input &input, Grid &grid, DataBlock &data, Output &output) { + // Get rotation rate along vertical axis + omega=input.Get("Hydro","rotation",0); + shear=input.Get("Hydro","shearingBox",0); + + tauGlob = input.Get("Dust","drag",1); + epsilon = input.Get("Setup","epsilon",0); + chi = input.Get("Setup","chi",0); + data.hydro->EnrollUserSourceTerm(&PressureGradient); + // Add our userstep to the timeintegrator + data.gravity->EnrollBodyForce(BodyForce); + +} + +// This routine initialize the flow +// Note that data is on the device. +// One can therefore define locally +// a datahost and sync it, if needed +void Setup::InitFlow(DataBlock &data) { + // Create a host copy + DataBlockHost d(data); + + real taus = tauGlob*omega; + real D = 1+chi; + + for(int k = 0; k < d.np_tot[KDIR] ; k++) { + for(int j = 0; j < d.np_tot[JDIR] ; j++) { + for(int i = 0; i < d.np_tot[IDIR] ; i++) { + real x=d.x[IDIR](i); + + + d.Vc(RHO,k,j,i) = 1.0; + // Equations 7a-7b of Youdin & Johansen (2007) with eta = epsilon/(2Omega Vk) + d.Vc(VX1,k,j,i) = chi * taus/((D*D+taus*taus))*epsilon/omega; + d.Vc(VX2,k,j,i) = -(1+ chi*taus*taus/(D*D+taus*taus))*epsilon/(2*D*omega); + d.Vc(VX1,k,j,i) += 1e-1*(idfx::randm()-0.5); + d.Vc(VX2,k,j,i) += shear*x; + d.Vc(VX3,k,j,i) = 0.0; + + d.dustVc[0](RHO,k,j,i) = chi; + // Equations 7c-7d of Youdin & Johansen (2007) with eta = epsilon/(2Omega Vk) + d.dustVc[0](VX1,k,j,i) = - taus/(D*D+taus*taus)*epsilon/omega; + d.dustVc[0](VX2,k,j,i) = - (1-taus*taus/(D*D+taus*taus))*epsilon/(2*D*omega); + + d.dustVc[0](VX1,k,j,i) += 0*1e-1*(idfx::randm()-0.5); + d.dustVc[0](VX2,k,j,i) += shear*x; + d.dustVc[0](VX3,k,j,i) = 0.0; + + } + } + } + + // Send it all, if needed + d.SyncToDevice(); +} + + +// Analyse data to produce an output + +void MakeAnalysis(DataBlock & data) { +} diff --git a/test/HD/FargoPlanet/idefix-rkl.ini b/test/HD/FargoPlanet/idefix-rkl.ini index d31dda61..8a4cfbbf 100644 --- a/test/HD/FargoPlanet/idefix-rkl.ini +++ b/test/HD/FargoPlanet/idefix-rkl.ini @@ -18,7 +18,8 @@ viscosity rkl userdef velocity userdef [Gravity] -potential userdef +potential central planet +Mcentral 1.0 [Boundary] X1-beg userdef @@ -35,11 +36,14 @@ h0 0.05 alpha 1.0e-4 [Planet] -qpl 1.0e-3 -Rpl 1.0 -thicknessSmoothing 0.6 +integrator analytical +planetToPrimary 1.0e-3 +initialDistance 1.0 +feelDisk false +feelPlanets false +smoothing plummer 0.03 0.0 [Output] vtk 10.0 -dmp 100.0 +dmp 10.0 log 100 diff --git a/test/HD/FargoPlanet/idefix.ini b/test/HD/FargoPlanet/idefix.ini index fea67dd2..8c886556 100644 --- a/test/HD/FargoPlanet/idefix.ini +++ b/test/HD/FargoPlanet/idefix.ini @@ -18,7 +18,8 @@ viscosity explicit userdef velocity userdef [Gravity] -potential userdef +potential central planet +Mcentral 1.0 [Boundary] X1-beg userdef @@ -35,11 +36,14 @@ h0 0.05 alpha 1.0e-4 [Planet] -qpl 1.0e-3 -Rpl 1.0 -thicknessSmoothing 0.6 +integrator analytical +planetToPrimary 1.0e-3 +initialDistance 1.0 +feelDisk false +feelPlanets false +smoothing plummer 0.03 0.0 [Output] vtk 10.0 -dmp 100.0 +dmp 10.0 log 100 diff --git a/test/HD/FargoPlanet/python/data.0001.ref.vtk b/test/HD/FargoPlanet/python/data.0001.ref.vtk deleted file mode 100644 index dab1bc40..00000000 --- a/test/HD/FargoPlanet/python/data.0001.ref.vtk +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:ce6d71bae9a60614d4cb8a8ce1c892af654115ba518896d89b5f7094e3590b4d -size 791306 diff --git a/test/HD/FargoPlanet/python/testidefix.py b/test/HD/FargoPlanet/python/testidefix.py deleted file mode 100755 index 599e814e..00000000 --- a/test/HD/FargoPlanet/python/testidefix.py +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -""" -Created on Thu Mar 5 11:29:41 2020 - -@author: glesur -""" - -import os -import sys -sys.path.append(os.getenv("IDEFIX_DIR")) -from pytools.vtk_io import readVTK -import numpy as np - -V=readVTK('../data.0001.vtk', geometry='polar') -U=readVTK('data.0001.ref.vtk', geometry='polar') - -# Compute the error on RHO -error=np.mean(np.abs(V.data['RHO']-U.data['rho']),axis=(0,1)) - -print("Error=%e"%error) -if error<5.0e-2: - print("SUCCESS!") - sys.exit(0) -else: - print("FAILURE!") - sys.exit(1) diff --git a/test/HD/FargoPlanet/setup.cpp b/test/HD/FargoPlanet/setup.cpp index 614674e9..125834a4 100644 --- a/test/HD/FargoPlanet/setup.cpp +++ b/test/HD/FargoPlanet/setup.cpp @@ -11,9 +11,6 @@ real AmMidGlob; real gammaGlob; real densityFloorGlob; real alphaGlob; -real qpGlob; -real RpGlob; -real thicknessSmoothingGlob; void MySoundSpeed(DataBlock &data, const real t, IdefixArray3D &cs) { @@ -27,7 +24,7 @@ void MySoundSpeed(DataBlock &data, const real t, IdefixArray3D &cs) { } void MyViscosity(DataBlock &data, const real t, IdefixArray3D &eta1, IdefixArray3D &eta2) { - IdefixArray4D Vc=data.hydro.Vc; + IdefixArray4D Vc=data.hydro->Vc; IdefixArray1D x1=data.x[IDIR]; real h0 = h0Glob; real alpha = alphaGlob; @@ -42,17 +39,21 @@ void MyViscosity(DataBlock &data, const real t, IdefixArray3D &eta1, Idefi } // User-defined boundaries -void UserdefBoundary(DataBlock& data, int dir, BoundarySide side, real t) { - IdefixArray4D Vc = data.hydro.Vc; - IdefixArray1D x1 = data.x[IDIR]; - IdefixArray1D x3 = data.x[KDIR]; +void UserdefBoundary(Hydro *hydro, int dir, BoundarySide side, real t) { + IdefixArray4D Vc = hydro->Vc; + auto *data = hydro->data; + IdefixArray1D x1 = data->x[IDIR]; + IdefixArray1D x3 = data->x[KDIR]; if(dir==IDIR) { int ighost,ibeg,iend; if(side == left) { - ighost = data.beg[IDIR]; + ighost = data->beg[IDIR]; ibeg = 0; - iend = data.beg[IDIR]; - idefix_for("UserDefBoundary",0,data.np_tot[KDIR],0,data.np_tot[JDIR],ibeg,iend, + iend = data->beg[IDIR]; + idefix_for("UserDefBoundary", + 0, data->np_tot[KDIR], + 0, data->np_tot[JDIR], + ibeg, iend, KOKKOS_LAMBDA (int k, int j, int i) { real R=x1(i); real z=x3(k); @@ -65,10 +66,13 @@ void UserdefBoundary(DataBlock& data, int dir, BoundarySide side, real t) { }); } else if(side==right) { - ighost = data.end[IDIR]-1; - ibeg=data.end[IDIR]; - iend=data.np_tot[IDIR]; - idefix_for("UserDefBoundary",0,data.np_tot[KDIR],0,data.np_tot[JDIR],ibeg,iend, + ighost = data->end[IDIR]-1; + ibeg=data->end[IDIR]; + iend=data->np_tot[IDIR]; + idefix_for("UserDefBoundary", + 0, data->np_tot[KDIR], + 0, data->np_tot[JDIR], + ibeg, iend, KOKKOS_LAMBDA (int k, int j, int i) { real R=x1(i); real z=x3(k); @@ -83,40 +87,7 @@ void UserdefBoundary(DataBlock& data, int dir, BoundarySide side, real t) { } } -void Potential(DataBlock& data, const real t, IdefixArray1D& x1, IdefixArray1D& x2, IdefixArray1D& x3, IdefixArray3D& phi) { - // const int subcycling = 5; - - real h0 = h0Glob; - real qp = qpGlob; - - - real Rp = RpGlob; - real phiPlanet = sqrt((1.0+qp)/(Rp*Rp*Rp))*t; - real xp = Rp * cos(phiPlanet); - real yp = Rp * sin(phiPlanet); - real zp = ZERO_F; - real thicknessSmoothing = thicknessSmoothingGlob; - real smoothing = h0*pow(Rp,0.0)*Rp*thicknessSmoothing; - smoothing*=smoothing; - // GWF do not forget indirect term due to planets (what about indirect term due gas) - - idefix_for("Potential",0,data.np_tot[KDIR], 0, data.np_tot[JDIR], 0, data.np_tot[IDIR], - KOKKOS_LAMBDA (int k, int j, int i) { - phi(k,j,i) = -1.0/sqrt(x1(i)*x1(i)+x3(k)*x3(k)); - real xc=x1(i)*cos(x2(j)); - real yc=x1(i)*sin(x2(j)); - real zc=x3(k); - real dist = ((xc-xp)*(xc-xp)+ - (yc-yp)*(yc-yp)+ - (zc-zp)*(zc-zp)); - // term due to planet - phi(k,j,i) += -qp/sqrt(dist+smoothing); - // indirect term due to planet - phi(k,j,i) += qp*(xc*xp+yc*yp+zc*zp)/(Rp*Rp*Rp); - }); - -} void FargoVelocity(DataBlock &data, IdefixArray2D &Vphi) { IdefixArray1D x1 = data.x[IDIR]; @@ -134,20 +105,14 @@ void FargoVelocity(DataBlock &data, IdefixArray2D &Vphi) { Setup::Setup(Input &input, Grid &grid, DataBlock &data, Output &output)// : m_planet(0)//, Planet &planet) { // Set the function for userdefboundary - data.hydro.EnrollUserDefBoundary(&UserdefBoundary); - data.gravity.EnrollPotential(&Potential); - data.hydro.EnrollIsoSoundSpeed(&MySoundSpeed); - data.hydro.viscosity.EnrollViscousDiffusivity(&MyViscosity); + data.hydro->EnrollUserDefBoundary(&UserdefBoundary); + data.hydro->EnrollIsoSoundSpeed(&MySoundSpeed); + data.hydro->viscosity->EnrollViscousDiffusivity(&MyViscosity); if(data.haveFargo) - data.fargo.EnrollVelocity(&FargoVelocity); + data.fargo->EnrollVelocity(&FargoVelocity); sigma0Glob = input.Get("Setup","sigma0",0); sigmaSlopeGlob = input.Get("Setup","sigmaSlope",0); h0Glob = input.Get("Setup","h0",0); - alphaGlob = input.Get("Setup","alpha",0); - qpGlob = input.Get("Planet","qpl",0); - RpGlob = input.Get("Planet","Rpl",0); - thicknessSmoothingGlob = input.Get("Planet","thicknessSmoothing",0); - idfx::cout << "alpha= " << alphaGlob << std::endl; } diff --git a/test/HD/FargoPlanet/testme.py b/test/HD/FargoPlanet/testme.py new file mode 100755 index 00000000..5890d010 --- /dev/null +++ b/test/HD/FargoPlanet/testme.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 + +""" + +@author: glesur +""" +import os +import sys +sys.path.append(os.getenv("IDEFIX_DIR")) + +import pytools.idfx_test as tst +tolerance=1e-13 +def testMe(test): + test.configure() + test.compile() + inifiles=["idefix.ini","idefix-rkl.ini"] + mytol=tolerance + for ini in inifiles: + test.run(inputFile=ini) + if test.init and not test.mpi: + test.makeReference(filename="dump.0001.dmp") + if ini == "idefix-rkl.ini": + mytol = 1e-12 + else: + mytol = tolerance + test.standardTest() + test.nonRegressionTest(filename="dump.0001.dmp",tolerance=mytol) + + +test=tst.idfxTest() +if not test.dec: + test.dec=['2','2'] + +if not test.all: + if(test.check): + test.checkOnly(filename="dump.0001.dmp",tolerance=tolerance) + else: + testMe(test) +else: + test.noplot = True + test.single=False + test.reconstruction=2 + test.mpi=False + testMe(test) + + test.mpi=True + testMe(test) diff --git a/test/HD/MachReflection/idefix-hll.ini b/test/HD/MachReflection/idefix-hll.ini index b774a0a2..8572dded 100644 --- a/test/HD/MachReflection/idefix-hll.ini +++ b/test/HD/MachReflection/idefix-hll.ini @@ -5,7 +5,7 @@ X3-grid 1 0.0 1 u 1.0 [TimeIntegrator] CFL 0.8 -tstop 0.200001 +tstop 0.2 first_dt 1.e-5 nstages 2 @@ -23,3 +23,4 @@ X3-end outflow [Output] vtk 0.2 +dmp 0.2 diff --git a/test/HD/MachReflection/idefix-hllc.ini b/test/HD/MachReflection/idefix-hllc.ini index bcd341b9..9dee90ee 100644 --- a/test/HD/MachReflection/idefix-hllc.ini +++ b/test/HD/MachReflection/idefix-hllc.ini @@ -5,7 +5,7 @@ X3-grid 1 0.0 1 u 1.0 [TimeIntegrator] CFL 0.8 -tstop 0.200001 +tstop 0.2 first_dt 1.e-5 nstages 2 @@ -23,3 +23,4 @@ X3-end outflow [Output] vtk 0.2 +dmp 0.2 diff --git a/test/HD/MachReflection/idefix-tvdlf.ini b/test/HD/MachReflection/idefix-tvdlf.ini index a3eb8cd4..dfee317a 100644 --- a/test/HD/MachReflection/idefix-tvdlf.ini +++ b/test/HD/MachReflection/idefix-tvdlf.ini @@ -5,7 +5,7 @@ X3-grid 1 0.0 1 u 1.0 [TimeIntegrator] CFL 0.8 -tstop 0.200001 +tstop 0.2 first_dt 1.e-5 nstages 2 @@ -23,3 +23,4 @@ X3-end outflow [Output] vtk 0.2 +dmp 0.2 diff --git a/test/HD/MachReflection/idefix.ini b/test/HD/MachReflection/idefix.ini index c07e3824..caccba34 100644 --- a/test/HD/MachReflection/idefix.ini +++ b/test/HD/MachReflection/idefix.ini @@ -5,7 +5,7 @@ X3-grid 1 0.0 1 u 1.0 [TimeIntegrator] CFL 0.8 -tstop 0.200001 +tstop 0.2 first_dt 1.e-5 nstages 2 @@ -23,3 +23,4 @@ X3-end outflow [Output] vtk 0.2 +dmp 0.2 diff --git a/test/HD/MachReflection/python/data.0001.ref.vtk b/test/HD/MachReflection/python/data.0001.ref.vtk deleted file mode 100644 index 9fbfdd3e..00000000 --- a/test/HD/MachReflection/python/data.0001.ref.vtk +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:4678b789caed58265d4a6c10a429cb17cdbd273512fc5acd9eb753fd47775dc8 -size 1154813 diff --git a/test/HD/MachReflection/python/testidefix.py b/test/HD/MachReflection/python/testidefix.py deleted file mode 100755 index ea4e7b51..00000000 --- a/test/HD/MachReflection/python/testidefix.py +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -""" -Created on Thu Mar 5 11:29:41 2020 - -@author: glesur -""" - -import os -import sys -sys.path.append(os.getenv("IDEFIX_DIR")) -from pytools.vtk_io import readVTK -import numpy as np - - -V=readVTK('../data.0001.vtk', geometry='cartesian') -U=readVTK('data.0001.ref.vtk', geometry='cartesian') - -# Compute the error on PRS -error=np.mean(np.abs(np.log(V.data['RHO'])-np.log(U.data['RHO'])),axis=(0,1)) - -print("Error=%e"%error) -if error<6e-3: - print("SUCCESS!") - sys.exit(0) -else: - print("FAILURE!") - sys.exit(1) diff --git a/test/HD/MachReflection/setup.cpp b/test/HD/MachReflection/setup.cpp index 1b82fa3f..7d7f2c55 100644 --- a/test/HD/MachReflection/setup.cpp +++ b/test/HD/MachReflection/setup.cpp @@ -2,13 +2,13 @@ #include "setup.hpp" // User-defined boundaries -void UserdefBoundary(DataBlock & data, int dir, BoundarySide side, const real t) { +void UserdefBoundary(Hydro* hydro, int dir, BoundarySide side, const real t) { const real alpha = 60./180.*M_PI; if( (dir==IDIR) && (side == left)) { - IdefixArray4D Vc = data.hydro.Vc; - int ighost = data.nghost[IDIR]; - idefix_for("UserDefBoundaryX1Beg",0,data.np_tot[KDIR],0,data.np_tot[JDIR],0,ighost, + IdefixArray4D Vc = hydro->Vc; + int ighost = hydro->data->nghost[IDIR]; + hydro->boundary->BoundaryFor("UserDefBoundaryX1Beg", dir, side, KOKKOS_LAMBDA (int k, int j, int i) { Vc(RHO,k,j,i) = 8.0; @@ -19,14 +19,12 @@ void UserdefBoundary(DataBlock & data, int dir, BoundarySide side, const real t) } if(dir==JDIR) { - IdefixArray4D Vc = data.hydro.Vc; - IdefixArray1D x1 = data.x[IDIR]; - IdefixArray1D x2 = data.x[JDIR]; - int jbeg,jend; + IdefixArray4D Vc = hydro->Vc; + IdefixArray1D x1 = hydro->data->x[IDIR]; + IdefixArray1D x2 = hydro->data->x[JDIR]; if(side == left) { - jbeg = 0; - jend = data.beg[JDIR]; - idefix_for("UserDefBoundaryX2Beg",0,data.np_tot[KDIR],jbeg,jend,0,data.np_tot[IDIR], + const int jend = hydro->data->beg[JDIR]; + hydro->boundary->BoundaryFor("UserDefBoundaryX2Beg", dir, side, KOKKOS_LAMBDA (int k, int j, int i) { if(x1(i) < 1.0/6.0) { Vc(RHO,k,j,i) = 8.0; @@ -43,11 +41,9 @@ void UserdefBoundary(DataBlock & data, int dir, BoundarySide side, const real t) }); //return; } else if(side==right) { - jbeg=data.end[JDIR]; - jend=data.np_tot[JDIR]; real xs = 10.0*t/sin(alpha) + 1.0/6.0 + 1.0/tan(alpha); - idefix_for("UserDefBoundaryX2End",0,data.np_tot[KDIR],jbeg,jend,0,data.np_tot[IDIR], + hydro->boundary->BoundaryFor("UserDefBoundaryX2End", dir, side, KOKKOS_LAMBDA (int k, int j, int i) { if(x1(i) < xs) { Vc(RHO,k,j,i) = 8.0; @@ -73,7 +69,7 @@ void UserdefBoundary(DataBlock & data, int dir, BoundarySide side, const real t) // Initialisation routine. Can be used to allocate // Arrays or variables which are used later on Setup::Setup(Input &input, Grid &grid, DataBlock &data, Output &output) { - data.hydro.EnrollUserDefBoundary(&UserdefBoundary); + data.hydro->EnrollUserDefBoundary(&UserdefBoundary); } // This routine initialize the flow diff --git a/test/HD/MachReflection/testme.py b/test/HD/MachReflection/testme.py new file mode 100755 index 00000000..ce87f392 --- /dev/null +++ b/test/HD/MachReflection/testme.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 + +""" + +@author: glesur +""" +import os +import sys +sys.path.append(os.getenv("IDEFIX_DIR")) + +import pytools.idfx_test as tst + +def testMe(test): + test.configure() + test.compile() + inifiles=["idefix.ini","idefix-hll.ini","idefix-hllc.ini","idefix-tvdlf.ini"] + + for ini in inifiles: + test.run(inputFile=ini) + if test.init and not test.mpi: + test.makeReference(filename="dump.0001.dmp") + test.standardTest() + test.nonRegressionTest(filename="dump.0001.dmp") + + +test=tst.idfxTest() +if not test.dec: + test.dec=['2','2'] + +if not test.all: + if(test.check): + test.checkOnly(filename="dump.0001.dmp") + else: + testMe(test) +else: + test.noplot = True + test.single=False + test.reconstruction=2 + test.mpi=False + testMe(test) + + test.mpi=True + testMe(test) diff --git a/test/HD/RWI-cavity/setup.cpp b/test/HD/RWI-cavity/setup.cpp index d97c2693..726690e6 100644 --- a/test/HD/RWI-cavity/setup.cpp +++ b/test/HD/RWI-cavity/setup.cpp @@ -45,12 +45,12 @@ void FargoVelocity(DataBlock &data, IdefixArray2D &Vphi) { // Arrays or variables which are used later on Setup::Setup(Input &input, Grid &grid, DataBlock &data, Output &output) { // Set the function for userdefboundary - data.hydro.EnrollIsoSoundSpeed(&LISOTHSoundSpeed); + data.hydro->EnrollIsoSoundSpeed(&LISOTHSoundSpeed); aspect_ratio_glob = input.Get("Setup","aspect_ratio",0); jump_radius_glob = input.Get("Setup", "jump_radius",0); jump_width_glob = input.Get("Setup", "jump_width",0); if(data.haveFargo) - data.fargo.EnrollVelocity(&FargoVelocity); + data.fargo->EnrollVelocity(&FargoVelocity); } // This routine initialize the flow diff --git a/test/HD/SedovBlastWave/idefix-spherical.ini b/test/HD/SedovBlastWave/idefix-spherical.ini index a4c565f6..7b114f56 100644 --- a/test/HD/SedovBlastWave/idefix-spherical.ini +++ b/test/HD/SedovBlastWave/idefix-spherical.ini @@ -26,3 +26,4 @@ X3-end periodic [Output] vtk 0.1 +dmp 0.1 diff --git a/test/HD/SedovBlastWave/idefix.ini b/test/HD/SedovBlastWave/idefix.ini index 26f463d2..51122521 100644 --- a/test/HD/SedovBlastWave/idefix.ini +++ b/test/HD/SedovBlastWave/idefix.ini @@ -27,3 +27,4 @@ X3-end periodic [Output] vtk 0.1 xdmf 0.1 +dmp 0.1 diff --git a/test/HD/SedovBlastWave/python/1Dsolution/1Dsolution.vtk b/test/HD/SedovBlastWave/python/1Dsolution/1Dsolution.vtk new file mode 100644 index 00000000..06ec6f0e Binary files /dev/null and b/test/HD/SedovBlastWave/python/1Dsolution/1Dsolution.vtk differ diff --git a/test/HD/SedovBlastWave/python/1Dsolution/data.0001.vtk b/test/HD/SedovBlastWave/python/1Dsolution/data.0001.vtk deleted file mode 100644 index c591c53c..00000000 --- a/test/HD/SedovBlastWave/python/1Dsolution/data.0001.vtk +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:981bb206e7b96b68bf1e73bc5a27941e6febebbe2faf4d27fb2c6a4397059e69 -size 98677 diff --git a/test/HD/SedovBlastWave/python/testidefix.py b/test/HD/SedovBlastWave/python/testidefix.py index cd306b48..a78bc88e 100755 --- a/test/HD/SedovBlastWave/python/testidefix.py +++ b/test/HD/SedovBlastWave/python/testidefix.py @@ -25,7 +25,7 @@ args, unknown=parser.parse_known_args() V=readVTK('../data.0001.vtk') -R=readVTK('1Dsolution/data.0001.vtk') +R=readVTK('1Dsolution/1Dsolution.vtk') diff --git a/test/HD/SedovBlastWave/setup.cpp b/test/HD/SedovBlastWave/setup.cpp index e366b559..7b17b6ba 100644 --- a/test/HD/SedovBlastWave/setup.cpp +++ b/test/HD/SedovBlastWave/setup.cpp @@ -54,7 +54,7 @@ void Setup::InitFlow(DataBlock &data) { MPI_Allreduce(MPI_IN_PLACE, &V, 1, realMPI, MPI_SUM, MPI_COMM_WORLD); #endif - real gamma = data.hydro.GetGamma(); + real gamma = data.hydro->eos->GetGamma(); for(int k = 0; k < d.np_tot[KDIR] ; k++) { for(int j = 0; j < d.np_tot[JDIR] ; j++) { for(int i = 0; i < d.np_tot[IDIR] ; i++) { diff --git a/test/HD/SedovBlastWave/testme.py b/test/HD/SedovBlastWave/testme.py new file mode 100755 index 00000000..95fdd030 --- /dev/null +++ b/test/HD/SedovBlastWave/testme.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python3 + +""" + +@author: glesur +""" +import os +import sys +sys.path.append(os.getenv("IDEFIX_DIR")) + +import pytools.idfx_test as tst + +name="dump.0001.dmp" + +test=tst.idfxTest() + +if not test.dec: + test.dec=['2','2','2'] + +if test.check: + test.standardTest() +else: + test.vectPot=False + test.single=False + test.reconstruction=2 + test.mpi=True + # Cartesian validation + test.configure(definitionFile="definitions.hpp") + test.compile() + test.run(inputFile="idefix.ini") + test.standardTest() + + #Spherical validation + test.configure(definitionFile="definitions-spherical.hpp") + test.compile() + test.run(inputFile="idefix-spherical.ini") + test.standardTest() diff --git a/test/HD/ShearingBox/.gitignore b/test/HD/ShearingBox/.gitignore new file mode 100644 index 00000000..773a6df9 --- /dev/null +++ b/test/HD/ShearingBox/.gitignore @@ -0,0 +1 @@ +*.dat diff --git a/test/HD/ShearingBox/analysis.cpp b/test/HD/ShearingBox/analysis.cpp index bdc9d14b..dacf2a70 100644 --- a/test/HD/ShearingBox/analysis.cpp +++ b/test/HD/ShearingBox/analysis.cpp @@ -1,5 +1,6 @@ #include "analysis.hpp" #include "idefix.hpp" +#include "fluid.hpp" #include #include #include @@ -8,7 +9,7 @@ Analysis::Analysis(Input &input, Grid &grid, DataBlock &data, Output &output, st this->d = new DataBlockHost(data); this->grid = &grid; this->filename = filename; - this->shear = data.hydro.sbS; + this->shear = data.hydro->sbS; this->precision = 10; } diff --git a/test/HD/ShearingBox/idefix-fargo.ini b/test/HD/ShearingBox/idefix-fargo.ini index e0377d89..9d0dd373 100644 --- a/test/HD/ShearingBox/idefix-fargo.ini +++ b/test/HD/ShearingBox/idefix-fargo.ini @@ -30,4 +30,5 @@ X3-end periodic [Output] # vtk 0.1 +dmp 2.0 analysis 0.1 diff --git a/test/HD/ShearingBox/idefix.ini b/test/HD/ShearingBox/idefix.ini index b29f8394..511d7e15 100644 --- a/test/HD/ShearingBox/idefix.ini +++ b/test/HD/ShearingBox/idefix.ini @@ -27,4 +27,5 @@ X3-end periodic [Output] # vtk 0.1 +dmp 2.0 analysis 0.1 diff --git a/test/HD/ShearingBox/python/testidefix.py b/test/HD/ShearingBox/python/testidefix.py index bb0a9baa..1a0c2509 100755 --- a/test/HD/ShearingBox/python/testidefix.py +++ b/test/HD/ShearingBox/python/testidefix.py @@ -46,7 +46,7 @@ def rhs(t, y, Omega, q, k0x, k0y, k0z): args, unknown=parser.parse_known_args() -#initial condition: vr=1, rest is 0, mode initial is nx=0, ny=1, nz=5) +#initial condition: vr=1, rest is 0, mode initial is nx=0, ny=1, nz=4) y=solve_ivp(rhs, [0,30], [1, 0, 0], args=(1, 1.5, 0, 2.0*np.pi, 8*np.pi), dense_output=True) @@ -112,3 +112,4 @@ def rhs(t, y, Omega, q, k0x, k0y, k0z): sys.exit(0) else: print("FAILED") + sys.exit(1) diff --git a/test/HD/ShearingBox/setup.cpp b/test/HD/ShearingBox/setup.cpp index 0a1c9db9..e82701d0 100644 --- a/test/HD/ShearingBox/setup.cpp +++ b/test/HD/ShearingBox/setup.cpp @@ -45,14 +45,14 @@ void AnalysisFunction(DataBlock &data) { // Initialisation routine. Can be used to allocate // Arrays or variables which are used later on Setup::Setup(Input &input, Grid &grid, DataBlock &data, Output &output) { - gammaIdeal=data.hydro.GetGamma(); + gammaIdeal=data.hydro->eos->GetGamma(); // Get rotation rate along vertical axis omega=input.Get("Hydro","rotation",0); shear=input.Get("Hydro","shearingBox",0); // Add our userstep to the timeintegrator - data.gravity.EnrollBodyForce(BodyForce); + data.gravity->EnrollBodyForce(BodyForce); analysis = new Analysis(input, grid, data, output,std::string("timevol.dat")); output.EnrollAnalysis(&AnalysisFunction); diff --git a/test/HD/ShearingBox/testme.py b/test/HD/ShearingBox/testme.py new file mode 100755 index 00000000..3739910a --- /dev/null +++ b/test/HD/ShearingBox/testme.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python3 + +""" + +@author: glesur +""" +import os +import sys +sys.path.append(os.getenv("IDEFIX_DIR")) + +import pytools.idfx_test as tst +tolerance=1e-15 +def testMe(test): + test.configure() + test.compile() + inifiles=["idefix.ini","idefix-fargo.ini"] + + for ini in inifiles: + test.run(inputFile=ini) + if test.init and not test.mpi: + test.makeReference(filename="dump.0001.dmp") + test.standardTest() + test.nonRegressionTest(filename="dump.0001.dmp",tolerance=tolerance) + + +test=tst.idfxTest() +if not test.dec: + test.dec=['2','1','2'] + +if not test.all: + if(test.check): + test.checkOnly(filename="dump.0001.dmp",tolerance=tolerance) + else: + testMe(test) +else: + test.noplot = True + test.single=False + test.reconstruction=2 + test.mpi=False + testMe(test) diff --git a/test/HD/VSI/setup.cpp b/test/HD/VSI/setup.cpp index b1458660..19800e67 100644 --- a/test/HD/VSI/setup.cpp +++ b/test/HD/VSI/setup.cpp @@ -23,16 +23,20 @@ void MySoundSpeed(DataBlock &data, const real t, IdefixArray3D &cs) { // User-defined boundaries -void UserdefBoundary(DataBlock& data, int dir, BoundarySide side, real t) { +void UserdefBoundary(Hydro *hydro, int dir, BoundarySide side, real t) { + auto *data = hydro->data; if( (dir==IDIR) && (side == left)) { - IdefixArray4D Vc = data.hydro.Vc; - IdefixArray1D x1 = data.x[IDIR]; - IdefixArray1D x2 = data.x[JDIR]; + IdefixArray4D Vc = hydro->Vc; + IdefixArray1D x1 = data->x[IDIR]; + IdefixArray1D x2 = data->x[JDIR]; - int ighost = data.nghost[IDIR]; + int ighost = data->nghost[IDIR]; real Omega=1.0; - idefix_for("UserDefBoundary",0,data.np_tot[KDIR],0,data.np_tot[JDIR],0,ighost, + idefix_for("UserDefBoundary", + 0, data->np_tot[KDIR], + 0, data->np_tot[JDIR], + 0, ighost, KOKKOS_LAMBDA (int k, int j, int i) { real R=x1(i)*sin(x2(j)); real z=x1(i)*cos(x2(j)); @@ -46,23 +50,26 @@ void UserdefBoundary(DataBlock& data, int dir, BoundarySide side, real t) { } if( dir==JDIR) { - IdefixArray4D Vc = data.hydro.Vc; + IdefixArray4D Vc = hydro->Vc; int jghost; int jbeg,jend; if(side == left) { - jghost = data.beg[JDIR]; + jghost = data->beg[JDIR]; jbeg = 0; - jend = data.beg[JDIR]; + jend = data->beg[JDIR]; //return; } else if(side==right) { - jghost = data.end[JDIR]-1; - jbeg=data.end[JDIR]; - jend=data.np_tot[JDIR]; + jghost = data->end[JDIR]-1; + jbeg=data->end[JDIR]; + jend=data->np_tot[JDIR]; } - idefix_for("UserDefBoundary",0,data.np_tot[KDIR],jbeg,jend,0,data.np_tot[IDIR], + idefix_for("UserDefBoundary", + 0, data->np_tot[KDIR], + jbeg, jend, + 0, data->np_tot[IDIR], KOKKOS_LAMBDA (int k, int j, int i) { Vc(RHO,k,j,i) = Vc(RHO,k,jghost,i); Vc(VX1,k,j,i) = ZERO_F; @@ -78,8 +85,8 @@ void UserdefBoundary(DataBlock& data, int dir, BoundarySide side, real t) { // Arrays or variables which are used later on Setup::Setup(Input &input, Grid &grid, DataBlock &data, Output &output) { // Set the function for userdefboundary - data.hydro.EnrollUserDefBoundary(&UserdefBoundary); - data.hydro.EnrollIsoSoundSpeed(&MySoundSpeed); + data.hydro->EnrollUserDefBoundary(&UserdefBoundary); + data.hydro->EnrollIsoSoundSpeed(&MySoundSpeed); epsilonGlob = input.Get("Setup","epsilon",0); } diff --git a/test/HD/ViscousDisk/idefix_rkl.ini b/test/HD/ViscousDisk/idefix-rkl.ini similarity index 97% rename from test/HD/ViscousDisk/idefix_rkl.ini rename to test/HD/ViscousDisk/idefix-rkl.ini index 361304d4..63888492 100644 --- a/test/HD/ViscousDisk/idefix_rkl.ini +++ b/test/HD/ViscousDisk/idefix-rkl.ini @@ -32,5 +32,5 @@ alpha 2.0e-3 [Output] vtk 400.0 -dmp -10.0 +dmp 400.0 log 1000 diff --git a/test/HD/ViscousDisk/idefix.ini b/test/HD/ViscousDisk/idefix.ini index bfb003d6..673d4be7 100644 --- a/test/HD/ViscousDisk/idefix.ini +++ b/test/HD/ViscousDisk/idefix.ini @@ -32,5 +32,5 @@ alpha 2.0e-3 [Output] vtk 400.0 -dmp -10.0 +dmp 400.0 log 1000 diff --git a/test/HD/ViscousDisk/idefix_fargo.ini b/test/HD/ViscousDisk/idefix_fargo.ini deleted file mode 100644 index 6fa28b63..00000000 --- a/test/HD/ViscousDisk/idefix_fargo.ini +++ /dev/null @@ -1,39 +0,0 @@ -[Grid] -X1-grid 1 1.0 64 u 3.0 -X2-grid 1 1.2707963267948965 64 u 1.8707963267948966 -X3-grid 1 0.0 1 u 6.283185307179586 - -[TimeIntegrator] -CFL 0.5 -tstop 400.0 -first_dt 1.e-3 -nstages 2 - -[Hydro] -solver hllc -csiso userdef -viscosity explicit userdef - -[Fargo] -velocity userdef - -[Gravity] -potential central -Mcentral 1.0 - -[Boundary] -X1-beg userdef -X1-end userdef -X2-beg userdef -X2-end userdef -X3-beg periodic -X3-end periodic - -[Setup] -epsilon 0.1 -alpha 2.0e-3 - -[Output] -vtk 400.0 -dmp -10.0 -log 1000 diff --git a/test/HD/ViscousDisk/idefix_fargo_rkl.ini b/test/HD/ViscousDisk/idefix_fargo_rkl.ini deleted file mode 100644 index ce805121..00000000 --- a/test/HD/ViscousDisk/idefix_fargo_rkl.ini +++ /dev/null @@ -1,39 +0,0 @@ -[Grid] -X1-grid 1 1.0 64 u 3.0 -X2-grid 1 1.2707963267948965 64 u 1.8707963267948966 -X3-grid 1 0.0 1 u 6.283185307179586 - -[TimeIntegrator] -CFL 0.5 -tstop 400.0 -first_dt 1.e-3 -nstages 2 - -[Hydro] -solver hllc -csiso userdef -viscosity rkl userdef - -[Fargo] -velocity userdef - -[Gravity] -potential central -Mcentral 1.0 - -[Boundary] -X1-beg userdef -X1-end userdef -X2-beg userdef -X2-end userdef -X3-beg periodic -X3-end periodic - -[Setup] -epsilon 0.1 -alpha 2.0e-3 - -[Output] -vtk 400.0 -dmp -10.0 -log 1000 diff --git a/test/HD/ViscousDisk/setup.cpp b/test/HD/ViscousDisk/setup.cpp index 8762abb2..826b0dc9 100644 --- a/test/HD/ViscousDisk/setup.cpp +++ b/test/HD/ViscousDisk/setup.cpp @@ -1,6 +1,5 @@ #include "idefix.hpp" #include "setup.hpp" -#include "boundaryloop.hpp" real epsilonGlob; real betaGlob; @@ -23,7 +22,7 @@ void MySoundSpeed(DataBlock &data, const real t, IdefixArray3D &cs) { } void MyViscosity(DataBlock &data, const real t, IdefixArray3D &eta1, IdefixArray3D &eta2) { - IdefixArray4D Vc=data.hydro.Vc; + IdefixArray4D Vc=data.hydro->Vc; IdefixArray1D r=data.x[IDIR]; IdefixArray1D th=data.x[JDIR]; real epsilon = epsilonGlob; @@ -48,23 +47,24 @@ void FargoVelocity(DataBlock &data, IdefixArray2D &Vphi) { } // User-defined boundaries -void UserdefBoundary(DataBlock& data, int dir, BoundarySide side, real t) { - IdefixArray4D Vc = data.hydro.Vc; - IdefixArray1D x1 = data.x[IDIR]; - IdefixArray1D x2 = data.x[JDIR]; +void UserdefBoundary(Hydro *hydro, int dir, BoundarySide side, real t) { + auto *data = hydro->data; + IdefixArray4D Vc = hydro->Vc; + IdefixArray1D x1 = data->x[IDIR]; + IdefixArray1D x2 = data->x[JDIR]; real epsilon=epsilonGlob; if(dir==IDIR) { int ighost; if(side == left) { - ighost = data.beg[IDIR]; + ighost = data->beg[IDIR]; } else if(side==right) { - ighost = data.end[IDIR]-1; + ighost = data->end[IDIR]-1; } - data.hydro.boundary.BoundaryFor("UserDefBoundary", dir, side, + hydro->boundary->BoundaryFor("UserDefBoundary", dir, side, KOKKOS_LAMBDA (int k, int j, int i) { real R=x1(i)*sin(x2(j)); real z=x1(i)*cos(x2(j)); @@ -81,14 +81,14 @@ void UserdefBoundary(DataBlock& data, int dir, BoundarySide side, real t) { if( dir==JDIR) { int jghost; if(side == left) { - jghost = data.beg[JDIR]; + jghost = data->beg[JDIR]; } else if(side==right) { - jghost = data.end[JDIR]-1; + jghost = data->end[JDIR]-1; } - data.hydro.boundary.BoundaryFor("UserDefBoundary", dir, side, + hydro->boundary->BoundaryFor("UserDefBoundary", dir, side, KOKKOS_LAMBDA (int k, int j, int i) { real R=x1(i)*sin(x2(j)); real z=x1(i)*cos(x2(j)); @@ -112,11 +112,11 @@ void UserdefBoundary(DataBlock& data, int dir, BoundarySide side, real t) { // Arrays or variables which are used later on Setup::Setup(Input &input, Grid &grid, DataBlock &data, Output &output) { // Set the function for userdefboundary - data.hydro.EnrollUserDefBoundary(&UserdefBoundary); - data.hydro.EnrollIsoSoundSpeed(&MySoundSpeed); - data.hydro.viscosity.EnrollViscousDiffusivity(&MyViscosity); + data.hydro->EnrollUserDefBoundary(&UserdefBoundary); + data.hydro->EnrollIsoSoundSpeed(&MySoundSpeed); + data.hydro->viscosity->EnrollViscousDiffusivity(&MyViscosity); if(data.haveFargo) - data.fargo.EnrollVelocity(&FargoVelocity); + data.fargo->EnrollVelocity(&FargoVelocity); epsilonGlob = input.Get("Setup","epsilon",0); alphaGlob = input.Get("Setup","alpha",0); idfx::cout << "alpha= " << alphaGlob << std::endl; diff --git a/test/HD/ViscousDisk/testme.py b/test/HD/ViscousDisk/testme.py new file mode 100755 index 00000000..1e8ad742 --- /dev/null +++ b/test/HD/ViscousDisk/testme.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 + +""" + +@author: glesur +""" +import os +import sys +sys.path.append(os.getenv("IDEFIX_DIR")) + +import pytools.idfx_test as tst +tolerance=3e-15 +def testMe(test): + test.configure() + test.compile() + inifiles=["idefix.ini","idefix-rkl.ini"] + for ini in inifiles: + mytol=tolerance + + test.run(inputFile=ini) + if test.init and not test.mpi: + test.makeReference(filename="dump.0001.dmp") + test.standardTest() + test.nonRegressionTest(filename="dump.0001.dmp",tolerance=mytol) + + +test=tst.idfxTest() +if not test.dec: + test.dec=['2','2'] + +if not test.all: + if(test.check): + test.checkOnly(filename="dump.0001.dmp",tolerance=tolerance) + else: + testMe(test) +else: + test.noplot = True + test.single=False + test.reconstruction=2 + test.mpi=False + testMe(test) diff --git a/test/HD/ViscousFlowPastCylinder/idefix-rkl.ini b/test/HD/ViscousFlowPastCylinder/idefix-rkl.ini index 13b86353..8b8b33e3 100644 --- a/test/HD/ViscousFlowPastCylinder/idefix-rkl.ini +++ b/test/HD/ViscousFlowPastCylinder/idefix-rkl.ini @@ -26,4 +26,5 @@ X3-end outflow [Output] vtk 1.0 +dmp 1.0 log 100 diff --git a/test/HD/ViscousFlowPastCylinder/idefix.ini b/test/HD/ViscousFlowPastCylinder/idefix.ini index e4d6cc2c..33793490 100644 --- a/test/HD/ViscousFlowPastCylinder/idefix.ini +++ b/test/HD/ViscousFlowPastCylinder/idefix.ini @@ -26,4 +26,5 @@ X3-end outflow [Output] vtk 1.0 +dmp 1.0 log 100 diff --git a/test/HD/ViscousFlowPastCylinder/python/data.0001.ref.vtk b/test/HD/ViscousFlowPastCylinder/python/data.0001.ref.vtk deleted file mode 100644 index a3d5da1f..00000000 --- a/test/HD/ViscousFlowPastCylinder/python/data.0001.ref.vtk +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:669da8a7b3af852a3f0d96f694284149990ba394349dc3f2e18eee9203c598fc -size 264789 diff --git a/test/HD/ViscousFlowPastCylinder/python/testidefix.py b/test/HD/ViscousFlowPastCylinder/python/testidefix.py deleted file mode 100755 index b0420f2e..00000000 --- a/test/HD/ViscousFlowPastCylinder/python/testidefix.py +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -""" -Created on Thu Mar 5 11:29:41 2020 - -@author: glesur -""" - -import os -import sys -sys.path.append(os.getenv("IDEFIX_DIR")) -from pytools.vtk_io import readVTK -import numpy as np - -V=readVTK('../data.0001.vtk', geometry='polar') -U=readVTK('data.0001.ref.vtk', geometry='polar') - -# Compute the error on PRS -error=np.mean(np.abs(V.data['VX2']-U.data['vx2']),axis=(0,1)) - -print("Error=%e"%error) -if error<1.5e-3: - print("SUCCESS!") - sys.exit(0) -else: - print("FAILURE!") - sys.exit(1) diff --git a/test/HD/ViscousFlowPastCylinder/setup.cpp b/test/HD/ViscousFlowPastCylinder/setup.cpp index aa5bc05c..1282ac0c 100644 --- a/test/HD/ViscousFlowPastCylinder/setup.cpp +++ b/test/HD/ViscousFlowPastCylinder/setup.cpp @@ -2,13 +2,17 @@ #include "setup.hpp" // User-defined boundaries -void UserdefBoundary(DataBlock & data, int dir, BoundarySide side, const real t) { +void UserdefBoundary(Hydro *hydro, int dir, BoundarySide side, const real t) { + auto *data = hydro->data; if(dir==IDIR) { - IdefixArray4D Vc = data.hydro.Vc; - IdefixArray1D th = data.x[JDIR]; + IdefixArray4D Vc = hydro->Vc; + IdefixArray1D th = data->x[JDIR]; if(side==right) { - int ighost = data.end[IDIR]-1; - idefix_for("UserDefBoundaryX1End",0,data.np_tot[KDIR],0,data.np_tot[JDIR],data.end[IDIR],data.np_tot[IDIR], + int ighost = data->end[IDIR]-1; + idefix_for("UserDefBoundaryX1End", + 0, data->np_tot[KDIR], + 0, data->np_tot[JDIR], + data->end[IDIR], data->np_tot[IDIR], KOKKOS_LAMBDA (int k, int j, int i) { if((th(j)>M_PI/2.0) && th(j)<3.0*M_PI/2) { // Incoming flow @@ -27,8 +31,11 @@ void UserdefBoundary(DataBlock & data, int dir, BoundarySide side, const real t) } }); } else if(side==left) { - int ighost = data.beg[IDIR]; - idefix_for("UserDefBoundaryX1Beg",0,data.np_tot[KDIR],0,data.np_tot[JDIR],0,ighost, + int ighost = data->beg[IDIR]; + idefix_for("UserDefBoundaryX1Beg", + 0, data->np_tot[KDIR], + 0, data->np_tot[JDIR], + 0, ighost, KOKKOS_LAMBDA (int k, int j, int i) { // Central cylinder Vc(RHO,k,j,i) = ONE_F; @@ -45,7 +52,7 @@ void UserdefBoundary(DataBlock & data, int dir, BoundarySide side, const real t) // Initialisation routine. Can be used to allocate // Arrays or variables which are used later on Setup::Setup(Input &input, Grid &grid, DataBlock &data, Output &output) { - data.hydro.EnrollUserDefBoundary(&UserdefBoundary); + data.hydro->EnrollUserDefBoundary(&UserdefBoundary); } @@ -64,8 +71,8 @@ void Setup::InitFlow(DataBlock &data) { // Flow past a cylinder d.Vc(RHO,k,j,i) = ONE_F; - d.Vc(VX1,k,j,i) = COS(d.x[JDIR](j))+0.02*(idfx::randm()-0.5); - d.Vc(VX2,k,j,i) = -SIN(d.x[JDIR](j))+0.02*(idfx::randm()-0.5); + d.Vc(VX1,k,j,i) = COS(d.x[JDIR](j)); + d.Vc(VX2,k,j,i) = -SIN(d.x[JDIR](j)); #if HAVE_ENERGY d.Vc(PRS,k,j,i) = ONE_F; diff --git a/test/HD/ViscousFlowPastCylinder/testme.py b/test/HD/ViscousFlowPastCylinder/testme.py new file mode 100755 index 00000000..c831e809 --- /dev/null +++ b/test/HD/ViscousFlowPastCylinder/testme.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 + +""" + +@author: glesur +""" +import os +import sys +sys.path.append(os.getenv("IDEFIX_DIR")) + +import pytools.idfx_test as tst +tolerance=3e-14 +def testMe(test): + test.configure() + test.compile() + inifiles=["idefix.ini","idefix-rkl.ini"] + + for ini in inifiles: + test.run(inputFile=ini) + mytol=tolerance + if ini=="idefix-rkl.ini": + mytol=1e-8 + + if test.init and not test.mpi: + test.makeReference(filename="dump.0001.dmp") + test.standardTest() + test.nonRegressionTest(filename="dump.0001.dmp",tolerance=mytol) + + +test=tst.idfxTest() +if not test.dec: + test.dec=['2','2'] + +if not test.all: + if(test.check): + test.checkOnly(filename="dump.0001.dmp",tolerance=tolerance) + else: + testMe(test) +else: + test.noplot = True + test.single=False + test.reconstruction=2 + test.mpi=False + testMe(test) + + test.mpi=True + testMe(test) diff --git a/test/HD/sod-iso/idefix-hll.ini b/test/HD/sod-iso/idefix-hll.ini index 7fc7cf3e..cb6ab432 100644 --- a/test/HD/sod-iso/idefix-hll.ini +++ b/test/HD/sod-iso/idefix-hll.ini @@ -23,3 +23,4 @@ X3-end outflow [Output] vtk 0.1 +dmp 0.2 diff --git a/test/HD/sod-iso/idefix-hllc-rk3.ini b/test/HD/sod-iso/idefix-hllc-rk3.ini new file mode 100644 index 00000000..7606c7fc --- /dev/null +++ b/test/HD/sod-iso/idefix-hllc-rk3.ini @@ -0,0 +1,26 @@ +[Grid] +X1-grid 1 0.0 500 u 1.0 +X2-grid 1 0.0 1 u 1.0 +X3-grid 1 0.0 1 u 1.0 + +[TimeIntegrator] +CFL 0.8 +tstop 0.2 +first_dt 1.e-4 +nstages 3 + +[Hydro] +solver hllc +csiso constant 1.0 + +[Boundary] +X1-beg outflow +X1-end outflow +X2-beg outflow +X2-end outflow +X3-beg outflow +X3-end outflow + +[Output] +vtk 0.1 +dmp 0.2 diff --git a/test/HD/sod-iso/idefix-hllc.ini b/test/HD/sod-iso/idefix-hllc.ini index 6835b6fe..751f5e86 100644 --- a/test/HD/sod-iso/idefix-hllc.ini +++ b/test/HD/sod-iso/idefix-hllc.ini @@ -23,3 +23,4 @@ X3-end outflow [Output] vtk 0.1 +dmp 0.2 diff --git a/test/HD/sod-iso/idefix-rk3.ini b/test/HD/sod-iso/idefix-rk3.ini new file mode 100644 index 00000000..e913683d --- /dev/null +++ b/test/HD/sod-iso/idefix-rk3.ini @@ -0,0 +1,26 @@ +[Grid] +X1-grid 1 0.0 500 u 1.0 +X2-grid 1 0.0 1 u 1.0 +X3-grid 1 0.0 1 u 1.0 + +[TimeIntegrator] +CFL 0.8 +tstop 0.2 +first_dt 1.e-4 +nstages 3 + +[Hydro] +solver roe +csiso constant 1.0 + +[Boundary] +X1-beg outflow +X1-end outflow +X2-beg outflow +X2-end outflow +X3-beg outflow +X3-end outflow + +[Output] +vtk 0.1 +dmp 0.2 diff --git a/test/HD/sod-iso/idefix.ini b/test/HD/sod-iso/idefix.ini index d80fd365..564286e3 100644 --- a/test/HD/sod-iso/idefix.ini +++ b/test/HD/sod-iso/idefix.ini @@ -23,3 +23,4 @@ X3-end outflow [Output] vtk 0.1 +dmp 0.2 diff --git a/test/HD/sod-iso/testme.py b/test/HD/sod-iso/testme.py new file mode 100755 index 00000000..e8f0f9ae --- /dev/null +++ b/test/HD/sod-iso/testme.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 + +""" + +@author: glesur +""" +import os +import sys +sys.path.append(os.getenv("IDEFIX_DIR")) + +import pytools.idfx_test as tst + +name="dump.0001.dmp" + +def testMe(test): + test.configure() + test.compile() + inifiles=["idefix.ini","idefix-hll.ini","idefix-hllc.ini","idefix-tvdlf.ini"] + if test.reconstruction==4: + inifiles=["idefix-rk3.ini","idefix-hllc-rk3.ini"] + + # loop on all the ini files for this test + for ini in inifiles: + test.run(inputFile=ini) + if test.init: + test.makeReference(filename=name) + test.standardTest() + test.nonRegressionTest(filename=name) + + +test=tst.idfxTest() + +if not test.all: + if(test.check): + test.checkOnly(filename=name) + else: + testMe(test) +else: + test.noplot = True + for rec in range(2,5): + test.vectPot=False + test.single=False + test.reconstruction=rec + test.mpi=False + testMe(test) + + # test in single precision + test.reconstruction=2 + test.single=True + testMe(test) diff --git a/test/HD/sod/idefix-hll.ini b/test/HD/sod/idefix-hll.ini index e639bb4e..0bfb03ce 100644 --- a/test/HD/sod/idefix-hll.ini +++ b/test/HD/sod/idefix-hll.ini @@ -23,3 +23,4 @@ X3-end outflow [Output] vtk 0.1 +dmp 0.2 diff --git a/test/HD/sod/idefix-hllc-rk3.ini b/test/HD/sod/idefix-hllc-rk3.ini new file mode 100644 index 00000000..68a97c55 --- /dev/null +++ b/test/HD/sod/idefix-hllc-rk3.ini @@ -0,0 +1,26 @@ +[Grid] +X1-grid 1 0.0 500 u 1.0 +X2-grid 1 0.0 1 u 1.0 +X3-grid 1 0.0 1 u 1.0 + +[TimeIntegrator] +CFL 0.8 +tstop 0.2 +first_dt 1.e-4 +nstages 3 + +[Hydro] +solver hllc +gamma 1.4 + +[Boundary] +X1-beg outflow +X1-end outflow +X2-beg outflow +X2-end outflow +X3-beg outflow +X3-end outflow + +[Output] +vtk 0.1 +dmp 0.2 diff --git a/test/HD/sod/idefix-hllc.ini b/test/HD/sod/idefix-hllc.ini index 32752517..3db50c3c 100644 --- a/test/HD/sod/idefix-hllc.ini +++ b/test/HD/sod/idefix-hllc.ini @@ -23,3 +23,4 @@ X3-end outflow [Output] vtk 0.1 +dmp 0.2 diff --git a/test/HD/sod/idefix-rk3.ini b/test/HD/sod/idefix-rk3.ini new file mode 100644 index 00000000..7fe738ed --- /dev/null +++ b/test/HD/sod/idefix-rk3.ini @@ -0,0 +1,26 @@ +[Grid] +X1-grid 1 0.0 500 u 1.0 +X2-grid 1 0.0 1 u 1.0 +X3-grid 1 0.0 1 u 1.0 + +[TimeIntegrator] +CFL 0.8 +tstop 0.2 +first_dt 1.e-4 +nstages 3 + +[Hydro] +solver roe +gamma 1.4 + +[Boundary] +X1-beg outflow +X1-end outflow +X2-beg outflow +X2-end outflow +X3-beg outflow +X3-end outflow + +[Output] +vtk 0.1 +dmp 0.2 diff --git a/test/HD/sod/idefix-tvdlf.ini b/test/HD/sod/idefix-tvdlf.ini index e8745045..36133b9b 100644 --- a/test/HD/sod/idefix-tvdlf.ini +++ b/test/HD/sod/idefix-tvdlf.ini @@ -23,3 +23,4 @@ X3-end outflow [Output] vtk 0.1 +dmp 0.2 diff --git a/test/HD/sod/idefix.ini b/test/HD/sod/idefix.ini index 8b796c51..b34a23c3 100644 --- a/test/HD/sod/idefix.ini +++ b/test/HD/sod/idefix.ini @@ -23,3 +23,4 @@ X3-end outflow [Output] vtk 0.1 +dmp 0.2 diff --git a/test/HD/sod/testme.py b/test/HD/sod/testme.py new file mode 100755 index 00000000..e8f0f9ae --- /dev/null +++ b/test/HD/sod/testme.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 + +""" + +@author: glesur +""" +import os +import sys +sys.path.append(os.getenv("IDEFIX_DIR")) + +import pytools.idfx_test as tst + +name="dump.0001.dmp" + +def testMe(test): + test.configure() + test.compile() + inifiles=["idefix.ini","idefix-hll.ini","idefix-hllc.ini","idefix-tvdlf.ini"] + if test.reconstruction==4: + inifiles=["idefix-rk3.ini","idefix-hllc-rk3.ini"] + + # loop on all the ini files for this test + for ini in inifiles: + test.run(inputFile=ini) + if test.init: + test.makeReference(filename=name) + test.standardTest() + test.nonRegressionTest(filename=name) + + +test=tst.idfxTest() + +if not test.all: + if(test.check): + test.checkOnly(filename=name) + else: + testMe(test) +else: + test.noplot = True + for rec in range(2,5): + test.vectPot=False + test.single=False + test.reconstruction=rec + test.mpi=False + testMe(test) + + # test in single precision + test.reconstruction=2 + test.single=True + testMe(test) diff --git a/test/HD/thermalDiffusion/idefix-rkl.ini b/test/HD/thermalDiffusion/idefix-rkl.ini index 0c8a7da3..f3d4c34e 100644 --- a/test/HD/thermalDiffusion/idefix-rkl.ini +++ b/test/HD/thermalDiffusion/idefix-rkl.ini @@ -26,3 +26,4 @@ X3-end periodic [Output] analysis 0.01 +dmp 0.2 diff --git a/test/HD/thermalDiffusion/idefix.ini b/test/HD/thermalDiffusion/idefix.ini index 590293d4..6018a31e 100644 --- a/test/HD/thermalDiffusion/idefix.ini +++ b/test/HD/thermalDiffusion/idefix.ini @@ -26,3 +26,4 @@ X3-end periodic [Output] analysis 0.01 +dmp 0.2 diff --git a/test/HD/thermalDiffusion/setup.cpp b/test/HD/thermalDiffusion/setup.cpp index 228b9406..dab61a16 100644 --- a/test/HD/thermalDiffusion/setup.cpp +++ b/test/HD/thermalDiffusion/setup.cpp @@ -23,9 +23,11 @@ void Analysis(DataBlock & data) { } -void InternalBoundary(DataBlock& data, const real t) { - IdefixArray4D Vc = data.hydro.Vc; - idefix_for("InternalBoundary",0,data.np_tot[KDIR],0,data.np_tot[JDIR],0,data.np_tot[IDIR], +void InternalBoundary(Fluid * hydro, const real t) { + IdefixArray4D Vc = hydro->Vc; + idefix_for("InternalBoundary",0,hydro->data->np_tot[KDIR], + 0,hydro->data->np_tot[JDIR], + 0,hydro->data->np_tot[IDIR], KOKKOS_LAMBDA (int k, int j, int i) { // Cancel any motion that could be happening Vc(VX1,k,j,i) = 0.0; @@ -39,7 +41,7 @@ void InternalBoundary(DataBlock& data, const real t) { // Arrays or variables which are used later on Setup::Setup(Input &input, Grid &grid, DataBlock &data, Output &output) { output.EnrollAnalysis(&Analysis); - data.hydro.EnrollInternalBoundary(&InternalBoundary); + data.hydro->EnrollInternalBoundary(&InternalBoundary); amplitude = input.Get("Setup","amplitude",0); // Initialise the output file diff --git a/test/HD/thermalDiffusion/testme.py b/test/HD/thermalDiffusion/testme.py new file mode 100755 index 00000000..3806293d --- /dev/null +++ b/test/HD/thermalDiffusion/testme.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 + +""" + +@author: glesur +""" +import os +import sys +sys.path.append(os.getenv("IDEFIX_DIR")) + +import pytools.idfx_test as tst + +def testMe(test): + test.configure() + test.compile() + inifiles=["idefix.ini","idefix-rkl.ini"] + + for ini in inifiles: + test.run(inputFile=ini) + if test.init and not test.mpi: + test.makeReference(filename="dump.0001.dmp") + test.standardTest() + test.nonRegressionTest(filename="dump.0001.dmp") + + +test=tst.idfxTest() + +if not test.all: + if(test.check): + test.checkOnly(filename="dump.0001.dmp") + else: + testMe(test) +else: + test.noplot = True + test.single=False + test.reconstruction=2 + test.mpi=False + testMe(test) diff --git a/test/MHD/AmbipolarCshock/idefix-rkl.ini b/test/MHD/AmbipolarCshock/idefix-rkl.ini index 108ff1b1..e02b3415 100644 --- a/test/MHD/AmbipolarCshock/idefix-rkl.ini +++ b/test/MHD/AmbipolarCshock/idefix-rkl.ini @@ -24,4 +24,5 @@ X3-end periodic [Output] vtk 100.0 +dmp 200.0 log 1000 diff --git a/test/MHD/AmbipolarCshock/idefix.ini b/test/MHD/AmbipolarCshock/idefix.ini index f71be8d8..fa21e04e 100644 --- a/test/MHD/AmbipolarCshock/idefix.ini +++ b/test/MHD/AmbipolarCshock/idefix.ini @@ -24,4 +24,5 @@ X3-end periodic [Output] vtk 100.0 +dmp 100.0 log 1000 diff --git a/test/MHD/AmbipolarCshock/setup.cpp b/test/MHD/AmbipolarCshock/setup.cpp index f249f3c9..65f42ffd 100644 --- a/test/MHD/AmbipolarCshock/setup.cpp +++ b/test/MHD/AmbipolarCshock/setup.cpp @@ -16,7 +16,7 @@ static real cs; void AmbipolarFunction(DataBlock &data, real t, IdefixArray3D &xAin ) { IdefixArray3D xA = xAin; - IdefixArray4D Vc = data.hydro.Vc; + IdefixArray4D Vc = data.hydro->Vc; idefix_for("AmbipolarFunction",0,data.np_tot[KDIR],0,data.np_tot[JDIR],0,data.np_tot[IDIR], KOKKOS_LAMBDA (int k, int j, int i) { xA(k,j,i) = 2.0/Vc(RHO,k,j,i); @@ -24,13 +24,17 @@ void AmbipolarFunction(DataBlock &data, real t, IdefixArray3D &xAin ) { } // User-defined boundaries -void UserdefBoundary(DataBlock& data, int dir, BoundarySide side, real t) { +void UserdefBoundary(Hydro *hydro, int dir, BoundarySide side, real t) { + auto *data = hydro->data; if( (dir==IDIR) && (side == left)) { - IdefixArray4D Vc = data.hydro.Vc; - IdefixArray4D Vs = data.hydro.Vs; - - int ighost = data.nghost[IDIR]; - idefix_for("UserDefBoundary",0,data.np_tot[KDIR],0,data.np_tot[JDIR],0,ighost, + IdefixArray4D Vc = hydro->Vc; + IdefixArray4D Vs = hydro->Vs; + + int ighost = data->nghost[IDIR]; + idefix_for("UserDefBoundary", + 0, data->np_tot[KDIR], + 0, data->np_tot[JDIR], + 0, ighost, KOKKOS_LAMBDA (int k, int j, int i) { Vc(RHO,k,j,i) = Vc(RHO,k,j,ighost); #if HAVE_ENERGY @@ -49,11 +53,14 @@ void UserdefBoundary(DataBlock& data, int dir, BoundarySide side, real t) { }); } if( (dir==IDIR) && (side == right)) { - IdefixArray4D Vc = data.hydro.Vc; - IdefixArray4D Vs = data.hydro.Vs; - - int ighost = data.end[IDIR]-1; - idefix_for("UserDefBoundary",0,data.np_tot[KDIR],0,data.np_tot[JDIR],data.end[IDIR],data.np_tot[IDIR], + IdefixArray4D Vc = hydro->Vc; + IdefixArray4D Vs = hydro->Vs; + + int ighost = data->end[IDIR]-1; + idefix_for("UserDefBoundary", + 0, data->np_tot[KDIR], + 0, data->np_tot[JDIR], + data->end[IDIR], data->np_tot[IDIR], KOKKOS_LAMBDA (int k, int j, int i) { Vc(RHO,k,j,i) = Vc(RHO,k,j,2*ighost-i+1); #if HAVE_ENERGY @@ -81,8 +88,8 @@ void UserdefBoundary(DataBlock& data, int dir, BoundarySide side, real t) { // Initialisation routine. Can be used to allocate // Arrays or variables which are used later on Setup::Setup(Input &input, Grid &grid, DataBlock &data, Output &output) { - data.hydro.EnrollUserDefBoundary(&UserdefBoundary); - data.hydro.EnrollAmbipolarDiffusivity(&AmbipolarFunction); + data.hydro->EnrollUserDefBoundary(&UserdefBoundary); + data.hydro->EnrollAmbipolarDiffusivity(&AmbipolarFunction); cs=input.Get("Hydro","csiso",1); } diff --git a/test/MHD/AmbipolarCshock/testme.py b/test/MHD/AmbipolarCshock/testme.py new file mode 100755 index 00000000..675958a7 --- /dev/null +++ b/test/MHD/AmbipolarCshock/testme.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 + +""" + +@author: glesur +""" +import os +import sys +sys.path.append(os.getenv("IDEFIX_DIR")) + +import pytools.idfx_test as tst +tolerance=0 + +def testMe(test): + test.configure() + test.compile() + inifiles=["idefix.ini","idefix-rkl.ini"] + for ini in inifiles: + test.run(inputFile=ini) + if test.init and not test.mpi: + test.makeReference(filename="dump.0001.dmp") + test.standardTest() + test.nonRegressionTest(filename="dump.0001.dmp",tolerance=tolerance) + + +test=tst.idfxTest() + +if not test.all: + if(test.check): + test.checkOnly(filename="dump.0001.dmp",tolerance=tolerance) + else: + testMe(test) +else: + test.noplot = True + test.single=False + test.reconstruction=2 + test.mpi=False + testMe(test) diff --git a/test/MHD/AmbipolarCshock3D/idefix-rkl.ini b/test/MHD/AmbipolarCshock3D/idefix-rkl.ini index 64029a2a..05542955 100644 --- a/test/MHD/AmbipolarCshock3D/idefix-rkl.ini +++ b/test/MHD/AmbipolarCshock3D/idefix-rkl.ini @@ -27,4 +27,5 @@ X3-end periodic [Output] vtk 100.0 +dmp 100.0 log 1000 diff --git a/test/MHD/AmbipolarCshock3D/idefix.ini b/test/MHD/AmbipolarCshock3D/idefix.ini index ef8a87fb..984e985e 100644 --- a/test/MHD/AmbipolarCshock3D/idefix.ini +++ b/test/MHD/AmbipolarCshock3D/idefix.ini @@ -1,7 +1,7 @@ [Grid] X1-grid 1 0.0 50 u 50.0 -X2-grid 1 0.0 4 u 10.0 -X3-grid 1 0.0 4 u 10.0 +X2-grid 1 0.0 2 u 10.0 +X3-grid 1 0.0 2 u 10.0 [TimeIntegrator] CFL 0.9 @@ -24,4 +24,5 @@ X3-end periodic [Output] vtk 100.0 +dmp 100.0 log 1000 diff --git a/test/MHD/AmbipolarCshock3D/setup.cpp b/test/MHD/AmbipolarCshock3D/setup.cpp index dd21f565..5f8ec931 100644 --- a/test/MHD/AmbipolarCshock3D/setup.cpp +++ b/test/MHD/AmbipolarCshock3D/setup.cpp @@ -16,7 +16,7 @@ static real cs; void AmbipolarFunction(DataBlock &data, real t, IdefixArray3D &xAin ) { IdefixArray3D xA = xAin; - IdefixArray4D Vc = data.hydro.Vc; + IdefixArray4D Vc = data.hydro->Vc; idefix_for("AmbipolarFunction",0,data.np_tot[KDIR],0,data.np_tot[JDIR],0,data.np_tot[IDIR], KOKKOS_LAMBDA (int k, int j, int i) { xA(k,j,i) = 2.0/Vc(RHO,k,j,i); @@ -24,13 +24,17 @@ void AmbipolarFunction(DataBlock &data, real t, IdefixArray3D &xAin ) { } // User-defined boundaries -void UserdefBoundary(DataBlock& data, int dir, BoundarySide side, real t) { +void UserdefBoundary(Hydro *hydro, int dir, BoundarySide side, real t) { + auto *data = hydro->data; if( (dir==IDIR) && (side == left)) { - IdefixArray4D Vc = data.hydro.Vc; - IdefixArray4D Vs = data.hydro.Vs; - - int ighost = data.nghost[IDIR]; - idefix_for("UserDefBoundary",0,data.np_tot[KDIR],0,data.np_tot[JDIR],0,ighost, + IdefixArray4D Vc = hydro->Vc; + IdefixArray4D Vs = hydro->Vs; + + int ighost = data->nghost[IDIR]; + idefix_for("UserDefBoundary", + 0, data->np_tot[KDIR], + 0, data->np_tot[JDIR], + 0, ighost, KOKKOS_LAMBDA (int k, int j, int i) { Vc(RHO,k,j,i) = Vc(RHO,k,j,ighost); #if HAVE_ENERGY @@ -49,11 +53,14 @@ void UserdefBoundary(DataBlock& data, int dir, BoundarySide side, real t) { }); } if( (dir==IDIR) && (side == right)) { - IdefixArray4D Vc = data.hydro.Vc; - IdefixArray4D Vs = data.hydro.Vs; - - int ighost = data.end[IDIR]-1; - idefix_for("UserDefBoundary",0,data.np_tot[KDIR],0,data.np_tot[JDIR],data.end[IDIR],data.np_tot[IDIR], + IdefixArray4D Vc = hydro->Vc; + IdefixArray4D Vs = hydro->Vs; + + int ighost = data->end[IDIR]-1; + idefix_for("UserDefBoundary", + 0, data->np_tot[KDIR], + 0, data->np_tot[JDIR], + data->end[IDIR], data->np_tot[IDIR], KOKKOS_LAMBDA (int k, int j, int i) { Vc(RHO,k,j,i) = Vc(RHO,k,j,2*ighost-i+1); #if HAVE_ENERGY @@ -81,8 +88,8 @@ void UserdefBoundary(DataBlock& data, int dir, BoundarySide side, real t) { // Initialisation routine. Can be used to allocate // Arrays or variables which are used later on Setup::Setup(Input &input, Grid &grid, DataBlock &data, Output &output) { - data.hydro.EnrollUserDefBoundary(&UserdefBoundary); - data.hydro.EnrollAmbipolarDiffusivity(&AmbipolarFunction); + data.hydro->EnrollUserDefBoundary(&UserdefBoundary); + data.hydro->EnrollAmbipolarDiffusivity(&AmbipolarFunction); cs=input.Get("Hydro","csiso",1); } diff --git a/test/MHD/AmbipolarCshock3D/testme.py b/test/MHD/AmbipolarCshock3D/testme.py new file mode 100755 index 00000000..f9fae676 --- /dev/null +++ b/test/MHD/AmbipolarCshock3D/testme.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 + +""" + +@author: glesur +""" +import os +import sys +sys.path.append(os.getenv("IDEFIX_DIR")) + +import pytools.idfx_test as tst +tolerance=2e-14 +def testMe(test): + test.configure() + test.compile() + inifiles=["idefix.ini","idefix-rkl.ini"] + for ini in inifiles: + mytol=tolerance + + test.run(inputFile=ini) + if test.init and not test.mpi: + test.makeReference(filename="dump.0001.dmp") + test.standardTest() + # When using RKL, except larger error due to B reconstruction and RKL # of substeps + if test.mpi and ini=="idefix-rkl.ini": + mytol=2e-10 + test.nonRegressionTest(filename="dump.0001.dmp",tolerance=mytol) + + +test=tst.idfxTest() +if not test.dec: + test.dec=['2','1','1'] + +if not test.all: + if(test.check): + test.checkOnly(filename="dump.0001.dmp",tolerance=tolerance) + else: + testMe(test) +else: + test.noplot = True + test.single=False + test.reconstruction=2 + test.mpi=False + testMe(test) + + test.vectPot=True + testMe(test) + + test.vectPot=False + test.mpi=True + testMe(test) diff --git a/test/MHD/AmbipolarShearingBox/setup.cpp b/test/MHD/AmbipolarShearingBox/setup.cpp index 3e48b32f..8c9092fd 100644 --- a/test/MHD/AmbipolarShearingBox/setup.cpp +++ b/test/MHD/AmbipolarShearingBox/setup.cpp @@ -8,18 +8,22 @@ static real shear; //#define STRATIFIED // UserStep, here only gravity (vertical and radial) -void UserStep(DataBlock &data, const real t, const real dt) { +void UserStep(Hydro *hydro, const real t, const real dt) { + auto *data = hydro->data; Kokkos::Profiling::pushRegion("Setup::UserStep"); - IdefixArray4D Uc = data.hydro.Uc; - IdefixArray4D Vc = data.hydro.Vc; - IdefixArray1D x = data.x[IDIR]; - IdefixArray1D z = data.x[KDIR]; + IdefixArray4D Uc = hydro->Uc; + IdefixArray4D Vc = hydro->Vc; + IdefixArray1D x = data->x[IDIR]; + IdefixArray1D z = data->x[KDIR]; // GPUS cannot capture static variables real omegaLocal=omega; real shearLocal =shear; - idefix_for("UserSourceTerms",data.beg[KDIR],data.end[KDIR],data.beg[JDIR],data.end[JDIR],data.beg[IDIR],data.end[IDIR], + idefix_for("UserSourceTerms", + data->beg[KDIR], data->end[KDIR], + data->beg[JDIR], data->end[JDIR], + data->beg[IDIR], data->end[IDIR], KOKKOS_LAMBDA (int k, int j, int i) { #ifdef STRATIFIED Uc(MX3,k,j,i) += - dt*omegaLocal*omegaLocal*z(k)*Vc(RHO,k,j,i); @@ -35,14 +39,14 @@ void UserStep(DataBlock &data, const real t, const real dt) { // Initialisation routine. Can be used to allocate // Arrays or variables which are used later on Setup::Setup(Input &input, Grid &grid, DataBlock &data, Output &output) { - gammaIdeal=data.hydro.GetGamma(); + gammaIdeal=data.hydro->eos->GetGamma(); // Get rotation rate along vertical axis omega=input.Get("Hydro","rotation",0); shear=input.Get("Hydro","shearingBox",0); // Add our userstep to the timeintegrator - data.hydro.EnrollUserSourceTerm(UserStep); + data.hydro->EnrollUserSourceTerm(UserStep); } // This routine initialize the flow diff --git a/test/MHD/AmbipolarWind/setup.cpp b/test/MHD/AmbipolarWind/setup.cpp index 32d39709..78620f6e 100644 --- a/test/MHD/AmbipolarWind/setup.cpp +++ b/test/MHD/AmbipolarWind/setup.cpp @@ -1,6 +1,5 @@ #include "idefix.hpp" #include "setup.hpp" -#include "boundaryloop.hpp" real epsilonGlob; real epsilonTopGlob; @@ -45,7 +44,7 @@ void Ambipolar(DataBlock& data, real t, IdefixArray3D &xAin) { IdefixArray3D xA = xAin; IdefixArray1D x1=data.x[IDIR]; IdefixArray1D x2=data.x[JDIR]; - IdefixArray4D Vc=data.hydro.Vc; + IdefixArray4D Vc=data.hydro->Vc; real Hideal = HidealGlob; real epsilon = epsilonGlob; @@ -85,7 +84,7 @@ void Resistivity(DataBlock& data, real t, IdefixArray3D &etain) { IdefixArray3D eta = etain; IdefixArray1D x1=data.x[IDIR]; IdefixArray1D x2=data.x[JDIR]; - IdefixArray4D Vc=data.hydro.Vc; + IdefixArray4D Vc=data.hydro->Vc; real epsilon = epsilonGlob; @@ -96,11 +95,12 @@ void Resistivity(DataBlock& data, real t, IdefixArray3D &etain) { } -void MySourceTerm(DataBlock &data, const real t, const real dtin) { - IdefixArray4D Vc = data.hydro.Vc; - IdefixArray4D Uc = data.hydro.Uc; - IdefixArray1D x1=data.x[IDIR]; - IdefixArray1D x2=data.x[JDIR]; +void MySourceTerm(Hydro *hydro, const real t, const real dtin) { + auto *data = hydro->data; + IdefixArray4D Vc = hydro->Vc; + IdefixArray4D Uc = hydro->Uc; + IdefixArray1D x1=data->x[IDIR]; + IdefixArray1D x2=data->x[JDIR]; real epsilonTop = epsilonTopGlob; real epsilon = epsilonGlob; real tauGlob=0.1; @@ -110,7 +110,10 @@ void MySourceTerm(DataBlock &data, const real t, const real dtin) { real Rin=1.0; real trSmoothing = trSmoothingGlob; - idefix_for("MySourceTerm",0,data.np_tot[KDIR],0,data.np_tot[JDIR],0,data.np_tot[IDIR], + idefix_for("MySourceTerm", + 0, data->np_tot[KDIR], + 0, data->np_tot[JDIR], + 0, data->np_tot[IDIR], KOKKOS_LAMBDA (int k, int j, int i) { real r=x1(i); real th=x2(j); @@ -163,18 +166,22 @@ void MySourceTerm(DataBlock &data, const real t, const real dtin) { -void InternalBoundary(DataBlock& data, const real t) { - IdefixArray4D Vc = data.hydro.Vc; - IdefixArray4D Vs = data.hydro.Vs; - IdefixArray1D x1=data.x[IDIR]; - IdefixArray1D x2=data.x[JDIR]; +void InternalBoundary(Hydro *hydro, const real t) { + auto *data = hydro->data; + IdefixArray4D Vc = hydro->Vc; + IdefixArray4D Vs = hydro->Vs; + IdefixArray1D x1=data->x[IDIR]; + IdefixArray1D x2=data->x[JDIR]; real vAmax=computeVaMax(4.0,50.0,8.0,t); real densityFloor0 = densityFloorGlob; real Rin = 1.0; real epsilon=epsilonGlob; - idefix_for("InternalBoundary",0,data.np_tot[KDIR],0,data.np_tot[JDIR],0,data.np_tot[IDIR], + idefix_for("InternalBoundary", + 0, data->np_tot[KDIR], + 0, data->np_tot[JDIR], + 0, data->np_tot[IDIR], KOKKOS_LAMBDA (int k, int j, int i) { real R=x1(i)*sin(x2(j)); real z=x1(i)*cos(x2(j)); @@ -207,15 +214,15 @@ void InternalBoundary(DataBlock& data, const real t) { } // User-defined boundaries -void UserdefBoundary(DataBlock& data, int dir, BoundarySide side, real t) { - +void UserdefBoundary(Hydro *hydro, int dir, BoundarySide side, real t) { + auto *data = hydro->data; if( (dir==IDIR) && (side == left)) { - IdefixArray4D Vc = data.hydro.Vc; - IdefixArray4D Vs = data.hydro.Vs; - IdefixArray1D x1 = data.x[IDIR]; - IdefixArray1D x2 = data.x[JDIR]; + IdefixArray4D Vc = hydro->Vc; + IdefixArray4D Vs = hydro->Vs; + IdefixArray1D x1 = data->x[IDIR]; + IdefixArray1D x2 = data->x[JDIR]; - int ighost = data.nghost[IDIR]; + int ighost = data->nghost[IDIR]; real Omega=1.0; real Rin = 1.0; real csdisk = epsilonGlob/sqrt(Rin); @@ -223,7 +230,7 @@ void UserdefBoundary(DataBlock& data, int dir, BoundarySide side, real t) { real densityFloor0 = densityFloorGlob; real epsilon=epsilonGlob; - data.hydro.boundary.BoundaryFor("UserDefX1",dir,side, + hydro->boundary->BoundaryFor("UserDefX1",dir,side, KOKKOS_LAMBDA (int k, int j, int i) { real R=x1(i)*sin(x2(j)); real z=x1(i)*cos(x2(j)); @@ -248,7 +255,7 @@ void UserdefBoundary(DataBlock& data, int dir, BoundarySide side, real t) { //Vc(BX3,k,j,i) = Vc(BX3,k,j,ighost); }); - data.hydro.boundary.BoundaryForX2s("UserDefX1",dir,side, + hydro->boundary->BoundaryForX2s("UserDefX1",dir,side, KOKKOS_LAMBDA (int k, int j, int i) { Vs(BX2s,k,j,i) = Vs(BX2s,k,j,ighost); }); @@ -256,17 +263,17 @@ void UserdefBoundary(DataBlock& data, int dir, BoundarySide side, real t) { } if( (dir==IDIR) && (side == right)) { - IdefixArray4D Vc = data.hydro.Vc; - IdefixArray4D Vs = data.hydro.Vs; - IdefixArray1D x1 = data.x[IDIR]; - IdefixArray1D x2 = data.x[JDIR]; + IdefixArray4D Vc = hydro->Vc; + IdefixArray4D Vs = hydro->Vs; + IdefixArray1D x1 = data->x[IDIR]; + IdefixArray1D x2 = data->x[JDIR]; - int ighost = data.end[IDIR]-1; + int ighost = data->end[IDIR]-1; real Rin = 1.0; real csdisk = epsilonGlob/sqrt(Rin); real cscorona = epsilonTopGlob/sqrt(Rin); - data.hydro.boundary.BoundaryFor("UserDefX1",dir,side, + hydro->boundary->BoundaryFor("UserDefX1",dir,side, KOKKOS_LAMBDA (int k, int j, int i) { real R=x1(i)*sin(x2(j)); real z=x1(i)*cos(x2(j)); @@ -285,7 +292,7 @@ void UserdefBoundary(DataBlock& data, int dir, BoundarySide side, real t) { //Vc(BX3,k,j,i) = Vc(BX3,k,j,ighost); }); - data.hydro.boundary.BoundaryForX2s("UserDefX1",dir,side, + hydro->boundary->BoundaryForX2s("UserDefX1",dir,side, KOKKOS_LAMBDA (int k, int j, int i) { Vs(BX2s,k,j,i) = Vs(BX2s,k,j,ighost); }); @@ -296,9 +303,9 @@ void UserdefBoundary(DataBlock& data, int dir, BoundarySide side, real t) { } void EmfBoundary(DataBlock& data, const real t) { - IdefixArray3D Ex1 = data.hydro.emf.ex; - IdefixArray3D Ex2 = data.hydro.emf.ey; - IdefixArray3D Ex3 = data.hydro.emf.ez; + IdefixArray3D Ex1 = data.hydro->emf->ex; + IdefixArray3D Ex2 = data.hydro->emf->ey; + IdefixArray3D Ex3 = data.hydro->emf->ez; if(data.lbound[IDIR] == userdef) { int ighost = data.beg[IDIR]; @@ -328,7 +335,7 @@ void EmfBoundary(DataBlock& data, const real t) { } void FluxBoundary(DataBlock & data, int dir, BoundarySide side, const real t) { - IdefixArray4D Flux = data.hydro.FluxRiemann; + IdefixArray4D Flux = data.hydro->FluxRiemann; if( dir==IDIR && side == left) { int iref = data.beg[IDIR]; @@ -389,15 +396,15 @@ void ComputeUserVars(DataBlock & data, UserDefVariablesContainer &variables) { // Arrays or variables which are used later on Setup::Setup(Input &input, Grid &grid, DataBlock &data, Output &output) { // Set the function for userdefboundary - data.hydro.EnrollUserDefBoundary(&UserdefBoundary); - data.hydro.EnrollAmbipolarDiffusivity(&Ambipolar); - //data.hydro.EnrollOhmicDiffusivity(&Resistivity); - data.hydro.EnrollUserSourceTerm(&MySourceTerm); - data.hydro.EnrollInternalBoundary(&InternalBoundary); - data.hydro.EnrollEmfBoundary(&EmfBoundary); - //data.hydro.EnrollFluxBoundary(&FluxBoundary); + data.hydro->EnrollUserDefBoundary(&UserdefBoundary); + data.hydro->EnrollAmbipolarDiffusivity(&Ambipolar); + //data.hydro->EnrollOhmicDiffusivity(&Resistivity); + data.hydro->EnrollUserSourceTerm(&MySourceTerm); + data.hydro->EnrollInternalBoundary(&InternalBoundary); + data.hydro->EnrollEmfBoundary(&EmfBoundary); + //data.hydro->EnrollFluxBoundary(&FluxBoundary); output.EnrollUserDefVariables(&ComputeUserVars); - gammaGlob=data.hydro.GetGamma(); + gammaGlob=data.hydro->eos->GetGamma(); epsilonGlob = input.Get("Setup","epsilon",0); epsilonTopGlob = input.Get("Setup","epsilonTop",0); betaGlob = input.Get("Setup","beta",0); diff --git a/test/MHD/AxisFluxTube/idefix-coarsening.ini b/test/MHD/AxisFluxTube/idefix-coarsening.ini index fe33f231..adf692d7 100644 --- a/test/MHD/AxisFluxTube/idefix-coarsening.ini +++ b/test/MHD/AxisFluxTube/idefix-coarsening.ini @@ -30,4 +30,5 @@ Rin 0.4 [Output] uservar divB Er vtk 2.0 +dmp 2.0 log 100 diff --git a/test/MHD/AxisFluxTube/idefix.ini b/test/MHD/AxisFluxTube/idefix.ini index 140ecc5c..b03798ad 100644 --- a/test/MHD/AxisFluxTube/idefix.ini +++ b/test/MHD/AxisFluxTube/idefix.ini @@ -29,4 +29,5 @@ Rin 0.4 [Output] uservar divB Er vtk 2.0 +dmp 2.0 log 100 diff --git a/test/MHD/AxisFluxTube/python/checkAxisBounds.py b/test/MHD/AxisFluxTube/python/checkAxisBounds.py index 0a3ca521..44eec58e 100755 --- a/test/MHD/AxisFluxTube/python/checkAxisBounds.py +++ b/test/MHD/AxisFluxTube/python/checkAxisBounds.py @@ -8,13 +8,13 @@ import os import sys sys.path.append(os.getenv("IDEFIX_DIR")) -from pytools.idfx_io import read_idfx +from pytools.idfx_io import readIdfxFile import matplotlib.pyplot as plt rep="../" -d=read_idfx(rep+"analysis.16.0.idfx") - +idfx=readIdfxFile(rep+"analysis.99.0.idfx") +d=idfx.data Bx1s=d['Vs'][0,:-1,:-1,:] Bx2s=d['Vs'][1,:-1,:,:-1] Bx3s=d['Vs'][2,:,:-1,:-1] @@ -25,10 +25,10 @@ Bx3slice=Bx3s[:,:,nxhalf] -plt.figure() plt.matshow(Bx1slice,cmap='RdBu_r') plt.colorbar() plt.title("Bx1") + plt.matshow(Bx2slice,cmap='RdBu_r') plt.colorbar() plt.title("Bx2") @@ -46,7 +46,7 @@ Vx2slice=d['Vc'][2,:,:5,nxhalf] Vx3slice=d['Vc'][3,:,:5,nxhalf] -plt.figure() + plt.matshow(Bx1slice,cmap='RdBu_r') plt.colorbar() plt.title("Bx1") @@ -57,7 +57,6 @@ plt.colorbar() plt.title("Bx3") -plt.figure() plt.matshow(Vx1slice,cmap='RdBu_r') plt.colorbar() plt.title("Vx1") @@ -67,3 +66,5 @@ plt.matshow(Vx3slice,cmap='RdBu_r') plt.colorbar() plt.title("Vx3") + +plt.show() diff --git a/test/MHD/AxisFluxTube/python/data.0001.ref-coarsening.vtk b/test/MHD/AxisFluxTube/python/data.0001.ref-coarsening.vtk deleted file mode 100644 index 793d5f75..00000000 --- a/test/MHD/AxisFluxTube/python/data.0001.ref-coarsening.vtk +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:17eab4057262df01574813ac82a2ea4c7f1481f384106f4e0954f710053847e8 -size 3471503 diff --git a/test/MHD/AxisFluxTube/python/data.0001.ref.vtk b/test/MHD/AxisFluxTube/python/data.0001.ref.vtk deleted file mode 100644 index 8ee1aabe..00000000 --- a/test/MHD/AxisFluxTube/python/data.0001.ref.vtk +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:75a7998cb31da5182f725888f4374e913cb54c551b2880767339e7ffcc94f2cd -size 3471503 diff --git a/test/MHD/AxisFluxTube/python/testidefix.py b/test/MHD/AxisFluxTube/python/testidefix.py deleted file mode 100755 index d082b5aa..00000000 --- a/test/MHD/AxisFluxTube/python/testidefix.py +++ /dev/null @@ -1,52 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -""" -Created on Thu Mar 5 11:29:41 2020 - -@author: glesur -""" - -import os -import sys -sys.path.append(os.getenv("IDEFIX_DIR")) -from pytools.vtk_io import readVTK -import numpy as np -import argparse - -def checkInput(key): - f = open(args.i,'r') - content = f.read() - if content.find(key) > -1: - return True - else: - return False - -parser = argparse.ArgumentParser() -parser.add_argument("-i", - default=False, - help="idefix input file") - -args, unknown=parser.parse_known_args() - -referenceFile = 'data.0001.ref.vtk' - -if(args.i): - if checkInput("coarsening"): - referenceFile = 'data.0001.ref-coarsening.vtk' - -V=readVTK('../data.0001.vtk') -U=readVTK(referenceFile) - -# Compute BRMS -Brms_ref=np.sqrt(V.data['BX1']**2+V.data['BX2']**2+V.data['BX3']**2) -Brms_sim=np.sqrt(U.data['BX1']**2+U.data['BX2']**2+U.data['BX3']**2) - -error=np.mean(np.abs(Brms_ref-Brms_sim)*1e20,axis=(0,1,2)) - -print("Error=%e"%error) -if error<8e-5: - print("SUCCESS!") - sys.exit(0) -else: - print("FAILURE!") - sys.exit(1) diff --git a/test/MHD/AxisFluxTube/setup.cpp b/test/MHD/AxisFluxTube/setup.cpp index e6b37de8..34009da8 100644 --- a/test/MHD/AxisFluxTube/setup.cpp +++ b/test/MHD/AxisFluxTube/setup.cpp @@ -39,18 +39,18 @@ void ComputeUserVars(DataBlock & data, UserDefVariablesContainer &variables) { } } -void UserdefBoundary(DataBlock& data, int dir, BoundarySide side, real t) { - +void UserdefBoundary(Hydro *hydro, int dir, BoundarySide side, real t) { + auto *data = hydro->data; if( (dir==IDIR) && (side == right)) { - IdefixArray4D Vc = data.hydro.Vc; - IdefixArray4D Vs = data.hydro.Vs; - IdefixArray1D x1Arr = data.x[IDIR]; - IdefixArray1D x2Arr = data.x[JDIR]; - IdefixArray1D x3Arr = data.x[KDIR]; + IdefixArray4D Vc = hydro->Vc; + IdefixArray4D Vs = hydro->Vs; + IdefixArray1D x1Arr = data->x[IDIR]; + IdefixArray1D x2Arr = data->x[JDIR]; + IdefixArray1D x3Arr = data->x[KDIR]; const real sq2 = sqrt(2); - data.hydro.boundary.BoundaryFor("UserDefX1",dir,side, + hydro->boundary->BoundaryFor("UserDefX1",dir,side, KOKKOS_LAMBDA (int k, int j, int i) { real x1 = x1Arr(i); real x2 = x2Arr(j); @@ -74,11 +74,11 @@ void UserdefBoundary(DataBlock& data, int dir, BoundarySide side, real t) { Vc(VX3,k,j,i) = (ex_p+ey_p)/sq2; }); - data.hydro.boundary.BoundaryForX2s("UserDefX2s",dir,side, + hydro->boundary->BoundaryForX2s("UserDefX2s",dir,side, KOKKOS_LAMBDA (int k, int j, int i) { Vs(BX2s,k,j,i) = ZERO_F; }); - data.hydro.boundary.BoundaryForX3s("UserDefX2s",dir,side, + hydro->boundary->BoundaryForX3s("UserDefX2s",dir,side, KOKKOS_LAMBDA (int k, int j, int i) { Vs(BX3s,k,j,i) = ZERO_F; }); @@ -87,7 +87,7 @@ void UserdefBoundary(DataBlock& data, int dir, BoundarySide side, real t) { void Analysis(DataBlock & data) { // Mirror data on Host - data.hydro.boundary.SetBoundaries(data.t); + data.hydro->boundary->SetBoundaries(data.t); data.DumpToFile("analysis"); } @@ -111,8 +111,8 @@ void CoarsenFunction(DataBlock &data) { // Arrays or variables which are used later on Setup::Setup(Input &input, Grid &grid, DataBlock &data, Output &output) { output.EnrollUserDefVariables(&ComputeUserVars); - data.hydro.EnrollUserDefBoundary(&UserdefBoundary); - //output.EnrollAnalysis(&Analysis); + data.hydro->EnrollUserDefBoundary(&UserdefBoundary); + output.EnrollAnalysis(&Analysis); Rtorus = input.Get("Setup","Rtorus",0); Ztorus = input.Get("Setup","Ztorus",0); Rin = input.Get("Setup","Rin",0); diff --git a/test/MHD/AxisFluxTube/testme.py b/test/MHD/AxisFluxTube/testme.py new file mode 100755 index 00000000..44a53dc2 --- /dev/null +++ b/test/MHD/AxisFluxTube/testme.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 + +""" + +@author: glesur +""" +import os +import sys +sys.path.append(os.getenv("IDEFIX_DIR")) + +import pytools.idfx_test as tst + +name="dump.0001.dmp" + +tolerance=1e-14 + +def testMe(test): + test.configure() + test.compile() + inifiles=["idefix.ini","idefix-coarsening.ini"] + + # loop on all the ini files for this test + for ini in inifiles: + test.run(inputFile=ini) + if test.init and not test.mpi: + test.makeReference(filename=name) + test.nonRegressionTest(filename=name,tolerance=tolerance) + + +test=tst.idfxTest() + +if not test.all: + if(test.check): + test.checkOnly(filename=name,tolerance=tolerance) + else: + testMe(test) +else: + test.noplot = True + + test.vectPot=False + test.single=False + test.reconstruction=2 + test.mpi=False + testMe(test) + + # test with MPI + test.mpi=True + testMe(test) diff --git a/test/MHD/FargoMHDSpherical/idefix.ini b/test/MHD/FargoMHDSpherical/idefix.ini index 88e8bdd2..4165ebc7 100644 --- a/test/MHD/FargoMHDSpherical/idefix.ini +++ b/test/MHD/FargoMHDSpherical/idefix.ini @@ -29,4 +29,5 @@ X3-end periodic [Output] vtk 2.0 +dmp 2.0 log 10 diff --git a/test/MHD/FargoMHDSpherical/python/data.0001.ref.vtk b/test/MHD/FargoMHDSpherical/python/data.0001.ref.vtk deleted file mode 100644 index a8de80b9..00000000 --- a/test/MHD/FargoMHDSpherical/python/data.0001.ref.vtk +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:eea8bfcc6f6619fe04dc48d99785186d7a98acb7d54dae481bedba78fe6e0a8e -size 2697336 diff --git a/test/MHD/FargoMHDSpherical/python/testidefix.py b/test/MHD/FargoMHDSpherical/python/testidefix.py deleted file mode 100755 index 3a4700b4..00000000 --- a/test/MHD/FargoMHDSpherical/python/testidefix.py +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -""" -Created on Thu Mar 5 11:29:41 2020 - -@author: glesur -""" - -import os -import sys -sys.path.append(os.getenv("IDEFIX_DIR")) -from pytools.vtk_io import readVTK -import numpy as np - -V=readVTK('../data.0001.vtk', geometry='spherical') -U=readVTK('data.0001.ref.vtk', geometry='spherical') - -# Compute the error on BX1 -error=np.mean(np.abs(V.data['BX1']-U.data['BX1']))/np.amax(V.data['BX1']) - -print("Error=%e"%error) -if error<3.0e-2: - print("SUCCESS!") - sys.exit(0) -else: - print("FAILURE!") - sys.exit(1) diff --git a/test/MHD/FargoMHDSpherical/setup.cpp b/test/MHD/FargoMHDSpherical/setup.cpp index 1760350d..0ff53699 100644 --- a/test/MHD/FargoMHDSpherical/setup.cpp +++ b/test/MHD/FargoMHDSpherical/setup.cpp @@ -14,17 +14,20 @@ void FargoVelocity(DataBlock &data, IdefixArray2D &Vphi) { // User-defined boundaries -void UserdefBoundary(DataBlock& data, int dir, BoundarySide side, real t) { - +void UserdefBoundary(Hydro *hydro, int dir, BoundarySide side, real t) { + auto *data = hydro->data; if( (dir==IDIR) && (side == left)) { - IdefixArray4D Vc = data.hydro.Vc; - IdefixArray4D Vs = data.hydro.Vs; - IdefixArray1D x1 = data.x[IDIR]; - IdefixArray1D x2 = data.x[JDIR]; + IdefixArray4D Vc = hydro->Vc; + IdefixArray4D Vs = hydro->Vs; + IdefixArray1D x1 = data->x[IDIR]; + IdefixArray1D x2 = data->x[JDIR]; - int ighost = data.nghost[IDIR]; + int ighost = data->nghost[IDIR]; real Omega=1.0; - idefix_for("UserDefBoundary",0,data.np_tot[KDIR],0,data.np_tot[JDIR],0,ighost, + idefix_for("UserDefBoundary", + 0, data->np_tot[KDIR], + 0, data->np_tot[JDIR], + 0, ighost, KOKKOS_LAMBDA (int k, int j, int i) { real R=x1(i)*sin(x2(j)); real z=x1(i)*cos(x2(j)); @@ -37,7 +40,10 @@ void UserdefBoundary(DataBlock& data, int dir, BoundarySide side, real t) { Vc(VX3,k,j,i) = R*Omega; }); - idefix_for("UserDefBoundaryX1S",0,data.np_tot[KDIR],0,data.np_tot[JDIR]+1,0,ighost, + idefix_for("UserDefBoundaryX1S", + 0, data->np_tot[KDIR], + 0, data->np_tot[JDIR]+1, + 0, ighost, KOKKOS_LAMBDA (int k, int j, int i) { Vs(BX2s,k,j,i) = Vs(BX2s,k,j,ighost); @@ -45,7 +51,10 @@ void UserdefBoundary(DataBlock& data, int dir, BoundarySide side, real t) { }); - idefix_for("UserDefBoundaryX3S",0,data.np_tot[KDIR]+1,0,data.np_tot[JDIR],0,ighost, + idefix_for("UserDefBoundaryX3S", + 0, data->np_tot[KDIR]+1, + 0, data->np_tot[JDIR], + 0, ighost, KOKKOS_LAMBDA (int k, int j, int i) { Vs(BX3s,k,j,i) = Vs(BX3s,k,j,ighost); @@ -69,10 +78,10 @@ void Potential(DataBlock& data, const real t, IdefixArray1D& x1, IdefixArr // Arrays or variables which are used later on Setup::Setup(Input &input, Grid &grid, DataBlock &data, Output &output) { // Set the function for userdefboundary - data.hydro.EnrollUserDefBoundary(&UserdefBoundary); - data.gravity.EnrollPotential(&Potential); + data.hydro->EnrollUserDefBoundary(&UserdefBoundary); + data.gravity->EnrollPotential(&Potential); if(data.haveFargo) - data.fargo.EnrollVelocity(&FargoVelocity); + data.fargo->EnrollVelocity(&FargoVelocity); } // This routine initialize the flow diff --git a/test/MHD/FargoMHDSpherical/testme.py b/test/MHD/FargoMHDSpherical/testme.py new file mode 100755 index 00000000..a09d3eeb --- /dev/null +++ b/test/MHD/FargoMHDSpherical/testme.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 + +""" + +@author: glesur +""" +import os +import sys +sys.path.append(os.getenv("IDEFIX_DIR")) + +import pytools.idfx_test as tst + +name="dump.0001.dmp" + +tolerance=1e-14 + +def testMe(test): + test.configure() + test.compile() + inifiles=["idefix.ini"] + + # loop on all the ini files for this test + for ini in inifiles: + test.run(inputFile=ini) + if test.init and not test.mpi: + test.makeReference(filename=name) + test.nonRegressionTest(filename=name,tolerance=tolerance) + + +test=tst.idfxTest() +if not test.dec: + test.dec=['2','2','2'] + +if not test.all: + if(test.check): + test.checkOnly(filename=name,tolerance=tolerance) + else: + testMe(test) +else: + test.noplot = True + + test.vectPot=False + test.single=False + test.reconstruction=2 + test.mpi=False + testMe(test) + + #test vector potential formulation + test.vectPot=True + testMe(test) + + # test with MPI + test.vectPot=False + test.mpi=True + testMe(test) diff --git a/test/MHD/HallDisk/setup.cpp b/test/MHD/HallDisk/setup.cpp index 0400a306..641126ab 100644 --- a/test/MHD/HallDisk/setup.cpp +++ b/test/MHD/HallDisk/setup.cpp @@ -3,14 +3,18 @@ // User-defined boundaries -void UserdefBoundary(DataBlock& data, int dir, BoundarySide side, real t) { +void UserdefBoundary(Hydro *hydro, int dir, BoundarySide side, real t) { + auto *data = hydro->data; if( (dir==IDIR) && (side == left)) { - IdefixArray4D Vc = data.hydro.Vc; - IdefixArray4D Vs = data.hydro.Vs; - IdefixArray1D x1 = data.x[IDIR]; - - int ighost = data.nghost[IDIR]; - idefix_for("UserDefBoundary",0,data.np_tot[KDIR],0,data.np_tot[JDIR],0,ighost, + IdefixArray4D Vc = hydro->Vc; + IdefixArray4D Vs = hydro->Vs; + IdefixArray1D x1 = data->x[IDIR]; + + int ighost = data->nghost[IDIR]; + idefix_for("UserDefBoundary", + 0, data->np_tot[KDIR], + 0, data->np_tot[JDIR], + 0, ighost, KOKKOS_LAMBDA (int k, int j, int i) { Vc(RHO,k,j,i) = Vc(RHO,k,j,ighost); Vc(PRS,k,j,i) = Vc(PRS,k,j,ighost); @@ -50,8 +54,8 @@ void Hall(DataBlock& data, const real t, IdefixArray3D &xH) { // Arrays or variables which are used later on Setup::Setup(Input &input, Grid &grid, DataBlock &data, Output &output) { // Set the function for userdefboundary - data.hydro.EnrollUserDefBoundary(&UserdefBoundary); - data.hydro.EnrollHallDiffusivity(&Hall); + data.hydro->EnrollUserDefBoundary(&UserdefBoundary); + data.hydro->EnrollHallDiffusivity(&Hall); } // This routine initialize the flow diff --git a/test/MHD/HallWhistler/idefix.ini b/test/MHD/HallWhistler/idefix.ini index 3b0302c8..d5ae8c8f 100644 --- a/test/MHD/HallWhistler/idefix.ini +++ b/test/MHD/HallWhistler/idefix.ini @@ -27,3 +27,4 @@ X3-end periodic [Output] log 100 analysis 0.02 +dmp 1.0 diff --git a/test/MHD/HallWhistler/testme.py b/test/MHD/HallWhistler/testme.py new file mode 100755 index 00000000..f8d63384 --- /dev/null +++ b/test/MHD/HallWhistler/testme.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 + +""" + +@author: glesur +""" +import os +import sys +sys.path.append(os.getenv("IDEFIX_DIR")) + +import pytools.idfx_test as tst + +name="dump.0001.dmp" +tolerance=1e-15 +def testMe(test): + test.configure() + test.compile() + inifiles=["idefix.ini"] + + # loop on all the ini files for this test + for ini in inifiles: + test.run(inputFile=ini) + test.standardTest(); + if test.init: + test.makeReference(filename=name) + test.nonRegressionTest(filename=name,tolerance=tolerance) + + +test=tst.idfxTest() + +if not test.all: + if(test.check): + test.checkOnly(filename=name,tolerance=tolerance) + else: + testMe(test) +else: + test.noplot = True + + test.vectPot=False + test.single=False + test.reconstruction=2 + test.mpi=False + testMe(test) diff --git a/test/MHD/LinearWaveTest/python/testidefix.py b/test/MHD/LinearWaveTest/python/testidefix.py index d79977c3..be82269a 100755 --- a/test/MHD/LinearWaveTest/python/testidefix.py +++ b/test/MHD/LinearWaveTest/python/testidefix.py @@ -28,7 +28,7 @@ def getError(rep): return math.nan keylist=[] - keylist=['Vc-RHO','Vc-VX1','Vc-VX2','Vc-VX3','Vc-BX1','Vc-BX2','Vc-BX3','Vc-PRS'] + keylist=['Vc-RHO','Vc-VX1','Vc-VX2','Vc-VX3','Vs-BX1s','Vs-BX2s','Vs-BX3s','Vc-PRS'] err = 0.0 for key in keylist: diff --git a/test/MHD/LinearWaveTest/setup.cpp b/test/MHD/LinearWaveTest/setup.cpp index c747eb5c..06f8a700 100644 --- a/test/MHD/LinearWaveTest/setup.cpp +++ b/test/MHD/LinearWaveTest/setup.cpp @@ -26,7 +26,7 @@ void Setup::InitFlow(DataBlock &data) { real x,y,z; - real gamma = data.hydro.GetGamma(); + real gamma = data.hydro->eos->GetGamma(); // The eigenmode real R[8]; real R0; diff --git a/test/MHD/LinearWaveTest/testme.py b/test/MHD/LinearWaveTest/testme.py new file mode 100755 index 00000000..8c928286 --- /dev/null +++ b/test/MHD/LinearWaveTest/testme.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 + +""" + +@author: glesur +""" +import os +import sys +sys.path.append(os.getenv("IDEFIX_DIR")) + +import pytools.idfx_test as tst +tolerance=1e-14 +def testMe(test): + test.configure() + test.compile() + inifiles=["idefix-fast.ini","idefix-slow.ini","idefix-alfven.ini","idefix-entropy.ini"] + for ini in inifiles: + mytol=tolerance + + test.run(inputFile=ini) + if test.init and not test.mpi: + test.makeReference(filename="dump.0001.dmp") + test.standardTest() + test.nonRegressionTest(filename="dump.0001.dmp",tolerance=mytol) + + +test=tst.idfxTest() +if not test.dec: + test.dec=['2','2','2'] + +if not test.all: + if(test.check): + test.checkOnly(filename="dump.0001.dmp",tolerance=tolerance) + else: + testMe(test) +else: + test.noplot = True + test.single=False + test.reconstruction=2 + test.mpi=False + testMe(test) + + test.mpi=True + testMe(test) + + test.mpi=False + test.reconstruction=3 + testMe(test) + + test.reconstruction=4 + testMe(test) diff --git a/test/MHD/MTI/CMakeLists.txt b/test/MHD/MTI/CMakeLists.txt new file mode 100644 index 00000000..7fcdaa86 --- /dev/null +++ b/test/MHD/MTI/CMakeLists.txt @@ -0,0 +1,4 @@ +set_idefix_property(Idefix_RECONSTRUCTION LimO3) +enable_idefix_property(Idefix_MHD) +add_idefix_source(analysis.hpp) +add_idefix_source(analysis.cpp) diff --git a/test/MHD/MTI/analysis.cpp b/test/MHD/MTI/analysis.cpp new file mode 100644 index 00000000..d0065cf9 --- /dev/null +++ b/test/MHD/MTI/analysis.cpp @@ -0,0 +1,131 @@ +#include +#include + +#include "analysis.hpp" +#include "idefix.hpp" +#include "fluid.hpp" +#include + +Analysis::Analysis(Input &input, Grid &grid, DataBlock &data, Output &output, std::string filename) { + this->d = new DataBlockHost(data); + this->grid = &grid; + this->filename = filename; + this->precision = 10; +} + +/* **************************************************************** */ +double Analysis::Average(const int nfields, int fields[]) + /* + * compute the weighted average: int dphi dz rho *infield/int dphi dz rho + * + **************************************************************** */ +{ + real outfield = 0; + + for(int k = d->beg[KDIR]; k < d->end[KDIR] ; k++) { + for(int j = d->beg[JDIR]; j < d->end[JDIR] ; j++) { + for(int i = d->beg[IDIR]; i < d->end[IDIR] ; i++) { + real q=1.0; + for(int n=0 ; n < nfields ; n++) { + q = q*d->Vc(fields[n],k,j,i); + } + outfield += q; + } + } + } + + // Reduce +#ifdef WITH_MPI + real reducedValue; + MPI_Reduce(&outfield, &reducedValue, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD); + outfield = reducedValue; +#endif + + outfield = outfield / ((double) grid->np_int[IDIR] * grid->np_int[JDIR] * grid->np_int[KDIR]); + + + return outfield; +} + +/* **************************************************************** */ +void Analysis::WriteField(double data) { +/* + * * Write a global profile to a file + * * + * * + * **************************************************************** */ + if(idfx::prank==0) { + int col_width = precision + 10; + this->file << std::scientific << std::setw(col_width) << data; + } + return ; +} + + +void Analysis::ResetAnalysis() { + GridHost gh(*this->grid); + gh.SyncFromDevice(); + int col_width = precision + 10; + if(idfx::prank==0) { + file.open(filename, std::ios::trunc); + file << std::setw(col_width) << "t"; + file << std::setw(col_width) << "vx_2"; + file << std::setw(col_width) << "vy_2"; + file << std::setw(col_width) << "kinx"; + file << std::setw(col_width) << "kiny"; + file << std::setw(col_width) << "bx_2"; + file << std::setw(col_width) << "by_2"; + + file << std::endl; + file.close(); + } +} + +void Analysis::PerformAnalysis(DataBlock &data) { + idfx::pushRegion("Analysis::PerformAnalysis"); + d->SyncFromDevice(); + int fields[3]; + if(idfx::prank==0) { + file.open(filename, std::ios::app); + file.precision(precision); + } + + const int nx0 = data.beg[IDIR] + data.np_int[IDIR]/2; + const int ny0 = data.beg[JDIR] + 3*data.np_int[JDIR]/4; + const int nz0 = data.beg[KDIR]; + + WriteField(data.t); + + fields[0] = VX1; + fields[1] = VX1; + WriteField(Average(2, fields)); + + fields[0] = VX2; + fields[1] = VX2; + WriteField(Average(2, fields)); + + fields[0] = RHO; + fields[1] = VX1; + fields[2] = VX1; + WriteField(HALF_F*Average(3, fields)); + + fields[0] = RHO; + fields[1] = VX2; + fields[2] = VX2; + WriteField(HALF_F*Average(3, fields)); + + fields[0] = BX1; + fields[1] = BX1; + WriteField(Average(2, fields)); + + fields[0] = BX2; + fields[1] = BX2; + WriteField(Average(2, fields)); + + + if(idfx::prank==0) { + file << std::endl; + file.close(); + } + idfx::popRegion(); +} diff --git a/test/MHD/MTI/analysis.hpp b/test/MHD/MTI/analysis.hpp new file mode 100644 index 00000000..9f8b9707 --- /dev/null +++ b/test/MHD/MTI/analysis.hpp @@ -0,0 +1,35 @@ +#ifndef ANALYSIS_HPP_ +#define ANALYSIS_HPP_ + +#include "idefix.hpp" +#include "input.hpp" +#include "output.hpp" +#include "grid.hpp" +#include "dataBlock.hpp" +#include "dataBlockHost.hpp" +#include +#include + + +class Analysis { + public: + // Constructor from Setup arguments + Analysis(Input&, Grid&, DataBlock& , Output&, std::string); + void ResetAnalysis(); + void PerformAnalysis(DataBlock &); + + private: + double Average(const int, int[]); + void WriteField(double); + + DataBlockHost *d; + DataBlock *datain; + Grid *grid; + + int precision; + std::string filename; + + std::ofstream file; +}; + +#endif // ANALYSIS_HPP__ diff --git a/test/MHD/MTI/definitions.hpp b/test/MHD/MTI/definitions.hpp new file mode 100644 index 00000000..69f0c49e --- /dev/null +++ b/test/MHD/MTI/definitions.hpp @@ -0,0 +1,4 @@ +#define COMPONENTS 2 +#define DIMENSIONS 2 + +#define GEOMETRY CARTESIAN diff --git a/test/MHD/MTI/idefix-rkl.ini b/test/MHD/MTI/idefix-rkl.ini new file mode 100644 index 00000000..2eeb7615 --- /dev/null +++ b/test/MHD/MTI/idefix-rkl.ini @@ -0,0 +1,37 @@ +[Grid] +X1-grid 1 0. 32 u 0.1 +X2-grid 1 0. 32 u 0.1 +X3-grid 1 0. 1 u 1. + +[TimeIntegrator] +CFL 0.7 +CFL_max_var 1.1 +tstop 10. +first_dt 1.e-8 +nstages 3 + +[Hydro] +solver hlld +gamma 1.6666666666666666 +bragTDiffusion rkl nolimiter userdef +bragViscosity rkl nolimiter userdef + +[Gravity] +potential userdef + +[Boundary] +X1-beg periodic +X1-end periodic +X2-beg userdef +X2-end userdef +X3-beg userdef +X3-end userdef + +[Setup] +ksi 5e-4 +pr 0.06 +fromDump true + +[Output] +analysis 0.5 +dmp 10. diff --git a/test/MHD/MTI/idefix-sl.ini b/test/MHD/MTI/idefix-sl.ini new file mode 100644 index 00000000..de020034 --- /dev/null +++ b/test/MHD/MTI/idefix-sl.ini @@ -0,0 +1,37 @@ +[Grid] +X1-grid 1 0. 32 u 0.1 +X2-grid 1 0. 32 u 0.1 +X3-grid 1 0. 1 u 1. + +[TimeIntegrator] +CFL 0.7 +CFL_max_var 1.1 +tstop 10. +first_dt 1.e-8 +nstages 3 + +[Hydro] +solver hlld +gamma 1.6666666666666666 +bragTDiffusion rkl mc userdef +bragViscosity rkl vanleer userdef + +[Gravity] +potential userdef + +[Boundary] +X1-beg periodic +X1-end periodic +X2-beg userdef +X2-end userdef +X3-beg userdef +X3-end userdef + +[Setup] +ksi 5e-4 +pr 0.06 +fromDump true + +[Output] +analysis 0.5 +dmp 10. diff --git a/test/MHD/MTI/idefix.ini b/test/MHD/MTI/idefix.ini new file mode 100644 index 00000000..b54bedb0 --- /dev/null +++ b/test/MHD/MTI/idefix.ini @@ -0,0 +1,37 @@ +[Grid] +X1-grid 1 0. 32 u 0.1 +X2-grid 1 0. 32 u 0.1 +X3-grid 1 0. 1 u 1. + +[TimeIntegrator] +CFL 0.7 +CFL_max_var 1.1 +tstop 10. +first_dt 1.e-8 +nstages 3 + +[Hydro] +solver hlld +gamma 1.6666666666666666 +bragTDiffusion explicit nolimiter userdef +bragViscosity explicit nolimiter userdef + +[Gravity] +potential userdef + +[Boundary] +X1-beg periodic +X1-end periodic +X2-beg userdef +X2-end userdef +X3-beg userdef +X3-end userdef + +[Setup] +ksi 5e-4 +pr 0.06 +fromDump true + +[Output] +analysis 0.5 +dmp 10.0 diff --git a/test/MHD/MTI/init.dmp b/test/MHD/MTI/init.dmp new file mode 100644 index 00000000..d584392e Binary files /dev/null and b/test/MHD/MTI/init.dmp differ diff --git a/test/MHD/MTI/python/testidefix.py b/test/MHD/MTI/python/testidefix.py new file mode 100644 index 00000000..4b60cc8b --- /dev/null +++ b/test/MHD/MTI/python/testidefix.py @@ -0,0 +1,74 @@ +""" +Created in July 2021 + +@author: Jean Kempf, Francois Rincon + +The test is inspired from the following paper: +Parrish, Ian J., et al. "The effects of anisotropic viscosity on turbulence and heat transport in the intracluster medium." Monthly Notices of the Royal Astronomical Society 422.1 (2012): 704-718. +""" + +import os +import sys +sys.path.append(os.getenv("IDEFIX_DIR")) +import numpy as np +import inifix + +n = 1 # fondamental mode + +conf = inifix.load("../idefix.ini") +Pr=conf["Setup"]["pr"] +ksi=conf["Setup"]["ksi"] + +L = 0.1 +H = 3. +wbuoy = 1/np.sqrt(3) +gamma = 5/3 + +kn = 2*np.pi*n*np.sqrt(2)/L + +fid=open('../average.dat',"r") +# read the first line to get data names +varnames=fid.readline().split() +fid.close() +# load the bulk of the file +data=np.loadtxt('../average.dat',skiprows=1) +# store this in our data structure +V={} +i=0 +for name in varnames: + V[name]=data[:,i] + i=i+1 + +sigma_vx = 0.5*(np.log(V['kinx'][20]) - np.log(V['kinx'][15]))/(V['t'][20] - V['t'][15]) +sigma_vy = 0.5*(np.log(V['kiny'][20]) - np.log(V['kiny'][15]))/(V['t'][20] - V['t'][15]) +sigma_by = 0.5*(np.log(V['by_2'][20]) - np.log(V['by_2'][15]))/(V['t'][20] - V['t'][15]) +sigma_num = (sigma_vx + sigma_vy + sigma_by)/3. + +def dispMTI(wcond, k, Pr): + cosk_2 = 0.5 # here cosk_2 = sqrt(bhat) dot khat i.e. xhat dot khat + k_2 = k**2 + ksi = 2.5/k_2/cosk_2*wcond + kx_2 = k_2*cosk_2 + K = kx_2/k_2 + + bk_2 = k_2*cosk_2 + bkhat_2 = cosk_2 + nu = Pr*ksi + wvisc = 3*nu*bk_2 + v = wvisc*(1 - bkhat_2) + N_2 = wbuoy**2/gamma*((H-1)*gamma-H) + return np.array([1, wcond+v, wcond*v+N_2*kx_2/k_2, K*wcond*-wbuoy**2]) + +def sigmaMTI(wcond, k, Pr): + roots = np.roots(dispMTI(wcond,k,Pr)) + return np.real(roots[np.real(roots) > 0])[0] + +wcd = 0.4*ksi*kn**2/2 +sigma_analytic = sigmaMTI(wcd, kn, Pr) + +if np.abs(np.log(sigma_analytic/sigma_num)/np.log(sigma_num)) >= 0.1: + print("Failed") + sys.exit(1) +else: + print("SUCCESS") + sys.exit(0) diff --git a/test/MHD/MTI/setup.cpp b/test/MHD/MTI/setup.cpp new file mode 100644 index 00000000..e631f7b2 --- /dev/null +++ b/test/MHD/MTI/setup.cpp @@ -0,0 +1,179 @@ +#include "idefix.hpp" +#include "setup.hpp" +#include "analysis.hpp" +#include "dumpImage.hpp" + +// Definition of the constants and parameters of the problem +const real T0 = 1.; +const real H = 3.; +const real L = 0.1; +const real g0 = 1.; + +const real beta0 = 1e12; +const real rho0 = 1.; //initial density +const real P0 = rho0*T0; //initial pressure +const real B0 = 100.*sqrt(1./beta0); + +const real vth0 = 1.; + +const real n = 1.; +const real kn = 2.*n*M_PI/L; +static real ksiGlob; +static real prGlob; + +Analysis *analysis; + +bool fromDump; + +void AnalysisFunction(DataBlock &data) { + analysis->PerformAnalysis(data); +} + +// Implement uniform gravitationnal field +void Potential(DataBlock& data, const real t, IdefixArray1D& x1, IdefixArray1D& x2, IdefixArray1D& x3, IdefixArray3D& phi) { + real g0 = 1.; + idefix_for("Potential", 0, data.np_tot[KDIR], 0, data.np_tot[JDIR], 0, data.np_tot[IDIR], + KOKKOS_LAMBDA (int k, int j, int i) { + phi(k,j,i) = g0*x2(j); + }); +} + + +void MyBragThermalConductivity(DataBlock &data, const real t, IdefixArray3D &kparArr, IdefixArray3D &knorArr) { + IdefixArray4D Vc = data.hydro->Vc; + IdefixArray1D x2 = data.x[JDIR]; + real ksi = ksiGlob; + idefix_for("MyThConductivity",0,data.np_tot[KDIR],0,data.np_tot[JDIR],0,data.np_tot[IDIR], + KOKKOS_LAMBDA (int k, int j, int i) { + kparArr(k,j,i) = ksi*Vc(RHO,k,j,i); + knorArr(k,j,i) = 0.; + }); +} + +void MyViscosity(DataBlock &data, const real t, IdefixArray3D &etaBrag) { + IdefixArray1D x1 = data.x[IDIR]; + IdefixArray1D x2 = data.x[JDIR]; + real ksi = ksiGlob; + real pr = prGlob; + IdefixArray4D Vc = data.hydro->Vc; + idefix_for("MyViscosity",0,data.np_tot[KDIR],0,data.np_tot[JDIR],0,data.np_tot[IDIR], + KOKKOS_LAMBDA (int k, int j, int i) { + etaBrag(k,j,i) = ksi*pr*Vc(RHO,k,j,i); + }); +} + +// Define our own boundary conditions. Basically, homogene Neummann (i.e. symmetry) on polar and azimuthal velocity, Dirichlet on the thermo fields. +void UserDefBoundary(Hydro *hydro, int dir, BoundarySide side, real t) { + auto *data = hydro->data; + real T0 = 1.; + real H = 3.; + real g0 = 1.; + real rho0 = 1.; //initial density + real P0 = rho0*T0; //initial pressure + IdefixArray4D Vc = hydro->Vc; + IdefixArray4D Vs = hydro->Vs; + IdefixArray1D x1 = data->x[IDIR]; + IdefixArray1D x2 = data->x[JDIR]; + int jbeg = (side == left) ? 0 : data->end[JDIR]; + int jend = (side == left) ? data->beg[JDIR] : data->np_tot[JDIR]; + hydro->boundary->BoundaryFor("UserDefBoundary", dir, side, + KOKKOS_LAMBDA (int k, int j, int i) { + int jref = (side == left) ? 2*jend - j -1 : jbeg + jend -j - 3; + Vc(PRS,k,j,i) = P0*pow(1.-x2(j)/H, H*g0/T0); + Vc(RHO,k,j,i) =rho0*pow(1.-x2(j)/H, H*g0/T0-1.) - (Vc(RHO,k,jref,i) - rho0*pow(1.-x2(jref)/H, H*g0/T0-1.)); + + Vc(VX1,k,j,i) = Vc(VX1,k,jref,i); + Vc(VX2,k,j,i) = -Vc(VX2,k,jref,i); + }); + + hydro->boundary->BoundaryForX1s("UserDefBoundaryBX1s", dir, side, + KOKKOS_LAMBDA (int k, int j, int i) { + //int jref = (side == left) ? 2*jend - j -1 : jbeg + jend -j - 3; + int jref = (side == left) ? 2*jend - j -1 : 2*jbeg -j -1; + Vs(BX1s,k,j,i) = Vs(BX1s,k,jref,i); + }); +} + +// Initialisation routine. Can be used to allocate Arrays or variables which are used later on +Setup::Setup(Input &input, Grid &grid, DataBlock &data, Output &output) { + analysis = new Analysis(input, grid, data, output,std::string("average.dat")); + output.EnrollAnalysis(&AnalysisFunction); + // Reset analysis if required + if(!input.restartRequested) { + analysis->ResetAnalysis(); + } + + data.gravity->EnrollPotential(&Potential); + data.hydro->EnrollUserDefBoundary(&UserDefBoundary); + data.hydro->bragThermalDiffusion->EnrollBragThermalDiffusivity(&MyBragThermalConductivity); + data.hydro->bragViscosity->EnrollBragViscousDiffusivity(&MyViscosity); + ksiGlob = input.Get("Setup","ksi",0); + prGlob = input.Get("Setup","pr",0); + fromDump = input.GetOrSet("Setup","fromDump",0,false); // Whether we build our own initial condition or we use the pre-defined one +} + +// This routine initializes the flow +// Note that data is on the device. +// One can therefore define locally +// a datahost and sync it, if needed +void Setup::InitFlow(DataBlock &data) { + // Create a host copy + DataBlockHost d(data); + + if(fromDump) { + DumpImage image("init.dmp",&data); + + for(int k = d.beg[KDIR]; k < d.end[KDIR] ; k++) { + for(int j = d.beg[JDIR]; j < d.end[JDIR] ; j++) { + for(int i = d.beg[IDIR]; i < d.end[IDIR] ; i++) { + + // Note that the restart dump array only contains the full (global) active domain + // (i.e. it excludes the boundaries, but it is not decomposed accross MPI procs) + int iglob=i-2*d.beg[IDIR]+d.gbeg[IDIR]; + int jglob=j-2*d.beg[JDIR]+d.gbeg[JDIR]; + int kglob=k-2*d.beg[KDIR]+d.gbeg[KDIR]; + + d.Vc(RHO,k,j,i) = image.arrays["Vc-RHO"](kglob,jglob,iglob); + d.Vc(PRS,k,j,i) = image.arrays["Vc-PRS"](kglob,jglob,iglob); + d.Vc(VX1,k,j,i) = image.arrays["Vc-VX1"](kglob,jglob,iglob); + d.Vc(VX2,k,j,i) = image.arrays["Vc-VX2"](kglob,jglob,iglob); + }}} + for(int k = d.beg[KDIR]; k < d.end[KDIR] ; k++) { + for(int j = d.beg[JDIR]; j < d.end[JDIR] ; j++) { + for(int i = d.beg[IDIR]; i < d.end[IDIR]+1 ; i++) { + int iglob=i-2*d.beg[IDIR]+d.gbeg[IDIR]; + int jglob=j-2*d.beg[JDIR]+d.gbeg[JDIR]; + int kglob=k-2*d.beg[KDIR]+d.gbeg[KDIR]; + d.Vs(BX1s,k,j,i) = image.arrays["Vs-BX1s"](kglob,jglob,iglob); + }}} + for(int k = d.beg[KDIR]; k < d.end[KDIR] ; k++) { + for(int j = d.beg[JDIR]; j < d.end[JDIR]+1 ; j++) { + for(int i = d.beg[IDIR]; i < d.end[IDIR] ; i++) { + int iglob=i-2*d.beg[IDIR]+d.gbeg[IDIR]; + int jglob=j-2*d.beg[JDIR]+d.gbeg[JDIR]; + int kglob=k-2*d.beg[KDIR]+d.gbeg[KDIR]; + d.Vs(BX2s,k,j,i) = image.arrays["Vs-BX2s"](kglob,jglob,iglob); + }}} + } else { + for(int k = 0; k < d.np_tot[KDIR] ; k++) { + for(int j = 0; j < d.np_tot[JDIR] ; j++) { + real x2 = d.x[JDIR](j); + real x2s = d.xl[JDIR](j); + for(int i = 0; i < d.np_tot[IDIR] ; i++) { + real x1=d.x[IDIR](i); + real x1s=d.xl[IDIR](i); + + d.Vc(PRS,k,j,i) = P0*pow(1.-x2/H, H*g0/T0); + d.Vc(RHO,k,j,i) = rho0*pow(1.-x2/H, H*g0/T0-1.); + + d.Vc(VX1,k,j,i) = -1e-4*std::sin(kn*x1)*std::cos(kn*x2)*vth0; + d.Vc(VX2,k,j,i) = 1e-4*std::cos(kn*x1)*std::sin(kn*x2)*vth0; + + d.Vs(BX1s,k,j,i) = B0; + d.Vs(BX2s,k,j,i) = 0.; + } + } + } + } + d.SyncToDevice(); +} diff --git a/test/MHD/MTI/testme.py b/test/MHD/MTI/testme.py new file mode 100755 index 00000000..7975c807 --- /dev/null +++ b/test/MHD/MTI/testme.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 + +""" + +@author: glesur +""" +import os +import sys +sys.path.append(os.getenv("IDEFIX_DIR")) + +import pytools.idfx_test as tst + + +def testMe(test): + test.configure() + test.compile() + inifiles=["idefix.ini","idefix-rkl.ini","idefix-sl.ini"] +# inifiles=["idefix-rkl.ini","idefix-sl.ini"] + + # loop on all the ini files for this test + name="dump.0001.dmp" + for ini in inifiles: + test.run(inputFile=ini) + test.standardTest() + if test.init: + test.makeReference(filename=name) + test.nonRegressionTest(filename=name) + + +test=tst.idfxTest() + +if not test.all: + if(test.check): + test.checkOnly(filename="dump.0001.dmp") + else: + testMe(test) +else: + test.noplot = True + test.vectPot=False + test.single=False + test.reconstruction=2 + test.mpi=False + testMe(test) diff --git a/test/MHD/OrszagTang/python/data.0001.ref.vtk b/test/MHD/OrszagTang/python/data.0001.ref.vtk deleted file mode 100644 index 2a7df718..00000000 --- a/test/MHD/OrszagTang/python/data.0001.ref.vtk +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:5c3831b86b830646fedc4c8fe6e31e1dff6efa95beb7138bf39f33ff8d7432c6 -size 394723 diff --git a/test/MHD/OrszagTang/python/testidefix.py b/test/MHD/OrszagTang/python/testidefix.py deleted file mode 100755 index 139a1edb..00000000 --- a/test/MHD/OrszagTang/python/testidefix.py +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -""" -Created on Thu Mar 5 11:29:41 2020 - -@author: glesur -""" - -import os -import sys -sys.path.append(os.getenv("IDEFIX_DIR")) -from pytools.vtk_io import readVTK -import numpy as np - -V=readVTK('../data.0001.vtk', geometry='cartesian') -U=readVTK('data.0001.ref.vtk', geometry='cartesian') - -# Compute the error on PRS -error=np.mean(np.abs(V.data['PRS']-U.data['prs'])/U.data['prs'],axis=(0,1)) - -print("Error=%e"%error) -if error<5.45e-2: - print("SUCCESS!") - sys.exit(0) -else: - print("FAILURE!") - sys.exit(1) diff --git a/test/MHD/OrszagTang/setup.cpp b/test/MHD/OrszagTang/setup.cpp index edacd671..50c981c1 100644 --- a/test/MHD/OrszagTang/setup.cpp +++ b/test/MHD/OrszagTang/setup.cpp @@ -12,7 +12,6 @@ generators on different architectures. // Default constructor - // Initialisation routine. Can be used to allocate // Arrays or variables which are used later on Setup::Setup(Input &input, Grid &grid, DataBlock &data, Output &output) { @@ -27,6 +26,8 @@ void Setup::InitFlow(DataBlock &data) { // Create a host copy DataBlockHost d(data); + bool haveTracer = data.hydro->haveTracer; + real B0=1.0/sqrt(4.0*M_PI); for(int k = 0; k < d.np_tot[KDIR] ; k++) { @@ -48,6 +49,9 @@ void Setup::InitFlow(DataBlock &data) { d.Vs(BX1s,k,j,i) = -B0*sin(2.0*M_PI*y); d.Vs(BX2s,k,j,i) = B0*sin(4.0*M_PI*x); #endif + if(haveTracer) { + d.Vc(TRG,k,j,i) = x>0.5? 1.0:0.0; + } } } diff --git a/test/MHD/OrszagTang/testme.py b/test/MHD/OrszagTang/testme.py new file mode 100755 index 00000000..761505ca --- /dev/null +++ b/test/MHD/OrszagTang/testme.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 + +""" + +@author: glesur +""" +import os +import sys +sys.path.append(os.getenv("IDEFIX_DIR")) + +import pytools.idfx_test as tst +tolerance=1e-12 +def testMe(test): + test.configure() + test.compile() + inifiles=["idefix.ini", + "idefix-hll.ini","idefix-hlld-arithmetic.ini", + "idefix-hlld-hll.ini", + "idefix-hlld-hlld.ini", + "idefix-hlld-uct0.ini", + "idefix-hlld.ini","idefix-tvdlf.ini"] + + for ini in inifiles: + mytol=tolerance + + test.run(inputFile=ini) + if test.init and not test.mpi: + test.makeReference(filename="dump.0001.dmp") + + if(test.single): + mytol=1e-5 + + test.nonRegressionTest(filename="dump.0001.dmp",tolerance=mytol) + + +test=tst.idfxTest() +if not test.dec: + test.dec=['2','2'] + +if not test.all: + if(test.check): + test.checkOnly(filename="dump.0001.dmp",tolerance=tolerance) + else: + testMe(test) +else: + for rec in [2,3,4]: + test.noplot = True + test.vectPot = False + test.single=False + test.reconstruction=rec + test.mpi=False + testMe(test) + test.mpi=True + testMe(test) + + # single precision validation + test.reconstruction=2 + test.mpi=False + test.single=True + testMe(test) + + # Vector potential validation + test.single=False + test.mpi=False + test.vectPot=True + testMe(test) diff --git a/test/MHD/OrszagTang3D/definitions.hpp b/test/MHD/OrszagTang3D/definitions.hpp index 3b581bd6..a854ad9e 100644 --- a/test/MHD/OrszagTang3D/definitions.hpp +++ b/test/MHD/OrszagTang3D/definitions.hpp @@ -1,4 +1,5 @@ #define COMPONENTS 3 #define DIMENSIONS 3 +//#define DEBUG #define GEOMETRY CARTESIAN diff --git a/test/MHD/OrszagTang3D/idefix-checkrestart.ini b/test/MHD/OrszagTang3D/idefix-checkrestart.ini index 816a86b8..f727322f 100644 --- a/test/MHD/OrszagTang3D/idefix-checkrestart.ini +++ b/test/MHD/OrszagTang3D/idefix-checkrestart.ini @@ -11,6 +11,7 @@ nstages 2 [Hydro] solver hlld +tracer 2 [Boundary] X1-beg periodic diff --git a/test/MHD/OrszagTang3D/idefix.ini b/test/MHD/OrszagTang3D/idefix.ini index 01d5fe14..1021bd33 100644 --- a/test/MHD/OrszagTang3D/idefix.ini +++ b/test/MHD/OrszagTang3D/idefix.ini @@ -11,6 +11,7 @@ nstages 2 [Hydro] solver hlld +tracer 2 [Boundary] X1-beg periodic @@ -22,4 +23,5 @@ X3-end periodic [Output] vtk 0.2 +dmp 0.2 log 10 diff --git a/test/MHD/OrszagTang3D/python/data.0001.ref.vtk b/test/MHD/OrszagTang3D/python/data.0001.ref.vtk deleted file mode 100644 index 11da857f..00000000 --- a/test/MHD/OrszagTang3D/python/data.0001.ref.vtk +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:c871e5332800e647a172673bef646735bcf0cc85dc0ae59c2d67130cf88ea601 -size 2098291 diff --git a/test/MHD/OrszagTang3D/python/testidefix.py b/test/MHD/OrszagTang3D/python/testidefix.py deleted file mode 100755 index f7e2af51..00000000 --- a/test/MHD/OrszagTang3D/python/testidefix.py +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -""" -Created on Thu Mar 5 11:29:41 2020 - -@author: glesur -""" - -import os -import sys -sys.path.append(os.getenv("IDEFIX_DIR")) -from pytools.vtk_io import readVTK -import numpy as np - -V=readVTK('../data.0001.vtk', geometry='cartesian') -U=readVTK('data.0001.ref.vtk', geometry='cartesian') - -# Compute the error on PRS -error=np.mean(np.abs(V.data['PRS']-U.data['PRS']),axis=(0,1,2)) - -print("Error=%e"%error) -if error<4.881e-3: - print("SUCCESS!") - sys.exit(0) -else: - print("FAILURE!") - sys.exit(1) diff --git a/test/MHD/OrszagTang3D/setup.cpp b/test/MHD/OrszagTang3D/setup.cpp index 89bcef45..8e7205c4 100644 --- a/test/MHD/OrszagTang3D/setup.cpp +++ b/test/MHD/OrszagTang3D/setup.cpp @@ -15,25 +15,32 @@ int outnum; // This analysis checks that the restart routines are performing as they should void Analysis(DataBlock& data) { - // Mirror data on Host + idfx::cout << "Analysis: Checking restart routines" << std::endl; + + // Trigger dump creation + myOutput->ForceWriteDump(data); + + // Mirror data on Host DataBlockHost d(data); // Sync it d.SyncFromDevice(); // Create local arrays to store the current physical state - IdefixHostArray4D myVc = IdefixHostArray4D("myVc", NVAR, data.np_tot[KDIR], data.np_tot[JDIR],data.np_tot[IDIR]); + IdefixHostArray4D myVc = IdefixHostArray4D("myVc", d.Vc.extent(0), data.np_tot[KDIR], data.np_tot[JDIR],data.np_tot[IDIR]); IdefixHostArray4D myVs = IdefixHostArray4D("myVs", DIMENSIONS, data.np_tot[KDIR]+KOFFSET, data.np_tot[JDIR]+JOFFSET,data.np_tot[IDIR]+IOFFSET); #ifdef EVOLVE_VECTOR_POTENTIAL IdefixHostArray4D myVe = IdefixHostArray4D("myVe", AX3e+1, data.np_tot[KDIR]+KOFFSET, data.np_tot[JDIR]+JOFFSET,data.np_tot[IDIR]+IOFFSET); #endif // Transfer the datablock to myVc and myVs - for(int n = 0; n < NVAR ; n++) { + for(int n = 0; n < d.Vc.extent(0) ; n++) { for(int k = 0; k < d.np_tot[KDIR] ; k++) { for(int j = 0; j < d.np_tot[JDIR] ; j++) { for(int i = 0; i < d.np_tot[IDIR] ; i++) { myVc(n,k,j,i) = d.Vc(n,k,j,i); + d.Vc(n,k,j,i) = 0.0; + } } } @@ -44,6 +51,7 @@ void Analysis(DataBlock& data) { for(int j = 0; j < d.np_tot[JDIR] + JOFFSET; j++) { for(int i = 0; i < d.np_tot[IDIR] + IOFFSET; i++) { myVs(n,k,j,i) = d.Vs(n,k,j,i); + d.Vs(n,k,j,i) = 0.0; } } } @@ -54,19 +62,22 @@ void Analysis(DataBlock& data) { for(int j = 0; j < d.np_tot[JDIR] + JOFFSET; j++) { for(int i = 0; i < d.np_tot[IDIR] + IOFFSET; i++) { myVe(n,k,j,i) = d.Ve(n,k,j,i); + d.Ve(n,k,j,i) = 0.0; } } } } #endif - // Trigger dump creation - myOutput->ForceWriteDump(data); + // Push our datablockHost to erase everything + d.SyncToDevice(); + // From this point, the dataBlock is full of zeros // Load back the restart dump myOutput->RestartFromDump(data, outnum); + data.SetBoundaries(); #ifdef EVOLVE_VECTOR_POTENTIAL - data.hydro.emf.ComputeMagFieldFromA(data.hydro.Ve, data.hydro.Vs); + data.hydro->emf->ComputeMagFieldFromA(data.hydro->Ve, data.hydro->Vs); #endif d.SyncFromDevice(); @@ -77,7 +88,7 @@ void Analysis(DataBlock& data) { errornum = 0; idfx::cout << "Analysis: checking consistency" << std::endl; // Check that the save/load routines have left everything unchanged. - for(int n = 0; n < NVAR ; n++) { + for(int n = 0; n < d.Vc.extent(0) ; n++) { for(int k = d.beg[KDIR]; k < d.end[KDIR] ; k++) { for(int j = d.beg[JDIR]; j < d.end[JDIR] ; j++) { for(int i = d.beg[IDIR]; i < d.end[IDIR] ; i++) { @@ -203,6 +214,15 @@ void Setup::InitFlow(DataBlock &data) { // Create a host copy DataBlockHost d(data); real x,y,z; + IdefixHostArray4D Ve; + + #ifndef EVOLVE_VECTOR_POTENTIAL + Ve = IdefixHostArray4D("Potential vector",3, d.np_tot[KDIR]+1, d.np_tot[JDIR]+1, d.np_tot[IDIR]+1); + #else + Ve = d.Ve; + #endif + + bool haveTracer = data.hydro->haveTracer; real B0=1.0/sqrt(4.0*M_PI); @@ -218,24 +238,26 @@ void Setup::InitFlow(DataBlock &data) { d.Vc(VX1,k,j,i) = -sin(2.0*M_PI*y); d.Vc(VX2,k,j,i) = sin(2.0*M_PI*x)+cos(2.0*M_PI*z); d.Vc(VX3,k,j,i) = cos(2.0*M_PI*x); - #ifdef EVOLVE_VECTOR_POTENTIAL - real xl=d.xl[IDIR](i); - real yl=d.xl[JDIR](j); - real zl=d.xl[KDIR](k); - d.Ve(AX1e,k,j,i) = B0/(2.0*M_PI)*(cos(2.0*M_PI*yl)); - d.Ve(AX2e,k,j,i) = B0/(2.0*M_PI)*sin(2.0*M_PI*xl); - d.Ve(AX3e,k,j,i) = B0/(2.0*M_PI)*( - cos(2.0*M_PI*yl) + cos(4.0*M_PI*xl)/2.0); - #else - d.Vs(BX1s,k,j,i) = -B0*sin(2.0*M_PI*y); - d.Vs(BX2s,k,j,i) = B0*sin(4.0*M_PI*x); - d.Vs(BX3s,k,j,i) = B0*(cos(2.0*M_PI*x)+sin(2.0*M_PI*y)); - #endif + real xl=d.xl[IDIR](i); + real yl=d.xl[JDIR](j); + real zl=d.xl[KDIR](k); + Ve(IDIR,k,j,i) = B0/(2.0*M_PI)*(cos(2.0*M_PI*yl)); + Ve(JDIR,k,j,i) = B0/(2.0*M_PI)*sin(2.0*M_PI*xl); + Ve(KDIR,k,j,i) = B0/(2.0*M_PI)*( + cos(2.0*M_PI*yl) + cos(4.0*M_PI*xl)/2.0); + + if(haveTracer) { + d.Vc(TRG ,k,j,i) = x>0.5? 1.0:0.0; + d.Vc(TRG+1,k,j,i) = z>0.5? 1.0:0.0; + } } } } + #ifndef EVOLVE_VECTOR_POTENTIAL + d.MakeVsFromAmag(Ve); + #endif // Send it all, if needed d.SyncToDevice(); } diff --git a/test/MHD/OrszagTang3D/testme.py b/test/MHD/OrszagTang3D/testme.py new file mode 100755 index 00000000..600a5ef5 --- /dev/null +++ b/test/MHD/OrszagTang3D/testme.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 +""" + +@author: glesur +""" +import os +import sys +sys.path.append(os.getenv("IDEFIX_DIR")) + +import pytools.idfx_test as tst + +# Whether we should reset our reference run (only do that on purpose!) + +tolerance=1e-13 + +def testMe(test): + test.configure() + test.compile() + tol=tolerance + if test.single: + tol=1e-6 + + # default with idefix.ini + test.run() + if test.init: + if not test.mpi: + test.makeReference(filename="dump.0001.dmp") + test.nonRegressionTest(filename="dump.0001.dmp",tolerance=tol) + + # Check restarts + test.run("idefix-checkrestart.ini") + #force override the inputfile since the result should be identical + test.inifile="idefix.ini" + test.nonRegressionTest(filename="dump.0002.dmp",tolerance=tol) + + +test=tst.idfxTest() + +# if no decomposition is specified, use that one +if not test.dec: + test.dec=["2","2","2"] + +if not test.all: + if(test.check): + test.checkOnly(filename="dump.0001.dmp",tolerance=tolerance) + else: + testMe(test) +else: + test.vectPot=False + test.reconstruction=2 + test.mpi=False + testMe(test) + # test in MPI mode + test.mpi=True + testMe(test) + + + # test with vector potential + test.mpi=False + test.vectPot=True + test.reconstruction=2 + testMe(test) + + test.mpi=True + testMe(test) + + # test with other precision + test.single=True + test.vectPot=False + test.reconstruction=2 + test.mpi=False + testMe(test) + # test in MPI mode + test.mpi=True + testMe(test) diff --git a/test/MHD/ResistiveAlfvenWave/idefix-rkl.ini b/test/MHD/ResistiveAlfvenWave/idefix-rkl.ini index fe2e055f..29299b06 100644 --- a/test/MHD/ResistiveAlfvenWave/idefix-rkl.ini +++ b/test/MHD/ResistiveAlfvenWave/idefix-rkl.ini @@ -24,4 +24,5 @@ X3-end periodic [Output] # vtk 0.1 log 1000 +dmp 10.0 analysis 0.01 diff --git a/test/MHD/ResistiveAlfvenWave/idefix.ini b/test/MHD/ResistiveAlfvenWave/idefix.ini index ed95489e..60873036 100644 --- a/test/MHD/ResistiveAlfvenWave/idefix.ini +++ b/test/MHD/ResistiveAlfvenWave/idefix.ini @@ -24,4 +24,5 @@ X3-end periodic [Output] # vtk 0.1 log 1000 +dmp 10.0 analysis 0.01 diff --git a/test/MHD/ResistiveAlfvenWave/setup.cpp b/test/MHD/ResistiveAlfvenWave/setup.cpp index 3f92fc0a..7b7828dc 100644 --- a/test/MHD/ResistiveAlfvenWave/setup.cpp +++ b/test/MHD/ResistiveAlfvenWave/setup.cpp @@ -6,7 +6,7 @@ // Analyse data to produce an output void Analysis(DataBlock & data) { double etot = 0; - IdefixArray4D Vc = data.hydro.Vc; + IdefixArray4D Vc = data.hydro->Vc; idefix_reduce("Analysis", data.beg[KDIR],data.end[KDIR], data.beg[JDIR],data.end[JDIR], diff --git a/test/MHD/ResistiveAlfvenWave/testme.py b/test/MHD/ResistiveAlfvenWave/testme.py new file mode 100755 index 00000000..dde3d7c6 --- /dev/null +++ b/test/MHD/ResistiveAlfvenWave/testme.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 + +""" + +@author: glesur +""" +import os +import sys +sys.path.append(os.getenv("IDEFIX_DIR")) + +import pytools.idfx_test as tst +tolerance=1e-14 + +def testMe(test): + test.configure() + test.compile() + inifiles=["idefix.ini","idefix-rkl.ini"] + for ini in inifiles: + test.run(inputFile=ini) + if test.init and not test.mpi: + test.makeReference(filename="dump.0001.dmp") + test.standardTest() + mytol=tolerance + if ini=="idefix-rkl.ini": + mytol=1e-10 + test.nonRegressionTest(filename="dump.0001.dmp",tolerance=mytol) + + +test=tst.idfxTest() + +if not test.all: + if(test.check): + test.checkOnly(filename="dump.0001.dmp",tolerance=tolerance) + else: + testMe(test) +else: + test.noplot = True + test.single=False + test.reconstruction=2 + test.mpi=False + testMe(test) diff --git a/test/MHD/RotorPolar/setup.cpp b/test/MHD/RotorPolar/setup.cpp index 365696f8..7e0d910a 100644 --- a/test/MHD/RotorPolar/setup.cpp +++ b/test/MHD/RotorPolar/setup.cpp @@ -10,14 +10,18 @@ generators on different architectures. /*********************************************/ // User-defined boundaries -void UserdefBoundary(DataBlock& data, int dir, BoundarySide side, real t) { +void UserdefBoundary(Hydro *hydro, int dir, BoundarySide side, real t) { + auto *data = hydro->data; if( (dir==IDIR) && (side == left)) { - IdefixArray4D Vc = data.hydro.Vc; - IdefixArray4D Vs = data.hydro.Vs; - IdefixArray1D x1 = data.x[IDIR]; - - int ighost = data.nghost[IDIR]; - idefix_for("UserDefBoundary",0,data.np_tot[KDIR],0,data.np_tot[JDIR],0,ighost, + IdefixArray4D Vc = hydro->Vc; + IdefixArray4D Vs = hydro->Vs; + IdefixArray1D x1 = data->x[IDIR]; + + int ighost = data->nghost[IDIR]; + idefix_for("UserDefBoundary", + 0, data->np_tot[KDIR], + 0, data->np_tot[JDIR], + 0, ighost, KOKKOS_LAMBDA (int k, int j, int i) { Vc(RHO,k,j,i) = Vc(RHO,k,j,ighost); Vc(PRS,k,j,i) = Vc(PRS,k,j,ighost); @@ -60,7 +64,7 @@ void ComputeUserVars(DataBlock & data, UserDefVariablesContainer &variables) { // Arrays or variables which are used later on Setup::Setup(Input &input, Grid &grid, DataBlock &data, Output &output) { // Set the function for userdefboundary - data.hydro.EnrollUserDefBoundary(&UserdefBoundary); + data.hydro->EnrollUserDefBoundary(&UserdefBoundary); output.EnrollUserDefVariables(&ComputeUserVars); } diff --git a/test/MHD/ShearingBox/analysis.cpp b/test/MHD/ShearingBox/analysis.cpp index e3799db3..95102b99 100644 --- a/test/MHD/ShearingBox/analysis.cpp +++ b/test/MHD/ShearingBox/analysis.cpp @@ -1,5 +1,6 @@ #include "analysis.hpp" #include "idefix.hpp" +#include "fluid.hpp" #include #include #include @@ -8,7 +9,7 @@ Analysis::Analysis(Input &input, Grid &grid, DataBlock &data, Output &output, st this->d = new DataBlockHost(data); this->grid = &grid; this->filename = filename; - this->shear = data.hydro.sbS; + this->shear = data.hydro->sbS; this->precision = 10; } diff --git a/test/MHD/ShearingBox/idefix-fargo.ini b/test/MHD/ShearingBox/idefix-fargo.ini index 3481825a..22ea5a97 100644 --- a/test/MHD/ShearingBox/idefix-fargo.ini +++ b/test/MHD/ShearingBox/idefix-fargo.ini @@ -34,4 +34,5 @@ B0z 0.05 [Output] # vtk 0.1 +dmp 2.0 analysis 0.1 diff --git a/test/MHD/ShearingBox/idefix.ini b/test/MHD/ShearingBox/idefix.ini index 52d34305..69bc441f 100644 --- a/test/MHD/ShearingBox/idefix.ini +++ b/test/MHD/ShearingBox/idefix.ini @@ -31,4 +31,5 @@ B0z 0.05 [Output] # vtk 0.1 +dmp 2.0 analysis 0.1 diff --git a/test/MHD/ShearingBox/setup.cpp b/test/MHD/ShearingBox/setup.cpp index b6f467e6..fb25092a 100644 --- a/test/MHD/ShearingBox/setup.cpp +++ b/test/MHD/ShearingBox/setup.cpp @@ -49,7 +49,7 @@ void AnalysisFunction(DataBlock &data) { // Initialisation routine. Can be used to allocate // Arrays or variables which are used later on Setup::Setup(Input &input, Grid &grid, DataBlock &data, Output &output) { - gammaIdeal=data.hydro.GetGamma(); + gammaIdeal=data.hydro->eos->GetGamma(); // Get rotation rate along vertical axis omega=input.Get("Hydro","rotation",0); @@ -58,7 +58,7 @@ Setup::Setup(Input &input, Grid &grid, DataBlock &data, Output &output) { B0z = input.Get("Setup","B0z",0); // Add our userstep to the timeintegrator - data.gravity.EnrollBodyForce(BodyForce); + data.gravity->EnrollBodyForce(BodyForce); analysis = new Analysis(input, grid, data, output,std::string("timevol.dat")); output.EnrollAnalysis(&AnalysisFunction); diff --git a/test/MHD/ShearingBox/testme.py b/test/MHD/ShearingBox/testme.py new file mode 100755 index 00000000..c89d24b5 --- /dev/null +++ b/test/MHD/ShearingBox/testme.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 + +""" + +@author: glesur +""" +import os +import sys +sys.path.append(os.getenv("IDEFIX_DIR")) + +import pytools.idfx_test as tst +tolerance=1e-14 + +def testMe(test): + test.configure() + test.compile() + inifiles=["idefix.ini","idefix-fargo.ini"] + for ini in inifiles: + mytol=tolerance + + test.run(inputFile=ini) + if test.init and not test.mpi: + test.makeReference(filename="dump.0001.dmp") + test.standardTest() + # When using RKL, except larger error due to B reconstruction and RKL # of substeps + test.nonRegressionTest(filename="dump.0001.dmp",tolerance=mytol) + + +test=tst.idfxTest() +if not test.dec: + test.dec=['2','1','2'] + +if not test.all: + if(test.check): + test.checkOnly(filename="dump.0001.dmp",tolerance=tolerance) + else: + testMe(test) +else: + test.noplot = True + test.single=False + test.reconstruction=2 + test.mpi=False + testMe(test) diff --git a/test/MHD/disk/setup.cpp b/test/MHD/disk/setup.cpp index a02c99ca..c5933207 100644 --- a/test/MHD/disk/setup.cpp +++ b/test/MHD/disk/setup.cpp @@ -2,15 +2,16 @@ #include "setup.hpp" // User-defined boundaries -void UserdefBoundary(DataBlock& data, int dir, BoundarySide side, real t) { +void UserdefBoundary(Hydro *hydro, int dir, BoundarySide side, real t) { + auto *data = hydro->data; if( (dir==IDIR) && (side == left)) { - IdefixArray4D Vc = data.hydro.Vc; - IdefixArray4D Vs = data.hydro.Vs; - IdefixArray1D x1 = data.x[IDIR]; + IdefixArray4D Vc = hydro->Vc; + IdefixArray4D Vs = hydro->Vs; + IdefixArray1D x1 = data->x[IDIR]; - int ighost = data.beg[IDIR]; + int ighost = data->beg[IDIR]; - data.hydro.boundary.BoundaryFor("UserDefBoundary", dir, side, + hydro->boundary->BoundaryFor("UserDefBoundary", dir, side, KOKKOS_LAMBDA (int k, int j, int i) { Vc(RHO,k,j,i) = Vc(RHO,k,j,ighost); Vc(PRS,k,j,i) = Vc(PRS,k,j,ighost); @@ -18,11 +19,11 @@ void UserdefBoundary(DataBlock& data, int dir, BoundarySide side, real t) { Vc(VX2,k,j,i) = Vc(VX2,k,j,ighost) * sqrt(x1(i)/x1(ighost)); Vc(VX3,k,j,i) = Vc(VX3,k,j,ighost) * sqrt(x1(i)/x1(ighost)); }); - data.hydro.boundary.BoundaryForX2s("UserDefBoundaryBX2s", dir, side, + hydro->boundary->BoundaryForX2s("UserDefBoundaryBX2s", dir, side, KOKKOS_LAMBDA (int k, int j, int i) { Vs(BX2s,k,j,i) = Vs(BX2s,k,j,ighost); }); - data.hydro.boundary.BoundaryForX3s("UserDefBoundaryBX3s", dir, side, + hydro->boundary->BoundaryForX3s("UserDefBoundaryBX3s", dir, side, KOKKOS_LAMBDA (int k, int j, int i) { Vs(BX3s,k,j,i) = Vs(BX3s,k,j,ighost); }); @@ -45,8 +46,8 @@ void Potential(DataBlock& data, const real t, IdefixArray1D& x1, IdefixArr // Arrays or variables which are used later on Setup::Setup(Input &input, Grid &grid, DataBlock &data, Output &output) { // Set the function for userdefboundary - data.hydro.EnrollUserDefBoundary(&UserdefBoundary); - data.gravity.EnrollPotential(&Potential); + data.hydro->EnrollUserDefBoundary(&UserdefBoundary); + data.gravity->EnrollPotential(&Potential); } // This routine initialize the flow diff --git a/test/MHD/diskSpherical/setup.cpp b/test/MHD/diskSpherical/setup.cpp index 983fcf2f..d18b6b7e 100644 --- a/test/MHD/diskSpherical/setup.cpp +++ b/test/MHD/diskSpherical/setup.cpp @@ -10,16 +10,20 @@ real gammaGlob; real densityFloorGlob; -void MySourceTerm(DataBlock &data, const real t, const real dtin) { - IdefixArray4D Vc = data.hydro.Vc; - IdefixArray4D Uc = data.hydro.Uc; - IdefixArray1D x1=data.x[IDIR]; - IdefixArray1D x2=data.x[JDIR]; +void MySourceTerm(Hydro *hydro, const real t, const real dtin) { + auto *data = hydro->data; + IdefixArray4D Vc = hydro->Vc; + IdefixArray4D Uc = hydro->Uc; + IdefixArray1D x1=data->x[IDIR]; + IdefixArray1D x2=data->x[JDIR]; real epsilon = epsilonGlob; real tauGlob=0.1; real gamma_m1=gammaGlob-1.0; real dt=dtin; - idefix_for("MySourceTerm",0,data.np_tot[KDIR],0,data.np_tot[JDIR],0,data.np_tot[IDIR], + idefix_for("MySourceTerm", + 0, data->np_tot[KDIR], + 0, data->np_tot[JDIR], + 0, data->np_tot[IDIR], KOKKOS_LAMBDA (int k, int j, int i) { real r=x1(i); real th=x2(j); @@ -58,15 +62,19 @@ void FargoVelocity(DataBlock &data, IdefixArray2D &Vphi) { } -void InternalBoundary(DataBlock& data, const real t) { - IdefixArray4D Vc = data.hydro.Vc; - IdefixArray4D Vs = data.hydro.Vs; - IdefixArray1D x1=data.x[IDIR]; - IdefixArray1D x2=data.x[JDIR]; +void InternalBoundary(Hydro *hydro, const real t) { + auto *data = hydro->data; + IdefixArray4D Vc = hydro->Vc; + IdefixArray4D Vs = hydro->Vs; + IdefixArray1D x1=data->x[IDIR]; + IdefixArray1D x2=data->x[JDIR]; real vAmax=10.0; real densityFloor = densityFloorGlob; - idefix_for("InternalBoundary",0,data.np_tot[KDIR],0,data.np_tot[JDIR],0,data.np_tot[IDIR], + idefix_for("InternalBoundary", + 0, data->np_tot[KDIR], + 0, data->np_tot[JDIR], + 0, data->np_tot[IDIR], KOKKOS_LAMBDA (int k, int j, int i) { real b2=EXPAND(Vc(BX1,k,j,i)*Vc(BX1,k,j,i) , +Vc(BX2,k,j,i)*Vc(BX2,k,j,i), +Vc(BX3,k,j,i)*Vc(BX3,k,j,i) ) ; real va2=b2/Vc(RHO,k,j,i); @@ -93,17 +101,20 @@ void InternalBoundary(DataBlock& data, const real t) { } // User-defined boundaries -void UserdefBoundary(DataBlock& data, int dir, BoundarySide side, real t) { - +void UserdefBoundary(Hydro *hydro, int dir, BoundarySide side, real t) { + auto *data = hydro->data; if( (dir==IDIR) && (side == left)) { - IdefixArray4D Vc = data.hydro.Vc; - IdefixArray4D Vs = data.hydro.Vs; - IdefixArray1D x1 = data.x[IDIR]; - IdefixArray1D x2 = data.x[JDIR]; + IdefixArray4D Vc = hydro->Vc; + IdefixArray4D Vs = hydro->Vs; + IdefixArray1D x1 = data->x[IDIR]; + IdefixArray1D x2 = data->x[JDIR]; - int ighost = data.nghost[IDIR]; + int ighost = data->nghost[IDIR]; real Omega=1.0; - idefix_for("UserDefBoundary",0,data.np_tot[KDIR],0,data.np_tot[JDIR],0,ighost, + idefix_for("UserDefBoundary", + 0, data->np_tot[KDIR], + 0, data->np_tot[JDIR], + 0, ighost, KOKKOS_LAMBDA (int k, int j, int i) { real R=x1(i)*sin(x2(j)); real z=x1(i)*cos(x2(j)); @@ -116,7 +127,10 @@ void UserdefBoundary(DataBlock& data, int dir, BoundarySide side, real t) { Vc(VX3,k,j,i) = R*Omega; }); - idefix_for("UserDefBoundaryX1S",0,data.np_tot[KDIR],0,data.np_tot[JDIR]+1,0,ighost, + idefix_for("UserDefBoundaryX1S", + 0, data->np_tot[KDIR], + 0, data->np_tot[JDIR]+1, + 0, ighost, KOKKOS_LAMBDA (int k, int j, int i) { Vs(BX2s,k,j,i) = ZERO_F; @@ -124,7 +138,10 @@ void UserdefBoundary(DataBlock& data, int dir, BoundarySide side, real t) { }); - idefix_for("UserDefBoundaryX3S",0,data.np_tot[KDIR]+1,0,data.np_tot[JDIR],0,ighost, + idefix_for("UserDefBoundaryX3S", + 0, data->np_tot[KDIR]+1, + 0, data->np_tot[JDIR], + 0, ighost, KOKKOS_LAMBDA (int k, int j, int i) { Vs(BX3s,k,j,i) = ZERO_F; @@ -133,24 +150,27 @@ void UserdefBoundary(DataBlock& data, int dir, BoundarySide side, real t) { } if( dir==JDIR) { - IdefixArray4D Vc = data.hydro.Vc; - IdefixArray4D Vs = data.hydro.Vs; + IdefixArray4D Vc = hydro->Vc; + IdefixArray4D Vs = hydro->Vs; int jghost; int jbeg,jend; if(side == left) { - jghost = data.beg[JDIR]; + jghost = data->beg[JDIR]; jbeg = 0; - jend = data.beg[JDIR]; + jend = data->beg[JDIR]; //return; } else if(side==right) { - jghost = data.end[JDIR]-1; - jbeg=data.end[JDIR]; - jend=data.np_tot[JDIR]; + jghost = data->end[JDIR]-1; + jbeg=data->end[JDIR]; + jend=data->np_tot[JDIR]; } - idefix_for("UserDefBoundary",0,data.np_tot[KDIR],jbeg,jend,0,data.np_tot[IDIR], + idefix_for("UserDefBoundary", + 0, data->np_tot[KDIR], + jbeg, jend, + 0, data->np_tot[IDIR], KOKKOS_LAMBDA (int k, int j, int i) { Vc(RHO,k,j,i) = Vc(RHO,k,jghost,i); Vc(PRS,k,j,i) = Vc(PRS,k,jghost,i); @@ -160,12 +180,18 @@ void UserdefBoundary(DataBlock& data, int dir, BoundarySide side, real t) { }); - idefix_for("UserDefBoundary_X1s",0,data.np_tot[KDIR],jbeg,jend,0,data.np_tot[IDIR]+1, + idefix_for("UserDefBoundary_X1s", + 0, data->np_tot[KDIR], + jbeg, jend, + 0, data->np_tot[IDIR]+1, KOKKOS_LAMBDA (int k, int j, int i) { Vs(BX1s,k,j,i) = ZERO_F; }); - idefix_for("UserDefBoundary_X3s",0,data.np_tot[KDIR]+1,jbeg,jend,0,data.np_tot[IDIR], + idefix_for("UserDefBoundary_X3s", + 0, data->np_tot[KDIR]+1, + jbeg, jend, + 0, data->np_tot[IDIR], KOKKOS_LAMBDA (int k, int j, int i) { Vs(BX3s,k,j,i) = ZERO_F; }); @@ -184,8 +210,8 @@ void ComputeUserVars(DataBlock & data, UserDefVariablesContainer &variables) { IdefixHostArray3D Er = variables["Er"]; IdefixHostArray3D Eth = variables["Eth"]; - Kokkos::deep_copy(Er,data.hydro.emf.Ex1); - Kokkos::deep_copy(Eth,data.hydro.emf.Ex2); + Kokkos::deep_copy(Er,data.hydro->emf->Ex1); + Kokkos::deep_copy(Eth,data.hydro->emf->Ex2); } @@ -194,14 +220,14 @@ void ComputeUserVars(DataBlock & data, UserDefVariablesContainer &variables) { // Arrays or variables which are used later on Setup::Setup(Input &input, Grid &grid, DataBlock &data, Output &output) { // Set the function for userdefboundary - data.hydro.EnrollUserDefBoundary(&UserdefBoundary); - data.hydro.EnrollUserSourceTerm(&MySourceTerm); - data.hydro.EnrollInternalBoundary(&InternalBoundary); + data.hydro->EnrollUserDefBoundary(&UserdefBoundary); + data.hydro->EnrollUserSourceTerm(&MySourceTerm); + data.hydro->EnrollInternalBoundary(&InternalBoundary); if(data.haveFargo) - data.fargo.EnrollVelocity(&FargoVelocity); + data.fargo->EnrollVelocity(&FargoVelocity); output.EnrollUserDefVariables(&ComputeUserVars); - gammaGlob=data.hydro.GetGamma(); + gammaGlob=data.hydro->eos->GetGamma(); epsilonGlob = input.Get("Setup","epsilon",0); densityFloorGlob = input.Get("Setup","densityFloor",0); } diff --git a/test/MHD/sod-iso/idefix-hll.ini b/test/MHD/sod-iso/idefix-hll.ini index 84e2a790..be20a405 100644 --- a/test/MHD/sod-iso/idefix-hll.ini +++ b/test/MHD/sod-iso/idefix-hll.ini @@ -23,4 +23,5 @@ X3-end periodic [Output] vtk 10.0 +dmp 10.0 log 200 diff --git a/test/MHD/sod-iso/idefix-hlld-rk3.ini b/test/MHD/sod-iso/idefix-hlld-rk3.ini new file mode 100644 index 00000000..aa0723e7 --- /dev/null +++ b/test/MHD/sod-iso/idefix-hlld-rk3.ini @@ -0,0 +1,27 @@ +[Grid] +X1-grid 1 0.0 800 u 100.0 +X2-grid 1 0.0 1 u 1.0 +X3-grid 1 0.0 1 u 1.0 + +[TimeIntegrator] +CFL 0.8 +tstop 10.0 +first_dt 1.e-4 +nstages 3 + +[Hydro] +solver hlld +csiso constant 1.0 + +[Boundary] +X1-beg outflow +X1-end outflow +X2-beg periodic +X2-end periodic +X3-beg periodic +X3-end periodic + +[Output] +vtk 10.0 +dmp 10.0 +log 200 diff --git a/test/MHD/sod-iso/idefix-hlld.ini b/test/MHD/sod-iso/idefix-hlld.ini index be1efab4..f9acb7aa 100644 --- a/test/MHD/sod-iso/idefix-hlld.ini +++ b/test/MHD/sod-iso/idefix-hlld.ini @@ -23,4 +23,5 @@ X3-end periodic [Output] vtk 10.0 +dmp 10.0 log 200 diff --git a/test/MHD/sod-iso/idefix-rk3.ini b/test/MHD/sod-iso/idefix-rk3.ini new file mode 100644 index 00000000..6e6cd525 --- /dev/null +++ b/test/MHD/sod-iso/idefix-rk3.ini @@ -0,0 +1,26 @@ +[Grid] +X1-grid 1 0.0 800 u 100.0 +X2-grid 1 0.0 1 u 1.0 +X3-grid 1 0.0 1 u 1.0 + +[TimeIntegrator] +CFL 0.8 +tstop 10.0 +first_dt 1.e-4 +nstages 3 + +[Hydro] +solver roe +csiso constant 1.0 + +[Boundary] +X1-beg outflow +X1-end outflow +X2-beg periodic +X2-end periodic +X3-beg periodic +X3-end periodic + +[Output] +vtk 10.0 +log 200 diff --git a/test/MHD/sod-iso/idefix-tvdlf.ini b/test/MHD/sod-iso/idefix-tvdlf.ini index 629f19cd..32a33959 100644 --- a/test/MHD/sod-iso/idefix-tvdlf.ini +++ b/test/MHD/sod-iso/idefix-tvdlf.ini @@ -23,4 +23,5 @@ X3-end periodic [Output] vtk 10.0 +dmp 10.0 log 200 diff --git a/test/MHD/sod-iso/idefix.ini b/test/MHD/sod-iso/idefix.ini index ba06fb6b..075eb2ae 100644 --- a/test/MHD/sod-iso/idefix.ini +++ b/test/MHD/sod-iso/idefix.ini @@ -23,4 +23,5 @@ X3-end periodic [Output] vtk 10.0 +dmp 10.0 log 200 diff --git a/test/MHD/sod-iso/python/data.ref.vtk b/test/MHD/sod-iso/python/data.ref.vtk deleted file mode 100644 index cd6ee5de..00000000 --- a/test/MHD/sod-iso/python/data.ref.vtk +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:120d02d5c1e568986cfaeb0be49a0210556acb951953e434a91efdbd423d648d -size 256477 diff --git a/test/MHD/sod-iso/python/testidefix.py b/test/MHD/sod-iso/python/testidefix.py deleted file mode 100755 index c95d8c23..00000000 --- a/test/MHD/sod-iso/python/testidefix.py +++ /dev/null @@ -1,72 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -""" -Created on Thu Mar 5 11:29:41 2020 - -@author: glesur -""" - -import os -import sys -sys.path.append(os.getenv("IDEFIX_DIR")) -from pytools.vtk_io import readVTK -import numpy as np -import argparse -import matplotlib.pyplot as plt -from scipy.interpolate import interp1d - -parser = argparse.ArgumentParser() -parser.add_argument("-noplot", - default=False, - help="disable plotting", - action="store_true") - - -args, unknown=parser.parse_known_args() - -V=readVTK('../data.0001.vtk', geometry='cartesian') -U=readVTK('data.ref.vtk', geometry='cartesian') - -solinterp=interp1d(U.x,U.data['rho'][:,0,0]) - -if(not args.noplot): - plt.figure(1) - plt.plot(V.x,V.data['RHO'][:,0,0],'+',markersize=2) - plt.plot(U.x,U.data['rho'][:,0,0]) - plt.title('Density') - - plt.figure(2) - #plt.plot(x,u) - plt.plot(V.x,V.data['VX1'][:,0,0],'+',markersize=2) - plt.plot(U.x,U.data['vx1'][:,0,0]) - plt.title('X Velocity') - - plt.figure(3) - #plt.plot(x,u) - plt.plot(V.x,V.data['VX2'][:,0,0],'+',markersize=2) - plt.plot(U.x,U.data['vx2'][:,0,0]) - plt.title('Y Velocity') - - plt.figure(4) - #plt.plot(x,u) - plt.plot(V.x,V.data['BX1'][:,0,0],'+',markersize=2) - plt.plot(U.x,U.data['Bx1'][:,0,0]) - plt.title('X field') - - plt.figure(5) - #plt.plot(x,u) - plt.plot(V.x,V.data['BX2'][:,0,0],'+',markersize=2) - plt.plot(U.x,U.data['Bx2'][:,0,0]) - plt.title('Y field') - - - plt.ioff() - plt.show() - -error=np.mean(np.fabs(V.data['RHO'][:,0,0]-solinterp(V.x))**2/solinterp(V.x)) - -print("Error=%e"%error) -if error<3e-4: - print("SUCCESS!") -else: - print("FAILURE!") diff --git a/test/MHD/sod-iso/testme.py b/test/MHD/sod-iso/testme.py new file mode 100755 index 00000000..798aace5 --- /dev/null +++ b/test/MHD/sod-iso/testme.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 + +""" + +@author: glesur +""" +import os +import sys +sys.path.append(os.getenv("IDEFIX_DIR")) + +import pytools.idfx_test as tst + +name="dump.0001.dmp" + +def testMe(test): + test.configure() + test.compile() + inifiles=["idefix.ini","idefix-hll.ini","idefix-hlld.ini","idefix-tvdlf.ini"] + if test.reconstruction==4: + inifiles=["idefix-rk3.ini","idefix-hlld-rk3.ini"] + + # loop on all the ini files for this test + for ini in inifiles: + test.run(inputFile=ini) + if test.init: + test.makeReference(filename=name) + test.nonRegressionTest(filename=name) + + +test=tst.idfxTest() + +if not test.all: + if(test.check): + test.checkOnly(filename=name) + else: + testMe(test) +else: + test.noplot = True + for rec in range(2,5): + test.vectPot=False + test.single=False + test.reconstruction=rec + test.mpi=False + testMe(test) + + # test in single precision + test.reconstruction=2 + test.single=True + testMe(test) diff --git a/test/MHD/sod/idefix-hll.ini b/test/MHD/sod/idefix-hll.ini index 6072cfa9..c6c5dbb1 100644 --- a/test/MHD/sod/idefix-hll.ini +++ b/test/MHD/sod/idefix-hll.ini @@ -24,4 +24,5 @@ X3-beg periodic X3-end periodic [Output] -vtk 10.0 -1 single_file +vtk 10.0 +dmp 10.0 diff --git a/test/MHD/sod/idefix-hlld-rk3.ini b/test/MHD/sod/idefix-hlld-rk3.ini new file mode 100644 index 00000000..68aeee11 --- /dev/null +++ b/test/MHD/sod/idefix-hlld-rk3.ini @@ -0,0 +1,28 @@ +[Grid] +X1-grid 1 0.0 800 u 100.0 +X2-grid 1 0.0 1 u 1.0 +X3-grid 1 0.0 1 u 1.0 + +[TimeIntegrator] +CFL 0.8 +CFL_max_var 1.1 # not used +tstop 10.0 +first_dt 1.e-4 +nstages 3 + +[Hydro] +solver hlld +gamma 1.66666666666667 + +[Boundary] +# not used +X1-beg outflow +X1-end outflow +X2-beg periodic +X2-end periodic +X3-beg periodic +X3-end periodic + +[Output] +vtk 10.0 +dmp 10.0 diff --git a/test/MHD/sod/idefix-hlld.ini b/test/MHD/sod/idefix-hlld.ini index b3f67dd0..cfbd486e 100644 --- a/test/MHD/sod/idefix-hlld.ini +++ b/test/MHD/sod/idefix-hlld.ini @@ -24,5 +24,5 @@ X3-beg periodic X3-end periodic [Output] -vtk 10.0 -1 single_file -log 100 +vtk 10.0 +dmp 10.0 diff --git a/test/MHD/sod/idefix-rk3.ini b/test/MHD/sod/idefix-rk3.ini new file mode 100644 index 00000000..c1926956 --- /dev/null +++ b/test/MHD/sod/idefix-rk3.ini @@ -0,0 +1,26 @@ +[Grid] +X1-grid 1 0.0 800 u 100.0 +X2-grid 1 0.0 1 u 1.0 +X3-grid 1 0.0 1 u 1.0 + +[TimeIntegrator] +CFL 0.8 +tstop 10.0 +first_dt 1.e-4 +nstages 3 + +[Hydro] +solver roe +gamma 1.66666666666667 + +[Boundary] +X1-beg outflow +X1-end outflow +X2-beg periodic +X2-end periodic +X3-beg periodic +X3-end periodic + +[Output] +vtk 10.0 +dmp 10.0 diff --git a/test/MHD/sod/idefix-tvdlf.ini b/test/MHD/sod/idefix-tvdlf.ini index 5ea5c69f..4bbc10c4 100644 --- a/test/MHD/sod/idefix-tvdlf.ini +++ b/test/MHD/sod/idefix-tvdlf.ini @@ -24,4 +24,5 @@ X3-beg periodic X3-end periodic [Output] -vtk 10.0 -1 single_file +vtk 10.0 +dmp 10.0 diff --git a/test/MHD/sod/idefix.ini b/test/MHD/sod/idefix.ini index 4477e3a4..67741314 100644 --- a/test/MHD/sod/idefix.ini +++ b/test/MHD/sod/idefix.ini @@ -23,3 +23,4 @@ X3-end periodic [Output] vtk 10.0 +dmp 10.0 diff --git a/test/MHD/sod/python/.DS_Store b/test/MHD/sod/python/.DS_Store deleted file mode 100644 index 5008ddfc..00000000 Binary files a/test/MHD/sod/python/.DS_Store and /dev/null differ diff --git a/test/MHD/sod/python/data.ref.vtk b/test/MHD/sod/python/data.ref.vtk deleted file mode 100644 index a76bb853..00000000 --- a/test/MHD/sod/python/data.ref.vtk +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:3373688cca12ae188a6211c026754c30ed31faef36fcbfb5960a6343d0f40f61 -size 288517 diff --git a/test/MHD/sod/python/testidefix.py b/test/MHD/sod/python/testidefix.py deleted file mode 100755 index 26ef57d6..00000000 --- a/test/MHD/sod/python/testidefix.py +++ /dev/null @@ -1,79 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -""" -Created on Thu Mar 5 11:29:41 2020 - -@author: glesur -""" - -import os -import sys -sys.path.append(os.getenv("IDEFIX_DIR")) -from pytools.vtk_io import readVTK -import numpy as np -import argparse -import matplotlib.pyplot as plt -from scipy.interpolate import interp1d - -parser = argparse.ArgumentParser() -parser.add_argument("-noplot", - default=False, - help="disable plotting", - action="store_true") - - -args, unknown=parser.parse_known_args() - -V=readVTK('../data.0001.vtk', geometry='cartesian') -U=readVTK('data.ref.vtk', geometry='cartesian') - -solinterp=interp1d(U.x,U.data['prs'][:,0,0]) - -if(not args.noplot): - plt.figure(1) - plt.plot(V.x,V.data['RHO'][:,0,0],'+',markersize=2) - plt.plot(U.x,U.data['rho'][:,0,0]) - plt.title('Density') - - plt.figure(2) - #plt.plot(x,u) - plt.plot(V.x,V.data['VX1'][:,0,0],'+',markersize=2) - plt.plot(U.x,U.data['vx1'][:,0,0]) - plt.title('X Velocity') - - plt.figure(3) - #plt.plot(x,u) - plt.plot(V.x,V.data['VX2'][:,0,0],'+',markersize=2) - plt.plot(U.x,U.data['vx2'][:,0,0]) - plt.title('Y Velocity') - - plt.figure(4) - #plt.plot(x,u) - plt.plot(V.x,V.data['BX1'][:,0,0],'+',markersize=2) - plt.plot(U.x,U.data['Bx1'][:,0,0]) - plt.title('X field') - - plt.figure(5) - #plt.plot(x,u) - plt.plot(V.x,V.data['BX2'][:,0,0],'+',markersize=2) - plt.plot(U.x,U.data['Bx2'][:,0,0]) - plt.title('Y field') - - - plt.figure(6) - #plt.plot(x,p) - plt.plot(V.x,V.data['PRS'][:,0,0],'+',markersize=2) - #plt.plot(V.x,solinterp(V.x)) - plt.plot(U.x,U.data['prs'][:,0,0]) - plt.title('Pressure') - - plt.ioff() - plt.show() - -error=np.mean(np.fabs(V.data['PRS'][:,0,0]-solinterp(V.x))**2/solinterp(V.x)) - -print("Error=%e"%error) -if error<1.5e-3: - print("SUCCESS!") -else: - print("FAILURE!") diff --git a/test/MHD/sod/testme.py b/test/MHD/sod/testme.py new file mode 100755 index 00000000..798aace5 --- /dev/null +++ b/test/MHD/sod/testme.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 + +""" + +@author: glesur +""" +import os +import sys +sys.path.append(os.getenv("IDEFIX_DIR")) + +import pytools.idfx_test as tst + +name="dump.0001.dmp" + +def testMe(test): + test.configure() + test.compile() + inifiles=["idefix.ini","idefix-hll.ini","idefix-hlld.ini","idefix-tvdlf.ini"] + if test.reconstruction==4: + inifiles=["idefix-rk3.ini","idefix-hlld-rk3.ini"] + + # loop on all the ini files for this test + for ini in inifiles: + test.run(inputFile=ini) + if test.init: + test.makeReference(filename=name) + test.nonRegressionTest(filename=name) + + +test=tst.idfxTest() + +if not test.all: + if(test.check): + test.checkOnly(filename=name) + else: + testMe(test) +else: + test.noplot = True + for rec in range(2,5): + test.vectPot=False + test.single=False + test.reconstruction=rec + test.mpi=False + testMe(test) + + # test in single precision + test.reconstruction=2 + test.single=True + testMe(test) diff --git a/test/MHD/sphBragTDiffusion/CMakeLists.txt b/test/MHD/sphBragTDiffusion/CMakeLists.txt new file mode 100644 index 00000000..629aed2d --- /dev/null +++ b/test/MHD/sphBragTDiffusion/CMakeLists.txt @@ -0,0 +1 @@ +enable_idefix_property(Idefix_MHD) diff --git a/test/MHD/sphBragTDiffusion/definitions.hpp b/test/MHD/sphBragTDiffusion/definitions.hpp new file mode 100644 index 00000000..53b4f38e --- /dev/null +++ b/test/MHD/sphBragTDiffusion/definitions.hpp @@ -0,0 +1,4 @@ +#define COMPONENTS 3 +#define DIMENSIONS 3 + +#define GEOMETRY SPHERICAL diff --git a/test/MHD/sphBragTDiffusion/idefix.ini b/test/MHD/sphBragTDiffusion/idefix.ini new file mode 100644 index 00000000..35ac54e8 --- /dev/null +++ b/test/MHD/sphBragTDiffusion/idefix.ini @@ -0,0 +1,31 @@ +[Grid] +X1-grid 1 5.76345919689455 32 u 12.322940970566583 # roots of sph_j1 (i.e. 2nd and 4th roots of sph_j0 derivative) +X2-grid 1 0.0 16 u 3.141592653589793 +X3-grid 1 0.0 32 u 6.283185307179586 + +[TimeIntegrator] +CFL 0.9 +CFL_max_var 1.1 +tstop 0.25 +first_dt 2.e-9 +nstages 3 + +[Hydro] +solver hlld +gamma 1.4 +bragTDiffusion rkl nolimiter constant 50.0 + +[Setup] +amplitude 1e-4 + +[Boundary] +X1-beg userdef +X1-end userdef +X2-beg axis +X2-end axis +X3-beg periodic +X3-end periodic + +[Output] +vtk 0.05 +dmp 0.25 diff --git a/test/MHD/sphBragTDiffusion/python/testidefix.py b/test/MHD/sphBragTDiffusion/python/testidefix.py new file mode 100644 index 00000000..8bf639cb --- /dev/null +++ b/test/MHD/sphBragTDiffusion/python/testidefix.py @@ -0,0 +1,88 @@ +import os +import sys +sys.path.append(os.getenv("IDEFIX_DIR")) +from pytools.vtk_io import readVTK +import numpy as np +import inifix + +conf = inifix.load("../idefix.ini") +amplitude = conf["Setup"]["amplitude"] +gamma = conf["Hydro"]["gamma"] +kappa = conf["Hydro"]["bragTDiffusion"][-1] +time_step = conf["Output"]["vtk"] + +nfile = 6 +list_time = [round(i*time_step, 3) for i in range(nfile)] +list_VTK = list() +for k in range(nfile): + if k >= 10: + current_number = str(k) + else: + current_number = '0' + str(k) + current_VTK = readVTK('../data.00' + current_number + '.vtk', geometry='spherical') + list_VTK.append(current_VTK) + +list_VX = list() +list_RHO = list() +list_PRS = list() +list_K = list() +list_T = list() +for VTK in list_VTK: + list_VX.append(VTK.data['VX1']) + list_RHO.append(VTK.data['RHO']) + list_PRS.append(VTK.data['PRS']) + list_K.append(VTK.data['PRS']/(VTK.data['RHO']**gamma)) + list_T.append(VTK.data['PRS']/VTK.data['RHO']) +VTK = list_VTK[-1] +R = VTK.r +TH = VTK.theta +PHI = VTK.phi +ir = len(R)//5 +ith = len(TH)//14 +iphi = len(PHI)*5//7 + +rho0 = 1. +c = 1./(gamma - 1.) +D = kappa/(rho0*c) + +def analytic_sol(r, th, phi, t): + lambd2 = 1./r**2/np.sin(th)**2 + return amplitude*((3/r**2 - 1)*np.sin(r)/r - 3*np.cos(r)/r**2)*(-3*np.cos(th)*np.sin(th))*(np.sin(phi))*np.exp(-t*D*lambd2) + 1 + +def analytic_rad_sol(r, t): + th = TH[ith] + phi = PHI[iphi] + return analytic_sol(r,th,phi,t) + +def analytic_th_sol(th, t): + r = R[ir] + phi = PHI[iphi] + return analytic_sol(r,th,phi,t) + + +def analytic_phi_sol(phi, t): + r = R[ir] + th = TH[ith] + return analytic_sol(r,th,phi,t) + + +success = True +eps = 6.4e-8 +for it,t in enumerate(list_time): + analytic_T = np.zeros((R.shape[0], TH.shape[0], PHI.shape[0])) + for ir,r in enumerate(R): + for ith,th in enumerate(TH): + for iphi,phi in enumerate(PHI): + analytic_T[ir,ith,iphi] = analytic_sol(r,th,phi,t) + TEMP = list_T[it] + if np.mean(np.fabs(TEMP - analytic_T)) > eps: + success = False + +if success: + print("SUCCESS") + print("Error: {0}".format(np.mean(np.fabs(TEMP - analytic_T)))) + sys.exit(0) +else: + print("Failed") + print("Error: {0}".format(np.mean(np.fabs(TEMP - analytic_T)))) + sys.exit(1) diff --git a/test/MHD/sphBragTDiffusion/setup.cpp b/test/MHD/sphBragTDiffusion/setup.cpp new file mode 100644 index 00000000..41a45f01 --- /dev/null +++ b/test/MHD/sphBragTDiffusion/setup.cpp @@ -0,0 +1,112 @@ +#include "idefix.hpp" +#include "setup.hpp" + + +real amplitude; +void InternalBoundary(Hydro *hydro, const real t) { + auto *data = hydro->data; + IdefixArray4D Vc = data->hydro->Vc; + idefix_for("InternalBoundary",0,data->np_tot[KDIR],0,data->np_tot[JDIR],0,data->np_tot[IDIR], + KOKKOS_LAMBDA (int k, int j, int i) { + // Cancel any motion that could be happening + Vc(VX1,k,j,i) = 0.0; + Vc(VX2,k,j,i) = 0.0; + Vc(VX3,k,j,i) = 0.0; + }); +} + +void UserDefBoundary(Hydro *hydro, int dir, BoundarySide side, real t) { + auto *data = hydro->data; + real B0 = 0.01; + IdefixArray4D Vc = data->hydro->Vc; + IdefixArray4D Vs = data->hydro->Vs; + IdefixArray1D x1 = data->x[IDIR]; + IdefixArray1D x2 = data->x[JDIR]; + IdefixArray1D x3 = data->x[KDIR]; + + IdefixArray1D x1l = data->xl[IDIR]; + IdefixArray1D x2l = data->xl[JDIR]; + IdefixArray1D x3l = data->xl[KDIR]; + if (dir==IDIR) { + int ibeg, iend; + if (side == left) { + ibeg = 0; + iend = data->beg[IDIR]; + idefix_for("InnerBoundary", 0, data->np_tot[KDIR], 0, data->np_tot[JDIR], ibeg, iend, + KOKKOS_LAMBDA (int k, int j, int i) { + Vc(RHO,k,j,i) = 1.; + Vc(PRS,k,j,i) = 1.; + Vc(VX1,k,j,i) = ZERO_F; + + Vs(BX1s,k,j,i) = ZERO_F; + Vs(BX2s,k,j,i) = ZERO_F; + Vs(BX3s,k,j,i) = B0*SIN(x2(j)); + }); + } + if (side == right) { + ibeg = data->end[IDIR]; + iend = data->np_tot[IDIR]; + idefix_for("InnerBoundary", 0, data->np_tot[KDIR], 0, data->np_tot[JDIR], ibeg, iend, + KOKKOS_LAMBDA (int k, int j, int i) { + Vc(RHO,k,j,i) = 1.; + Vc(PRS,k,j,i) = 1.; + Vc(VX1,k,j,i) = ZERO_F; + + Vs(BX1s,k,j,i) = ZERO_F; + Vs(BX2s,k,j,i) = ZERO_F; + Vs(BX3s,k,j,i) = B0*SIN(x2(j)); + }); + } + } +} + +// Initialisation routine. Can be used to allocate +// Arrays or variables which are used later on +Setup::Setup(Input &input, Grid &grid, DataBlock &data, Output &output) { + data.hydro->EnrollUserDefBoundary(&UserDefBoundary); + data.hydro->EnrollInternalBoundary(&InternalBoundary); + + amplitude = input.Get("Setup","amplitude",0); +} + +// This routine initialize the flow +// Note that data is on the device. +// One can therefore define locally +// a datahost and sync it, if needed +void Setup::InitFlow(DataBlock &data) { + // Create a host copy + DataBlockHost d(data); + real B0 = 0.01; + + for(int k = 0; k < d.np_tot[KDIR] ; k++) { + real x3 = d.x[KDIR](k); + real x3l = d.xl[KDIR](k); + for(int j = 0; j < d.np_tot[JDIR] ; j++) { + real x2 = d.x[JDIR](j); + real x2l = d.xl[JDIR](j); + for(int i = 0; i < d.np_tot[IDIR] ; i++) { + real x1=d.x[IDIR](i); + real x1l=d.xl[IDIR](i); + + //compute an eigen mode for the temperature as a product of r, theta and phi eigen modes with l=2, m=1 + d.Vc(RHO,k,j,i) = (3./pow(x1,2.) - 1.)*SIN(x1)/x1 - 3*COS(x1)/pow(x1,2.); //jl spherical function + d.Vc(RHO,k,j,i) *= -3.*COS(x2)*SIN(x2); //Plm(cos(theta)) + d.Vc(RHO,k,j,i) *= SIN(x3); //sin(m*phi) + d.Vc(RHO,k,j,i) *= amplitude; + d.Vc(RHO,k,j,i) += 1.; + d.Vc(RHO,k,j,i) = 1./d.Vc(RHO,k,j,i); + d.Vc(VX1,k,j,i) = ZERO_F; + d.Vc(VX2,k,j,i) = ZERO_F; + d.Vc(VX3,k,j,i) = ZERO_F; + d.Vc(PRS,k,j,i) = 1.0; + + d.Vs(BX1s,k,j,i) = ZERO_F; + d.Vs(BX2s,k,j,i) = ZERO_F; + d.Vs(BX3s,k,j,i) = B0*SIN(x2); + } + } + } + + // Send it all, if needed + d.SyncToDevice(); +} diff --git a/test/MHD/sphBragTDiffusion/testme.py b/test/MHD/sphBragTDiffusion/testme.py new file mode 100755 index 00000000..6ed5b164 --- /dev/null +++ b/test/MHD/sphBragTDiffusion/testme.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python3 + +""" + +@author: glesur +""" +import os +import sys +sys.path.append(os.getenv("IDEFIX_DIR")) + +import pytools.idfx_test as tst + + +def testMe(test): + test.configure() + test.compile() + inifiles=["idefix.ini"] + + # loop on all the ini files for this test + name = "dump.0001.dmp" + for ini in inifiles: + test.run(inputFile=ini) + test.standardTest() + if test.init: + test.makeReference(filename=name) + test.nonRegressionTest(filename=name, tolerance=2e-15) + + +test=tst.idfxTest() + +if not test.all: + if(test.check): + test.checkOnly(filename="dump.0001.dmp") + else: + testMe(test) +else: + test.noplot = True + test.vectPot=False + test.single=False + test.reconstruction=2 + test.mpi=False + testMe(test) diff --git a/test/MHD/sphBragViscosity/CMakeLists.txt b/test/MHD/sphBragViscosity/CMakeLists.txt new file mode 100644 index 00000000..ba8dc8b1 --- /dev/null +++ b/test/MHD/sphBragViscosity/CMakeLists.txt @@ -0,0 +1,4 @@ +enable_idefix_property(Idefix_MHD) +## The current test is not physics, but merely a non-regression check +## The physical test needs to have the divergence of the velocity field artifiacially set to zero +#replace_idefix_source(fluid/braginskii/bragViscosity.cpp noDivBragViscosity.cpp) diff --git a/test/MHD/sphBragViscosity/definitions.hpp b/test/MHD/sphBragViscosity/definitions.hpp new file mode 100644 index 00000000..efd699f1 --- /dev/null +++ b/test/MHD/sphBragViscosity/definitions.hpp @@ -0,0 +1,6 @@ +#define COMPONENTS 3 +#define DIMENSIONS 3 + +#define ISOTHERMAL + +#define GEOMETRY SPHERICAL diff --git a/test/MHD/sphBragViscosity/idefix.ini b/test/MHD/sphBragViscosity/idefix.ini new file mode 100644 index 00000000..a980670a --- /dev/null +++ b/test/MHD/sphBragViscosity/idefix.ini @@ -0,0 +1,31 @@ +[Grid] +X1-grid 1 3.8317059702075125 32 u 7.015586669815619 # 1st and 2nd roots of J1 +X2-grid 1 0.0 16 u 3.141592653589793 +X3-grid 1 0.0 32 u 6.283185307179586 + +[TimeIntegrator] +CFL 0.5 +CFL_max_var 1.1 # not used +tstop 0.1 +first_dt 1.e-9 +nstages 3 + +[Hydro] +solver hlld +csiso constant 1. +bragViscosity explicit nolimiter constant 5. + +[Setup] +amplitude 1e-3 + +[Boundary] +X1-beg userdef +X1-end userdef +X2-beg axis +X2-end axis +X3-beg periodic +X3-end periodic + +[Output] +vtk 0.01 +dmp 0.1 diff --git a/test/MHD/sphBragViscosity/noDivBragViscosity.cpp b/test/MHD/sphBragViscosity/noDivBragViscosity.cpp new file mode 100644 index 00000000..befb93d0 --- /dev/null +++ b/test/MHD/sphBragViscosity/noDivBragViscosity.cpp @@ -0,0 +1,872 @@ +// *********************************************************************************** +// Idefix MHD astrophysical code +// Copyright(C) Geoffroy R. J. Lesur +// and other code contributors +// Licensed under CeCILL 2.1 License, see COPYING for more information +// *********************************************************************************** + +// This source code is largely inspired from the viscous_flux of Pluto4.2 +// ((c) P. Tzeferacos & A. Mignone) + +// Implementation of monotonicity-preserving viscous flux following ZuHone et al., +// ApJ + +#include + +#include "bragViscosity.hpp" +#include "dataBlock.hpp" +#include "fluid.hpp" + + +#define D_DX_I(q,n) (q(n,k,j,i) - q(n,k,j,i - 1)) +#define D_DY_J(q,n) (q(n,k,j,i) - q(n,k,j - 1,i)) +#define D_DZ_K(q,n) (q(n,k,j,i) - q(n,k - 1,j,i)) + +#define SL_DX(q,n,iz,iy,ix) (q(n,iz,iy,ix) - q(n,iz,iy,ix - 1)) +#define SL_DY(q,n,iz,iy,ix) (q(n,iz,iy,ix) - q(n,iz,iy - 1,ix)) +#define SL_DZ(q,n,iz,iy,ix) (q(n,iz,iy,ix) - q(n,iz - 1,iy,ix)) + +#define D_DY_I(q,n) ( 0.25*(q(n,k,j + 1,i) + q(n,k,j + 1,i - 1)) \ + - 0.25*(q(n,k,j - 1,i) + q(n,k,j - 1,i - 1))) + +#define D_DZ_I(q,n) ( 0.25*(q(n,k + 1,j,i) + q(n,k + 1,j,i - 1)) \ + - 0.25*(q(n,k - 1,j,i) + q(n,k - 1,j,i - 1))) + +#define D_DX_J(q,n) ( 0.25*(q(n,k,j,i + 1) + q(n,k,j - 1,i + 1)) \ + - 0.25*(q(n,k,j,i - 1) + q(n,k,j - 1,i - 1))) + +#define D_DZ_J(q,n) ( 0.25*(q(n,k + 1,j,i) + q(n,k + 1,j - 1,i)) \ + - 0.25*(q(n,k - 1,j,i) + q(n,k - 1,j - 1,i))) + +#define D_DX_K(q,n) ( 0.25*(q(n,k,j,i + 1) + q(n,k - 1,j,i + 1)) \ + - 0.25*(q(n,k,j,i - 1) + q(n,k - 1,j,i - 1))) + +#define D_DY_K(q,n) ( 0.25*(q(n,k,j + 1,i) + q(n,k - 1,j + 1,i)) \ + - 0.25*(q(n,k,j - 1,i) + q(n,k - 1,j - 1,i))) + + + +#define BX_I Vs(BX1s,k,j,i) +#define BY_J Vs(BX2s,k,j,i) +#define BZ_K Vs(BX3s,k,j,i) +#define BY_I (0.25*(Vs(BX2s,k,j,i) + Vs(BX2s,k,j + 1,i) \ + + Vs(BX2s,k,j,i - 1) + Vs(BX2s,k,j + 1,i - 1))) +#define BZ_I (0.25*(Vs(BX3s,k,j,i) + Vs(BX3s,k + 1,j,i) \ + + Vs(BX3s,k,j,i - 1) + Vs(BX3s,k + 1,j,i - 1))) +#define BX_J (0.25*(Vs(BX1s,k,j,i) + Vs(BX1s,k,j,i + 1) \ + + Vs(BX1s,k,j - 1,i) + Vs(BX1s,k,j - 1,i + 1))) +#define BZ_J (0.25*(Vs(BX3s,k,j,i) + Vs(BX3s,k + 1,j,i) \ + + Vs(BX3s,k,j - 1,i) + Vs(BX3s,k + 1,j - 1,i))) +#define BX_K (0.25*(Vs(BX1s,k,j,i) + Vs(BX1s,k,j,i + 1) \ + + Vs(BX1s,k - 1,j,i) + Vs(BX1s,k - 1,j,i + 1))) +#define BY_K (0.25*(Vs(BX2s,k,j,i) + Vs(BX2s,k,j + 1,i) \ + + Vs(BX2s,k - 1,j,i) + Vs(BX2s,k - 1,j + 1,i))) + +////or what should be the exact equivalent, but for some reason isn't, maybe ask Geoffroy why: +//#define BX_I 0.5*(Vc(BX1,k,j,i) + Vc(BX1,k,j,i - 1)) +//#define BY_J 0.5*(Vc(BX2,k,j,i) + Vc(BX2,k,j - 1,i)) +//#define BZ_K 0.5*(Vc(BX3,k,j,i) + Vc(BX3,k - 1,j,i)) +//#define BY_I 0.5*(Vc(BX2,k,j,i) + Vc(BX2,k,j,i - 1)) +//#define BZ_I 0.5*(Vc(BX3,k,j,i) + Vc(BX3,k,j,i - 1)) +//#define BX_J 0.5*(Vc(BX1,k,j,i) + Vc(BX1,k,j - 1,i)) +//#define BZ_J 0.5*(Vc(BX3,k,j,i) + Vc(BX3,k,j - 1,i)) +//#define BX_K 0.5*(Vc(BX1,k,j,i) + Vc(BX1,k - 1,j,i)) +//#define BY_K 0.5*(Vc(BX2,k,j,i) + Vc(BX2,k - 1,j,i)) + + +void BragViscosity::InitArrays() { + // Allocate and fill arrays when needed + #if GEOMETRY != CARTESIAN + one_dmu = IdefixArray1D("BragViscosity_1dmu", data->np_tot[JDIR]); + IdefixArray1D dmu = one_dmu; + IdefixArray1D th = data->x[JDIR]; + idefix_for("ViscousInitGeometry",1,data->np_tot[JDIR], + KOKKOS_LAMBDA(int j) { + real scrch = FABS((1.0-cos(th(j)))*(sin(th(j)) >= 0.0 ? 1.0:-1.0) + -(1.0-cos(th(j-1))) * (sin(th(j-1)) > 0.0 ? 1.0:-1.0)); + dmu(j) = 1.0/scrch; + }); + #endif + bragViscSrc = IdefixArray4D("BragViscosity_source", COMPONENTS, data->np_tot[KDIR], + data->np_tot[JDIR], + data->np_tot[IDIR]); +} +void BragViscosity::ShowConfig() { + if(status.status==Constant) { + idfx::cout << "Braginskii Viscosity: ENABLED with constant braginskii viscosity etaBrag=" + << this->etaBrag << " ."<< std::endl; + } else if (status.status==UserDefFunction) { + idfx::cout << "Braginskii Viscosity: ENABLED with user-defined braginskii viscosity function." + << std::endl; + if(!bragViscousDiffusivityFunc) { + IDEFIX_ERROR("No braginskii viscosity function has been enrolled"); + } + } else { + IDEFIX_ERROR("Unknown braginskii viscosity mode"); + } + if(this->status.isExplicit) { + idfx::cout << "Braginskii Viscosity: uses an explicit time integration." << std::endl; + } else if(this->status.isRKL) { + idfx::cout << "Braginskii Viscosity: uses a Runge-Kutta-Legendre time integration." + << std::endl; + } else { + IDEFIX_ERROR("Unknown time integrator for braginskii viscosity."); + } + if(haveSlopeLimiter) { + idfx::cout << "Braginskii Viscosity: uses a slope limiter." << std::endl; + } +} + +void BragViscosity::EnrollBragViscousDiffusivity(DiffusivityFunc myFunc) { + if(this->status.status < UserDefFunction) { + IDEFIX_WARNING("Braginskii viscous diffusivity enrollment requires Hydro/BragViscosity " + "to be set to userdef in .ini file"); + } + this->bragViscousDiffusivityFunc = myFunc; +} + +// This function computes the viscous flux and stores it in hydro->fluxRiemann +// (this avoids an extra array) +// Associated source terms, present in non-cartesian geometry are also computed +// and stored in this->viscSrc for later use (in calcRhs). +void BragViscosity::AddBragViscousFlux(int dir, const real t, const IdefixArray4D &Flux) { + idfx::pushRegion("BragViscosity::AddBragViscousFlux"); + IdefixArray4D Vc = this->Vc; + IdefixArray4D Vs = this->Vs; + IdefixArray4D bragViscSrc = this->bragViscSrc; + IdefixArray3D dMax = this->dMax; + IdefixArray3D etaBragArr = this->etaBragArr; + IdefixArray1D one_dmu = this->one_dmu; + IdefixArray1D x1 = this->data->x[IDIR]; + IdefixArray1D x2 = this->data->x[JDIR]; + IdefixArray1D x3 = this->data->x[KDIR]; + IdefixArray1D x1l = this->data->xl[IDIR]; + IdefixArray1D x2l = this->data->xl[JDIR]; + IdefixArray1D x3l = this->data->xl[KDIR]; + IdefixArray1D dx1 = this->data->dx[IDIR]; + IdefixArray1D dx2 = this->data->dx[JDIR]; + IdefixArray1D dx3 = this->data->dx[KDIR]; + + #if GEOMETRY == SPHERICAL + IdefixArray1D sinx2 = this->data->sinx2; + IdefixArray1D tanx2 = this->data->tanx2; + IdefixArray1D sinx2m = this->data->sinx2m; + IdefixArray1D tanx2m = this->data->tanx2m; + #endif + + HydroModuleStatus haveViscosity = this->status.status; + + // Braginskii Viscosity + // Compute viscosity if needed + if(haveViscosity == UserDefFunction && dir == IDIR) { + if(bragViscousDiffusivityFunc) { + bragViscousDiffusivityFunc(*this->data, t, etaBragArr); + } else { + IDEFIX_ERROR("No user-defined Braginskii viscosity function has been enrolled"); + } + } + + #if GEOMETRY == CYLINDRICAL || GEOMETRY == POLAR + IDEFIX_WARNING("Braginskii viscosity in cylindrical and polar coordinates " + "has been implemented but never tested so far."); + #endif + + int ibeg, iend, jbeg, jend, kbeg, kend; + ibeg = this->data->beg[IDIR]; + iend = this->data->end[IDIR]; + jbeg = this->data->beg[JDIR]; + jend = this->data->end[JDIR]; + kbeg = this->data->beg[KDIR]; + kend = this->data->end[KDIR]; + + // Determine the offset along which we do the extrapolation + if(dir==IDIR) iend++; + if(dir==JDIR) jend++; + if(dir==KDIR) kend++; + + real etaBragConstant = this->etaBrag; + + idefix_for("ViscousFlux",kbeg, kend, jbeg, jend, ibeg, iend, + KOKKOS_LAMBDA (int k, int j, int i) { + real bi, bj, bk; + real Bmag; + real Pnor_par; + real bbgradV; + + real tau_xx, tau_xy, tau_xz; + real tau_yy, tau_yz; + real tau_zz; + + real dVxi, dVxj, dVxk; + real dVyi, dVyj, dVyk; + real dVzi, dVzj, dVzk; + + real divV; + + bi = bj = bk = ZERO_F; + + tau_xx = tau_xy = tau_xz = ZERO_F; + tau_yy = tau_yz = ZERO_F; + tau_zz = ZERO_F; + + dVxi = dVxj = dVxk = ZERO_F; + dVyi = dVyj = dVyk = ZERO_F; + dVzi = dVzj = dVzk = ZERO_F; + + real etaBrag; + + //Cell-centered values to compute source terms + #if GEOMETRY != CARTESIAN + real biC, bjC, bkC; + real BmagC; + real Pnor_parC; + real bbgradVC; + + real tau_xxC, tau_xyC, tau_xzC; + real tau_yyC, tau_yzC; + real tau_zzC; + + real dVxiC, dVxjC, dVxkC; + real dVyiC, dVyjC, dVykC; + real dVziC, dVzjC, dVzkC; + + real divVC; + + biC = bjC = bkC = ZERO_F; + tau_xxC = tau_xyC = tau_xzC = ZERO_F; + tau_yyC = tau_yzC = ZERO_F; + tau_zzC = ZERO_F; + + dVxiC = dVxjC = dVxkC = ZERO_F; + dVyiC = dVyjC = dVykC = ZERO_F; + dVziC = dVzjC = dVzkC = ZERO_F; + + real etaBragC; + + //Compute values at the center of the cells, no slope limiter are used at this stage + EXPAND( dVxiC = HALF_F*(Vc(VX1,k,j,i + 1) - Vc(VX1,k,j,i - 1))/dx1(i); , + dVyiC = HALF_F*(Vc(VX2,k,j,i + 1) - Vc(VX2,k,j,i - 1))/dx1(i); , + dVziC = HALF_F*(Vc(VX3,k,j,i + 1) - Vc(VX3,k,j,i - 1))/dx1(i); ) + EXPAND( dVxjC = HALF_F*(Vc(VX1,k,j + 1,i) - Vc(VX1,k,j - 1,i))/dx2(j); , + dVyjC = HALF_F*(Vc(VX2,k,j + 1,i) - Vc(VX2,k,j - 1,i))/dx2(j); , + dVzjC = HALF_F*(Vc(VX3,k,j + 1,i) - Vc(VX3,k,j - 1,i))/dx2(j); ) + EXPAND ( dVxkC = HALF_F*(Vc(VX1,k + 1,j,i) - Vc(VX1,k - 1,j,i))/dx3(k); , + dVykC = HALF_F*(Vc(VX2,k + 1,j,i) - Vc(VX2,k - 1,j,i))/dx3(k); , + dVzkC = HALF_F*(Vc(VX3,k + 1,j,i) - Vc(VX3,k - 1,j,i))/dx3(k); ) + + EXPAND( biC = Vc(BX1,k,j,i); , + bjC = Vc(BX2,k,j,i); , + bkC = Vc(BX3,k,j,i); ) + + BmagC = EXPAND( Vc(BX1,k,j,i)*Vc(BX1,k,j,i) , + + Vc(BX2,k,j,i)*Vc(BX2,k,j,i) , + + Vc(BX3,k,j,i)*Vc(BX3,k,j,i) ); + if(BmagC< 0.001*SMALL_NUMBER) { + BmagC = sqrt(BmagC) + 0.000001*SMALL_NUMBER; + } else { + BmagC = sqrt(BmagC); + } + biC /= BmagC; + bjC /= BmagC; + bkC /= BmagC; + #endif //GEOMETRY != CARTESIAN + + #if GEOMETRY == CYLINDRICAL + bbgradVC = D_EXPAND( + biC*biC*dVxiC + biC*bjC*dVyiC + biC*bkC*(dVziC - Vc(VX3,k,j,i)/x1(i)), + bjC*biC*dVxjC + bjC*bjC*dVyjC + bjC*bkC*dVzjC + bkC*bkC*Vc(VX1,k,j,i)/x1(i), + ); + + divVC = D_EXPAND(dVxiC + Vc(VX1,k,j,i)/x1(i) , + + dVyjC , + ); + // No cylindrical geometry in 3D! + #elif GEOMETRY == POLAR + bbgradVC = EXPAND( + biC*biC*dVxiC + bjC*biC*1./x1(i)*dVxjC + + bkC*biC*dVxkC, + + biC*bjC*(dVyiC - Vc(VX2,k,j,i)/x1(i)) + bjC*bjC*(1./x1(i)*dVyjC + Vc(VX1,k,j,i)/x1(i)) + + bkC*bjC*dVykC, + + biC*bkC*dVziC + bjC*bkC*1./x1(i)*dVzjC + + bkC*bkC*dVzkC, + ); + + divVC = D_EXPAND(dVxiC + Vc(VX1,k,j,i)/x1(i) , + + 1./x1(i)*dVyjC , + + dVzkC); + #elif GEOMETRY == SPHERICAL + real tan_1 = tanx2(j); + real s_1 = sinx2(j); + + // Trick to ensure that the axis does not lead to Nans + if(FABS(tan_1) < SMALL_NUMBER) { + tan_1 = ZERO_F; + } else { + tan_1 = ONE_F/tan_1; + } + + if(FABS(s_1) < SMALL_NUMBER) { + s_1 = ZERO_F; + } else { + s_1 = ONE_F/s_1; + } + + bbgradVC = EXPAND( + biC*biC*dVxiC + bjC*biC*1./x1(i)*dVxjC + + bkC*biC*s_1/x1(i)*dVxkC, + + biC*bjC*(dVyiC - Vc(VX2,k,j,i)/x1(i)) + bjC*bjC*(1./x1(i)*dVyjC + Vc(VX1,k,j,i)/x1(i)) + + bkC*bjC*s_1/x1(i)*dVykC, + + biC*bkC*(dVziC - Vc(VX3,k,j,i)/x1(i)) + bjC*bkC*(1./x1(i)*dVzjC - tan_1/x1(i)*Vc(VX3,k,j,i)) + + bkC*bkC*(s_1/x1(i)*dVzkC + Vc(VX1,k,j,i)/x1(i) + tan_1/x1(i)*Vc(VX2,k,j,i))); + + divVC = EXPAND(2.*Vc(VX1,k,j,i)/x1(i) + dVxiC, + + dVyjC/x1(i) + tan_1*Vc(VX2,k,j,i)/x1(i), + + dVzkC/x1(i)*s_1 ); + divVC = 0.; + #endif + + /////////////////////////////////////////// + // IDIR sweep // + /////////////////////////////////////////// + if(dir == IDIR) { + if(haveViscosity == UserDefFunction) { + etaBragC = etaBragArr(k,j,i); + if(haveSlopeLimiter) { + etaBrag = 2.*(etaBragArr(k,j,i-1)*etaBragArr(k,j,i))/(etaBragArr(k,j,i-1)+etaBragArr(k,j,i)); + } else { + etaBrag = HALF_F*(etaBragArr(k,j,i - 1)+etaBragArr(k,j,i)); + } + } else { + etaBragC = etaBrag = etaBragConstant; + } + + + EXPAND( dVxi = D_DX_I(Vc,VX1)/dx1(i); , + dVyi = D_DX_I(Vc,VX2)/dx1(i); , + dVzi = D_DX_I(Vc,VX3)/dx1(i); ) + + bi = BX_I; + Bmag = BX_I*BX_I; + #if DIMENSIONS >= 2 + bj = BY_I; + Bmag += BY_I*BY_I; + if (haveSlopeLimiter) { + dVxj = slopeLimiter(SL_DY(Vc,VX1,k,j + 1,i)/dx2(j+1), SL_DY(Vc,VX1,k,j + 1,i - 1)/dx2(j+1)); + dVxj = slopeLimiter(dVxj, slopeLimiter(SL_DY(Vc,VX1,k,j,i)/dx2(j), SL_DY(Vc,VX1,k,j,i - 1)/dx2(j))); + dVyj = slopeLimiter(SL_DY(Vc,VX2,k,j + 1,i)/dx2(j+1), SL_DY(Vc,VX2,k,j + 1,i - 1)/dx2(j+1)); + dVyj = slopeLimiter(dVyj, slopeLimiter(SL_DY(Vc,VX2,k,j,i)/dx2(j), SL_DY(Vc,VX2,k,j,i - 1)/dx2(j))); + #if DIMENSIONS == 3 + dVzj = slopeLimiter(SL_DY(Vc,VX3,k,j + 1,i)/dx2(j+1), SL_DY(Vc,VX3,k,j + 1,i - 1)/dx2(j+1)); + dVzj = slopeLimiter(dVzj, slopeLimiter(SL_DY(Vc,VX3,k,j,i)/dx2(j), SL_DY(Vc,VX3,k,j,i - 1)/dx2(j))); + #endif + } else { + EXPAND( dVxj = D_DY_I(Vc,VX1)/dx2(j); , + dVyj = D_DY_I(Vc,VX2)/dx2(j); , + dVzj = D_DY_I(Vc,VX3)/dx2(j); ) + } + #if DIMENSIONS == 3 + bk = BZ_I; + Bmag += BZ_I*BZ_I; + if (haveSlopeLimiter) { + dVxk = slopeLimiter(SL_DZ(Vc,VX1,k + 1,j,i)/dx3(k+1), SL_DZ(Vc,VX1,k + 1,j,i - 1)/dx3(k+1)); + dVxk = slopeLimiter(dVxk, slopeLimiter(SL_DZ(Vc,VX1,k,j,i)/dx3(k), SL_DZ(Vc,VX1,k,j,i - 1)/dx3(k))); + dVyk = slopeLimiter(SL_DZ(Vc,VX2,k + 1,j,i)/dx3(k+1), SL_DZ(Vc,VX2,k + 1,j,i - 1)/dx3(k+1)); + dVyk = slopeLimiter(dVyk, slopeLimiter(SL_DZ(Vc,VX2,k,j,i)/dx3(k), SL_DZ(Vc,VX2,k,j,i - 1)/dx3(k))); + dVzk = slopeLimiter(SL_DZ(Vc,VX3,k + 1,j,i)/dx3(k+1), SL_DZ(Vc,VX3,k + 1,j,i - 1)/dx3(k+1)); + dVzk = slopeLimiter(dVzk, slopeLimiter(SL_DZ(Vc,VX3,k,j,i)/dx3(k), SL_DZ(Vc,VX3,k,j,i - 1)/dx3(k))); + } else { + EXPAND ( dVxk = D_DZ_I(Vc,VX1)/dx3(k); , + dVyk = D_DZ_I(Vc,VX2)/dx3(k); , + dVzk = D_DZ_I(Vc,VX3)/dx3(k); ) + } + #endif + #endif + if(Bmag< 0.001*SMALL_NUMBER) { + Bmag = sqrt(Bmag) + 0.000001*SMALL_NUMBER; + } else { + Bmag = sqrt(Bmag); + } + bi /= Bmag; + bj /= Bmag; + bk /= Bmag; + + #if GEOMETRY == CARTESIAN + bbgradV = EXPAND( bi*bi*dVxi + bj*bi*dVxj + bk*bi*dVxk, + + bi*bj*dVyi + bj*bj*dVyj + bk*bj*dVyk, + + bi*bk*dVzi + bj*bk*dVzj + bk*bk*dVzk); + + divV = EXPAND(dVxi, + dVyj, + dVzk); + // No source term in cartesian geometry + #else + real vx1i, vx2i, vx3i; + vx1i = vx2i = vx3i = ZERO_F; + + EXPAND( vx1i = 0.5*(Vc(VX1,k,j,i-1)+Vc(VX1,k,j,i)); , + vx2i = 0.5*(Vc(VX2,k,j,i-1)+Vc(VX2,k,j,i)); , + vx3i = 0.5*(Vc(VX3,k,j,i-1)+Vc(VX3,k,j,i)); ) + + Pnor_parC = 3.0*etaBragC*(bbgradVC - divVC/3.0); + #endif // GEOMETRY != CARTESIAN + + #if GEOMETRY == CYLINDRICAL + bbgradV = D_EXPAND( + bi*bi*dVxi + bi*bj*dVyi + bi*bk*(dVzi - vx3i/x1l(i)), + bj*bi*dVxj + bj*bj*dVyj + bj*bk*dVzj + bk*bk*vx1i/x1l(i), + ); + + divV = D_EXPAND(dVxi + vx1i/x1l(i) , + + dVyj , + ); + // No cylindrical geometry in 3D! + + //cell-centered values for source terms + tau_zzC = Pnor_parC*(bkC*bkC - 1./3.); + + EXPAND( bragViscSrc(IDIR,k,j,i) = -tau_zzC/x1(i); , + bragViscSrc(JDIR,k,j,i) = ZERO_F; , + bragViscSrc(KDIR,k,j,i) = ZERO_F; ) + + #elif GEOMETRY == POLAR + bbgradV = EXPAND( + bi*bi*dVxi + bj*bi*1./x1l(i)*dVxj + + bk*bi*s_1/x(i)*dVxk, + + bi*bj*(dVyi - vx2i/x1l(i)) + bj*bj*(1./x1l(i)*dVyj + vx1i/x1l(i)) + + bk*bj*dVyk, + + bi*bk*dVzi + bj*bk*1./x1l(i)*dVzj + + bk*bk*dVzk, + ); + + divV = D_EXPAND(dVxi + vx1i/x1l(i) , + + 1./x1l(i)*dVyj , + + dVzk); + + //cell-centered values for source terms + tau_yyC = Pnor_parC*(bjC*bjC - 1./3.); + + EXPAND( bragViscSrc(IDIR,k,j,i) = -tau_yyC/x1(i); , + bragViscSrc(JDIR,k,j,i) = ZERO_F; , + bragViscSrc(KDIR,k,j,i) = ZERO_F; ) + + #elif GEOMETRY == SPHERICAL + // NOLINT + bbgradV = EXPAND( + bi*bi*dVxi + bj*bi*1./x1l(i)*dVxj + bk*bi*s_1/x1l(i)*dVxk, + + bi*bj*(dVyi - vx2i/x1l(i)) + bj*bj*(1./x1l(i)*dVyj + vx1i/x1l(i)) + bk*bj*s_1/x1l(i)*dVyk, + + bi*bk*(dVzi - vx3i/x1l(i)) + bj*bk*(1./x1l(i)*dVzj - tan_1/x1l(i)*vx3i) + + bk*bk*(s_1/x1l(i)*dVzk + vx1i/x1l(i) + tan_1/x1l(i)*vx2i)); + + divV = EXPAND(2.0*vx1i/x1l(i) + dVxi, + + dVyj/x1l(i) + tan_1*vx2i/x1l(i), + + dVzk/x1l(i)*s_1 ); + divV = 0.; + + //cell-centered values for source terms + tau_yyC = Pnor_parC*(bjC*bjC - 1./3.); + tau_zzC = Pnor_parC*(bkC*bkC - 1./3.); + + EXPAND( bragViscSrc(IDIR,k,j,i) = -(tau_yyC + tau_zzC)/x1(i); , + bragViscSrc(JDIR,k,j,i) = ZERO_F; , + bragViscSrc(KDIR,k,j,i) = ZERO_F; ) + #endif + + Pnor_par = 3.0*etaBrag*(bbgradV - divV/3.0); + + tau_xx = Pnor_par*(bi*bi-1./3.); + tau_xy = Pnor_par*bi*bj; + tau_xz = Pnor_par*bi*bk; + + // Update flux with the stress tensor + EXPAND( Flux(MX1, k, j, i) -= tau_xx; , + Flux(MX2, k, j, i) -= tau_xy; , + Flux(MX3, k, j, i) -= tau_xz; ) + + #if HAVE_ENERGY + Flux(ENG, k, j, i) -= EXPAND( 0.5*(Vc(VX1,k,j,i) + Vc(VX1,k,j,i-1))*tau_xx , + + 0.5*(Vc(VX2,k,j,i) + Vc(VX2,k,j,i-1))*tau_xy , + + 0.5*(Vc(VX3,k,j,i) + Vc(VX3,k,j,i-1))*tau_xz); + #endif + + real locdmax = etaBrag/(0.5*(Vc(RHO,k,j,i)+Vc(RHO,k,j,i-1))); + dMax(k,j,i) = FMAX(dMax(k,j,i),locdmax); + } + + + /////////////////////////////////////////// + // JDIR sweep // + /////////////////////////////////////////// + if(dir == JDIR) { + if(haveViscosity == UserDefFunction) { + etaBragC = etaBragArr(k,j,i); + if(haveSlopeLimiter) { + etaBrag = 2.*(etaBragArr(k,j-1,i)*etaBragArr(k,j,i))/(etaBragArr(k,j-1,i)+etaBragArr(k,j,i)); + } else { + etaBrag = HALF_F*(etaBragArr(k,j - 1,i)+etaBragArr(k,j,i)); + } + } else { + etaBragC = etaBrag = etaBragConstant; + } + + if (haveSlopeLimiter) { + dVxi = slopeLimiter(SL_DX(Vc,VX1,k,j,i + 1)/dx1(i+1), SL_DX(Vc,VX1,k,j - 1,i + 1)/dx1(i+1)); + dVxi = slopeLimiter(dVxi, slopeLimiter(SL_DX(Vc,VX1,k,j,i)/dx1(i), SL_DX(Vc,VX1,k,j - 1,i)/dx1(i))); + #if DIMENSIONS >= 2 + dVyi = slopeLimiter(SL_DX(Vc,VX2,k,j,i + 1)/dx1(i+1), SL_DX(Vc,VX2,k,j - 1,i + 1)/dx1(i+1)); + dVyi = slopeLimiter(dVyi, slopeLimiter(SL_DX(Vc,VX2,k,j,i)/dx1(i), SL_DX(Vc,VX2,k,j - 1,i)/dx1(i))); + #if DIMENSIONS == 3 + dVzi = slopeLimiter(SL_DX(Vc,VX3,k,j,i + 1)/dx1(i+1), SL_DX(Vc,VX3,k,j - 1,i + 1)/dx1(i+1)); + dVzi = slopeLimiter(dVzi, slopeLimiter(SL_DX(Vc,VX3,k,j,i)/dx1(i), SL_DX(Vc,VX3,k,j - 1,i)/dx1(i))); + #endif + #endif + } else { + EXPAND( dVxi = D_DX_J(Vc,VX1)/dx1(i); , + dVyi = D_DX_J(Vc,VX2)/dx1(i); , + dVzi = D_DX_J(Vc,VX3)/dx1(i); ) + } + + bi = BX_J; + Bmag = BX_J*BX_J; + #if DIMENSIONS >= 2 + bj = BY_J; + Bmag += BY_J*BY_J; + EXPAND( dVxj = D_DY_J(Vc,VX1)/dx2(j); , + dVyj = D_DY_J(Vc,VX2)/dx2(j); , + dVzj = D_DY_J(Vc,VX3)/dx2(j); ) + #if DIMENSIONS == 3 + bk = BZ_J; + Bmag += BZ_J*BZ_J; + if (haveSlopeLimiter) { + dVxk = slopeLimiter(SL_DZ(Vc,VX1,k + 1,j,i)/dx3(k+1), SL_DZ(Vc,VX1,k + 1,j - 1,i)/dx3(k+1)); + dVxk = slopeLimiter(dVxk, slopeLimiter(SL_DZ(Vc,VX1,k,j,i)/dx3(k), SL_DZ(Vc,VX1,k,j - 1,i)/dx3(k))); + dVyk = slopeLimiter(SL_DZ(Vc,VX2,k + 1,j,i)/dx3(k+1), SL_DZ(Vc,VX2,k + 1,j - 1,i)/dx3(k+1)); + dVyk = slopeLimiter(dVyk, slopeLimiter(SL_DZ(Vc,VX2,k,j,i)/dx3(k), SL_DZ(Vc,VX2,k,j - 1,i)/dx3(k))); + dVzk = slopeLimiter(SL_DZ(Vc,VX3,k + 1,j,i)/dx3(k+1), SL_DZ(Vc,VX3,k + 1,j - 1,i)/dx3(k+1)); + dVzk = slopeLimiter(dVzk, slopeLimiter(SL_DZ(Vc,VX3,k,j,i)/dx3(k), SL_DZ(Vc,VX3,k,j - 1,i)/dx3(k))); + } else { + EXPAND ( dVxk = D_DZ_J(Vc,VX1)/dx3(k); , + dVyk = D_DZ_J(Vc,VX2)/dx3(k); , + dVzk = D_DZ_J(Vc,VX3)/dx3(k); ) + } + #endif + #endif + if(Bmag< 0.001*SMALL_NUMBER) { + Bmag = sqrt(Bmag) + 0.000001*SMALL_NUMBER; + } else { + Bmag = sqrt(Bmag); + } + bi /= Bmag; + bj /= Bmag; + bk /= Bmag; + + #if GEOMETRY == CARTESIAN + bbgradV = EXPAND( bi*bi*dVxi + bj*bi*dVxj + bk*bi*dVxk, + + bi*bj*dVyi + bj*bj*dVyj + bk*bj*dVyk, + + bi*bk*dVzi + bj*bk*dVzj + bk*bk*dVzk); + + divV = EXPAND(dVxi, + dVyj, + dVzk); + + Pnor_par = 3.0*etaBrag*(bbgradV - divV/3.0); + + tau_xy = Pnor_par*bi*bj; + tau_yy = Pnor_par*(bj*bj - 1./3.); + tau_yz = Pnor_par*bj*bk; + // No source term in cartesian geometry + #else + real vx1i, vx2i, vx3i; + vx1i = vx2i = vx3i = ZERO_F; + + EXPAND( vx1i = 0.5*(Vc(VX1,k,j-1,i)+Vc(VX1,k,j,i)); , + vx2i = 0.5*(Vc(VX2,k,j-1,i)+Vc(VX2,k,j,i)); , + vx3i = 0.5*(Vc(VX3,k,j-1,i)+Vc(VX3,k,j,i)); ) + + Pnor_parC = 3.0*etaBragC*(bbgradVC - divVC/3.0); + #endif // GEOMETRY != CARTESIAN + + #if GEOMETRY == CYLINDRICAL + bbgradV = D_EXPAND( + bi*bi*dVxi + bi*bj*dVyi + bi*bk*(dVzi - vx3i/x1(i)), + bj*bi*dVxj + bj*bj*dVyj + bj*bk*dVzj + bk*bk*vx1i/x1(i), + ); + + divV = D_EXPAND(dVxi + vx1i/x1(i) , + + dVyj , + ); + // No cylindrical geometry in 3D! + // No source term along the z-axis in cylindrical/polar coordinates + EXPAND( bragViscSrc(IDIR,k,j,i) = ZERO_F; , + bragViscSrc(JDIR,k,j,i) = ZERO_F; , + bragViscSrc(KDIR,k,j,i) = ZERO_F; ) + #elif GEOMETRY == POLAR + bbgradV = EXPAND( + bi*bi*dVxi + bj*bi*1./x1(i)*dVxj + + bk*bi*s_1/x(i)*dVxk, + + bi*bj*(dVyi - vx2i/x1(i)) + bj*bj*(1./x1(i)*dVyj + vx1i/x1(i)) + + bk*bj*dVyk, + + bi*bk*dVzi + bj*bk*1./x1(i)*dVzj + + bk*bk*dVzk, + ); + + divV = D_EXPAND(dVxi + vx1i/x1(i) , + + 1./x1(i)*dVyj , + + dVzk); + + // See Geoffroy's trick for curvature source terms in cylindrical/polar coordinates + EXPAND( bragViscSrc(IDIR,k,j,i) = ZERO_F; , + bragViscSrc(JDIR,k,j,i) = ZERO_F; , + bragViscSrc(KDIR,k,j,i) = ZERO_F; ) +// //cell-centered values for source terms +// tau_xyC = Pnor_parC*biC*bjC; +// +// EXPAND( bragViscSrc(IDIR,k,j,i) = ZERO_F; , +// bragViscSrc(JDIR,k,j,i) = tau_xyC/x1(i); , +// bragViscSrc(KDIR,k,j,i) = ZERO_F; ) + + #elif GEOMETRY == SPHERICAL + tan_1 = tanx2m(j); + s_1 = sinx2m(j); + + // Trick to ensure that the axis does not lead to Nans + if(FABS(tan_1) < SMALL_NUMBER) { + tan_1 = ZERO_F; + } else { + tan_1 = ONE_F/tan_1; + } + + if(FABS(s_1) < SMALL_NUMBER) { + s_1 = ZERO_F; + } else { + s_1 = ONE_F/s_1; + } + + bbgradV = EXPAND( + bi*bi*dVxi + bj*bi*1./x1(i)*dVxj + bk*bi*s_1/x1(i)*dVxk, + + bi*bj*(dVyi - vx2i/x1(i)) + bj*bj*(1./x1(i)*dVyj + vx1i/x1(i)) + bk*bj*s_1/x1(i)*dVyk, + + bi*bk*(dVzi - vx3i/x1(i)) + bj*bk*(1./x1(i)*dVzj - tan_1/x1(i)*vx3i) + + bk*bk*(s_1/x1(i)*dVzk + vx1i/x1(i) + tan_1/x1(i)*vx2i)); + + divV = EXPAND( 2.0*vx1i/x1(i) + dVxi, + +(SIN(x2(j))*Vc(VX2,k,j,i) - FABS(SIN(x2(j-1)))*Vc(VX2,k,j-1,i))/x1(i) + *one_dmu(j) , + + dVzk/x1(i)*s_1 ); + divV = 0.; + + //Cell-centered values for the source terms + tan_1 = tanx2(j); + + // Trick to ensure that the axis does not lead to Nans + if(FABS(tan_1) < SMALL_NUMBER) { + tan_1 = ZERO_F; + } else { + tan_1 = ONE_F/tan_1; + } + + tau_xyC = Pnor_parC*biC*bjC; + tau_zzC = Pnor_parC*(bkC*bkC - 1./3.); + + EXPAND( bragViscSrc(IDIR,k,j,i) = ZERO_F; , + bragViscSrc(JDIR,k,j,i) = (tau_xyC - tau_zzC*tan_1)/x1(i); , + bragViscSrc(KDIR,k,j,i) = ZERO_F; ) + #endif + + Pnor_par = 3.0*etaBrag*(bbgradV - divV/3.0); + + tau_xy = Pnor_par*bi*bj; + tau_yy = Pnor_par*(bj*bj - 1./3.); + tau_yz = Pnor_par*bj*bk; + + // Update flux with the stress tensor + EXPAND( Flux(MX1, k, j, i) -= tau_xy; , + Flux(MX2, k, j, i) -= tau_yy; , + Flux(MX3, k, j, i) -= tau_yz; ) + + #if HAVE_ENERGY + Flux(ENG, k, j, i) -= EXPAND( 0.5*(Vc(VX1,k,j,i) + Vc(VX1,k,j-1,i))*tau_xy , + + 0.5*(Vc(VX2,k,j,i) + Vc(VX2,k,j-1,i))*tau_yy , + + 0.5*(Vc(VX3,k,j,i) + Vc(VX3,k,j-1,i))*tau_yz); + #endif + + real locdmax = etaBrag/(0.5*(Vc(RHO,k,j,i)+Vc(RHO,k,j-1,i))); + dMax(k,j,i) = FMAX(dMax(k,j,i),locdmax); + } + + + /////////////////////////////////////////// + // KDIR sweep // + /////////////////////////////////////////// + if(dir == KDIR) { + if(haveViscosity == UserDefFunction) { + etaBragC = etaBragArr(k,j,i); + if(haveSlopeLimiter) { + etaBrag = 2.*(etaBragArr(k-1,j,i)*etaBragArr(k,j,i))/(etaBragArr(k-1,j,i)+etaBragArr(k,j,i)); + } else { + etaBrag = HALF_F*(etaBragArr(k - 1,j,i)+etaBragArr(k,j,i)); + } + } else { + etaBragC = etaBrag = etaBragConstant; + } + + if (haveSlopeLimiter) { + dVxi = slopeLimiter(SL_DX(Vc,VX1,k,j,i + 1)/dx1(i+1), SL_DX(Vc,VX1,k - 1,j,i + 1)/dx1(i+1)); + dVxi = slopeLimiter(dVxi, slopeLimiter(SL_DX(Vc,VX1,k,j,i)/dx1(i), SL_DX(Vc,VX1,k - 1,j,i)/dx1(i))); + dVyi = slopeLimiter(SL_DX(Vc,VX2,k,j,i + 1)/dx1(i+1), SL_DX(Vc,VX2,k - 1,j,i + 1)/dx1(i+1)); + dVyi = slopeLimiter(dVyi, slopeLimiter(SL_DX(Vc,VX2,k,j,i)/dx1(i), SL_DX(Vc,VX2,k - 1,j,i)/dx1(i))); + dVzi = slopeLimiter(SL_DX(Vc,VX3,k,j,i + 1)/dx1(i+1), SL_DX(Vc,VX3,k - 1,j,i + 1)/dx1(i+1)); + dVzi = slopeLimiter(dVzi, slopeLimiter(SL_DX(Vc,VX3,k,j,i)/dx1(i), SL_DX(Vc,VX3,k - 1,j,i)/dx1(i))); + } else { + EXPAND( dVxi = D_DX_K(Vc,VX1)/dx1(i); , + dVyi = D_DX_K(Vc,VX2)/dx1(i); , + dVzi = D_DX_K(Vc,VX3)/dx1(i); ) + } + + bi = BX_K; + Bmag = BX_K*BX_K; + #if DIMENSIONS >= 2 + bj = BY_K; + Bmag += BY_K*BY_K; + if (haveSlopeLimiter) { + dVxj = slopeLimiter(SL_DY(Vc,VX1,k,j + 1,i)/dx2(j+1), SL_DY(Vc,VX1,k - 1,j + 1,i)/dx2(j+1)); + dVxj = slopeLimiter(dVxj, slopeLimiter(SL_DY(Vc,VX1,k,j,i)/dx2(j), SL_DY(Vc,VX1,k - 1,j,i)/dx2(j))); + dVyj = slopeLimiter(SL_DY(Vc,VX2,k,j + 1,i)/dx2(j+1), SL_DY(Vc,VX2,k - 1,j + 1,i)/dx2(j+1)); + dVyj = slopeLimiter(dVyj, slopeLimiter(SL_DY(Vc,VX2,k,j,i)/dx2(j), SL_DY(Vc,VX2,k - 1,j,i)/dx2(j))); + dVzj = slopeLimiter(SL_DY(Vc,VX3,k,j + 1,i)/dx2(j+1), SL_DY(Vc,VX3,k - 1,j + 1,i)/dx2(j+1)); + dVzj = slopeLimiter(dVzj, slopeLimiter(SL_DY(Vc,VX3,k,j,i)/dx2(j), SL_DY(Vc,VX3,k - 1,j,i)/dx2(j))); + } else { + EXPAND( dVxj = D_DY_K(Vc,VX1)/dx2(j); , + dVyj = D_DY_K(Vc,VX2)/dx2(j); , + dVzj = D_DY_K(Vc,VX3)/dx2(j); ) + } + + #if DIMENSIONS == 3 + bk = BZ_K; + Bmag += BZ_K*BZ_K; + EXPAND ( dVxk = D_DZ_K(Vc,VX1)/dx3(k); , + dVyk = D_DZ_K(Vc,VX2)/dx3(k); , + dVzk = D_DZ_K(Vc,VX3)/dx3(k); ) + #endif + #endif + if(Bmag< 0.001*SMALL_NUMBER) { + Bmag = sqrt(Bmag) + 0.000001*SMALL_NUMBER; + } else { + Bmag = sqrt(Bmag); + } + bi /= Bmag; + bj /= Bmag; + bk /= Bmag; + + #if GEOMETRY == CARTESIAN + bbgradV = EXPAND( bi*bi*dVxi + bj*bi*dVxj + bk*bi*dVxk, + + bi*bj*dVyi + bj*bj*dVyj + bk*bj*dVyk, + + bi*bk*dVzi + bj*bk*dVzj + bk*bk*dVzk); + + divV = EXPAND(dVxi, + dVyj, + dVzk); + + Pnor_par = 3.0*etaBrag*(bbgradV - divV/3.0); + + tau_xz = Pnor_par*bi*bk; + tau_yz = Pnor_par*bk*bj; + tau_zz = Pnor_par*(bk*bk - 1./3.); + + // No source term in cartesian geometry + #else + real vx1i, vx2i, vx3i; + vx1i = vx2i = vx3i = ZERO_F; + + EXPAND( vx1i = 0.5*(Vc(VX1,k-1,j,i)+Vc(VX1,k,j,i)); , + vx2i = 0.5*(Vc(VX2,k-1,j,i)+Vc(VX2,k,j,i)); , + vx3i = 0.5*(Vc(VX3,k-1,j,i)+Vc(VX3,k,j,i)); ) + #endif // GEOMETRY != CARTESIAN + + #if GEOMETRY == CYLINDRICAL + bbgradV = D_EXPAND( + bi*bi*dVxi + bi*bj*dVyi + bi*bk*(dVzi - vx3i/x1(i)), + bj*bi*dVxj + bj*bj*dVyj + bj*bk*dVzj + bk*bk*vx1i/x1(i), + ); + + divV = D_EXPAND(dVxi + vx1i/x1(i) , + + dVyj , + ); + // No cylindrical geometry in 3D! + // See Geoffroy's trick for curvature source terms in cylindrical/polar coordinates + bragViscSrc(IDIR,k,j,i) = ZERO_F; + bragViscSrc(JDIR,k,j,i) = ZERO_F; + bragViscSrc(KDIR,k,j,i) = ZERO_F; + #elif GEOMETRY == POLAR + bbgradV = EXPAND( + bi*bi*dVxi + bj*bi*1./x1(i)*dVxj + + bk*bi*s_1/x(i)*dVxk, + + bi*bj*(dVyi - vx2i/x1(i)) + bj*bj*(1./x1(i)*dVyj + vx1i/x1(i)) + + bk*bj*dVyk, + + bi*bk*dVzi + bj*bk*1./x1(i)*dVzj + + bk*bk*dVzk, + ); + + divV = D_EXPAND(dVxi + vx1i/x1(i) , + + 1./x1(i)*dVyj , + + dVzk); + + // No source term along the z-axis in cylindrical/polar coordinates + bragViscSrc(IDIR,k,j,i) = ZERO_F; + bragViscSrc(JDIR,k,j,i) = ZERO_F; + bragViscSrc(KDIR,k,j,i) = ZERO_F; + + #elif GEOMETRY == SPHERICAL + bbgradV = EXPAND( + bi*bi*dVxi + bj*bi*1./x1(i)*dVxj + bk*bi*s_1/x1(i)*dVxk, + + bi*bj*(dVyi - vx2i/x1(i)) + bj*bj*(1./x1(i)*dVyj + vx1i/x1(i)) + bk*bj*s_1/x1(i)*dVyk, + + bi*bk*(dVzi - vx3i/x1(i)) + bj*bk*(1./x1(i)*dVzj - tan_1/x1(i)*vx3i) + + bk*bk*(s_1/x1(i)*dVzk + vx1i/x1(i) + tan_1/x1(i)*vx2i)); + + divV = 2.0*vx1i/x1(i) + dVxi + dVyj/x1(i) + tan_1*vx2i/x1(i) + dVzk/x1(i)*s_1; + divV = 0.; + + bragViscSrc(IDIR,k,j,i) = ZERO_F; + bragViscSrc(JDIR,k,j,i) = ZERO_F; + bragViscSrc(KDIR,k,j,i) = ZERO_F; + #endif + + Pnor_par = 3.0*etaBrag*(bbgradV - divV/3.0); + + tau_xz = Pnor_par*bi*bk; + tau_yz = Pnor_par*bk*bj; + tau_zz = Pnor_par*(bk*bk - 1./3.); + + // Update flux with the stress tensor + EXPAND( Flux(MX1, k, j, i) -= tau_xz; , + Flux(MX2, k, j, i) -= tau_yz; , + Flux(MX3, k, j, i) -= tau_zz; ) + + #if HAVE_ENERGY + Flux(ENG, k, j, i) -= EXPAND( 0.5*(Vc(VX1,k,j,i) + Vc(VX1,k-1,j,i))*tau_xz , + + 0.5*(Vc(VX2,k,j,i) + Vc(VX2,k-1,j,i))*tau_yz , + + 0.5*(Vc(VX3,k,j,i) + Vc(VX3,k-1,j,i))*tau_zz); + #endif + + real locdmax = etaBrag/(0.5*(Vc(RHO,k,j,i)+Vc(RHO,k-1,j,i))); + dMax(k,j,i) = FMAX(dMax(k,j,i),locdmax); + } + }); + idfx::popRegion(); +} + +real minmodV(const real dvp, const real dvm) { + real dq= 0.0; + if(dvp*dvm >0.0) { + real dq = ( fabs(dvp) < fabs(dvm) ? dvp : dvm); + } + return(dq); +} + +real monotonizedCentralV(const real dvp, const real dvm) { + real dq = 0; + if(dvp*dvm >0.0) { + real dqc = 0.5*(dvp+dvm); + real d2q = 2.0*( fabs(dvp) < fabs(dvm) ? dvp : dvm); + dq= fabs(d2q) < fabs(dqc) ? d2q : dqc; + } + return(dq); +} + +real vanLeerV(const real dvp, const real dvm) { + real dq= 0.0; + dq = (dvp*dvm > ZERO_F ? TWO_F*dvp*dvm/(dvp + dvm) : ZERO_F); + return(dq); +} diff --git a/test/MHD/sphBragViscosity/python/testidefix.py b/test/MHD/sphBragViscosity/python/testidefix.py new file mode 100644 index 00000000..c016242d --- /dev/null +++ b/test/MHD/sphBragViscosity/python/testidefix.py @@ -0,0 +1,59 @@ +import os +import sys +sys.path.append(os.getenv("IDEFIX_DIR")) +from pytools.vtk_io import readVTK +import numpy as np +import inifix +from scipy.special import jn + +conf = inifix.load("../idefix.ini") +amplitude = conf["Setup"]["amplitude"] +time_step = conf["Output"]["vtk"] + +nfile = 10 +list_time = [round(i*time_step, 7) for i in range(nfile)] +list_VTK = list() +for k in range(nfile): + current_VTK = readVTK('../data.00{0:02d}.vtk'.format(k), 'spherical') + list_VTK.append(current_VTK) + +list_VX = list() +list_VY = list() +list_VZ = list() +list_RHO = list() +list_BX = list() +list_BY = list() +list_BZ = list() +for VTK in list_VTK: + list_VX.append(VTK.data['VX1']) + list_VY.append(VTK.data['VX2']) + list_VZ.append(VTK.data['VX3']) + list_BX.append(VTK.data['BX1']) + list_BY.append(VTK.data['BX2']) + list_BZ.append(VTK.data['BX3']) + list_RHO.append(VTK.data['RHO']) +X = VTK.r +Y = VTK.theta +Z = VTK.phi + +mu = conf["Hydro"]["bragViscosity"][-1] + +def analytic_sol(r, th, phi, t): + lambd2 = 2. + return amplitude*jn(1,r)/r*np.exp(-t*lambd2*mu) + + +eps = 7e-7 +for k, Vx in enumerate(list_VX): + t = list_time[k] + current_sol = np.zeros((len(X),len(Y),len(Z))) + for ix,x in enumerate(X): + for iy,y in enumerate(Y): + for iz,z in enumerate(Z): + current_sol[ix,iy,iz] = analytic_sol(x,y,z,t) + if np.mean(abs(current_sol - Vx)) > eps: + print("Failed") + sys.exit(1) + +print("SUCCESS") +sys.exit(0) diff --git a/test/MHD/sphBragViscosity/setup.cpp b/test/MHD/sphBragViscosity/setup.cpp new file mode 100644 index 00000000..fa20e168 --- /dev/null +++ b/test/MHD/sphBragViscosity/setup.cpp @@ -0,0 +1,79 @@ +#include "idefix.hpp" +#include "setup.hpp" + + +real amplitude; +void UserDefBoundary(Hydro *hydro, int dir, BoundarySide side, real t) { + auto *data = hydro->data; + IdefixArray4D Vc = data->hydro->Vc; + IdefixArray4D Vs = data->hydro->Vs; + IdefixArray1D x1 = data->x[IDIR]; + IdefixArray1D x2 = data->x[JDIR]; + IdefixArray1D x3 = data->x[KDIR]; + IdefixArray1D x1l = data->xl[IDIR]; + if (dir==IDIR) { + int ibeg, iend, iref; + if (side == left) { + ibeg = 0; + iend = data->beg[IDIR]; + iref = data->beg[IDIR]; + } + if (side == right) { + ibeg = data->end[IDIR]; + iend = data->np_tot[IDIR]; + iref = data->end[IDIR] - 1; + } + idefix_for("InnerBoundary", 0, data->np_tot[KDIR], 0, data->np_tot[JDIR], ibeg, iend, + KOKKOS_LAMBDA (int k, int j, int i) { + Vc(RHO,k,j,i) = 1.; + + Vc(VX1,k,j,i) = -Vc(VX1,k,j,iref); + Vc(VX2,k,j,i) = ZERO_F; + Vc(VX3,k,j,i) = ZERO_F; + }); + } +} + +// Initialisation routine. Can be used to allocate +// Arrays or variables which are used later on +Setup::Setup(Input &input, Grid &grid, DataBlock &data, Output &output) { + // Set the function for userdefboundary + data.hydro->EnrollUserDefBoundary(&UserDefBoundary); + + amplitude = input.Get("Setup","amplitude",0); +} + +// This routine initialize the flow +// Note that data is on the device. +// One can therefore define locally +// a datahost and sync it, if needed +void Setup::InitFlow(DataBlock &data) { + // Create a host copy + DataBlockHost d(data); + real B0 = 1e-5; + + for(int k = 0; k < d.np_tot[KDIR] ; k++) { + real x3 = d.x[KDIR](k); + real x3l = d.xl[KDIR](k); + for(int j = 0; j < d.np_tot[JDIR] ; j++) { + real x2 = d.x[JDIR](j); + real x2l = d.xl[JDIR](j); + for(int i = 0; i < d.np_tot[IDIR] ; i++) { + real x1=d.x[IDIR](i); + real x1l=d.xl[IDIR](i); + + d.Vc(RHO,k,j,i) = 1.0; + d.Vc(VX1,k,j,i) = amplitude*std::cyl_bessel_j(1, x1)/x1; +// d.Vc(VX1,k,j,i) = amplitude; + d.Vc(VX2,k,j,i) = 0.0; + d.Vc(VX3,k,j,i) = 0.0; + + d.Vs(BX1s,k,j,i) = B0/pow(x1l,2.); + d.Vs(BX2s,k,j,i) = 0.0; + d.Vs(BX3s,k,j,i) = 0.0; + } + } + } + // Send it all, if needed + d.SyncToDevice(); +} diff --git a/test/MHD/sphBragViscosity/testme.py b/test/MHD/sphBragViscosity/testme.py new file mode 100755 index 00000000..254417f4 --- /dev/null +++ b/test/MHD/sphBragViscosity/testme.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 + +""" + +@author: glesur +""" +import os +import sys +sys.path.append(os.getenv("IDEFIX_DIR")) + +import pytools.idfx_test as tst + + +def testMe(test): + test.configure() + test.compile() + inifiles=["idefix.ini"] + + # loop on all the ini files for this test + name = "dump.0001.dmp" + for ini in inifiles: + test.run(inputFile=ini) +## The current test is not physics, but merely a non-regression check +## The physical test needs to have the divergence of the velocity field artifiacially set to zero +# test.standardTest() + if test.init: + test.makeReference(filename=name) + + test.nonRegressionTest(filename=name, tolerance=1e-15) + + +test=tst.idfxTest() + +if not test.all: + if(test.check): + test.checkOnly(filename="dump.0001.dmp") + else: + testMe(test) +else: + test.noplot = True + test.vectPot=False + test.single=False + test.reconstruction=2 + test.mpi=False + testMe(test) diff --git a/test/Planet/Planet3Body/definitions.hpp b/test/Planet/Planet3Body/definitions.hpp new file mode 100644 index 00000000..e2c9ceee --- /dev/null +++ b/test/Planet/Planet3Body/definitions.hpp @@ -0,0 +1,8 @@ +#define COMPONENTS 2 +#define DIMENSIONS 2 + +#define ISOTHERMAL + +#define GEOMETRY POLAR +// Order of the scheme. 1=donnor cell, 2= linear reconstruction +//#define ORDER 2 diff --git a/test/Planet/Planet3Body/idefix.ini b/test/Planet/Planet3Body/idefix.ini new file mode 100644 index 00000000..a8b7bea0 --- /dev/null +++ b/test/Planet/Planet3Body/idefix.ini @@ -0,0 +1,56 @@ +[Grid] +X1-grid 1 0.042 64 u 2.3 +X2-grid 1 0.0 32 u 6.283185307179586 +X3-grid 1 -0.00125 1 u 0.00125 + +[TimeIntegrator] +CFL 0.5 +CFL_max_var 1.1 # not used +tstop 12.566370614359172 +# fixed_dt 1.e-4 +first_dt 1.e-4 +nstages 2 + +[Hydro] +solver hllc +csiso userdef +# rotation 1.0 #sqrt((1+qp)/rp/rp/rp) + +[Gravity] +potential central planet +Mcentral 1.0 + +[Boundary] +# not used +X1-beg userdef +X1-end userdef +X2-beg periodic +X2-end periodic +X3-beg outflow +X3-end outflow + +[Setup] +sigma0 0.001 +sigmaSlope 1.5 +h0 0.05 +flaringIndex 0.0 +alpha 1.0e-3 +densityFloor 1.0e-11 +wkzMin 0.05 +wkzMax 2.1 +wkzDamping 0.01 + +[Planet] +initialEccentricity 0.2 0.0 +planetToPrimary 0.0 0.0 +initialDistance -0.7631428283688879 1.0 +feelDisk false +feelPlanets true +smoothing plummer 0.03 0.0 + +[Output] +analysis 0. # 6283185307179586 +uservar PRS +vtk 6.283185307179586 +dmp 12.566370614359172 +log 100 diff --git a/test/Planet/Planet3Body/python/testidefix.py b/test/Planet/Planet3Body/python/testidefix.py new file mode 100755 index 00000000..a3b7f64d --- /dev/null +++ b/test/Planet/Planet3Body/python/testidefix.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +Created on Tue Dec 21 12:04:41 2021 + +@author: gwafflard +""" + +import os +import sys +sys.path.append(os.getenv("IDEFIX_DIR")) +import numpy as np + +def separation(resonance): + a = int(resonance[-1]) + b = int(resonance[0]) + return pow(a/b,2/3) + +def datafile(filename): + return np.loadtxt(filename, dtype="float64").T + +planet0 = datafile("../planet0.dat") +planet1 = datafile("../planet1.dat") + +resonance = "3:2" +plot_orbit = False + +theta1 = np.arctan2(planet1[2],planet1[1]) +apl1 = 1.0 +ecc1 = 0.0 +rpl1 = (apl1*(1-ecc1**2)/(1-ecc1*np.cos(theta1))) +xpl1 = rpl1*np.cos(theta1) +ypl1 = rpl1*np.sin(theta1) + +theta0 = np.arctan2(planet0[2],planet0[1]) +apl0 = separation(resonance) +ecc0 = 0.2 +rpl0 = (apl0*(1-ecc0**2)/(1+ecc0*np.cos(theta0))) +xpl0 = rpl0*np.cos(theta0) +ypl0 = rpl0*np.sin(theta0) + +if plot_orbit: + import matplotlib.pyplot as plt + plt.close("all") + fig, ax = plt.subplots(figsize=(6,6)) + ax.scatter(planet0[1], planet0[2], ls="-", c="k") + ax.scatter(planet1[1], planet1[2], ls="--", c="r") + ax.scatter(0,0,marker="+") + ax.plot(xpl0, ypl0, c="b") + ax.plot(xpl1, ypl1, c="g") + ax.set(xlabel="x", ylabel="y", aspect="equal", xlim=(-1.2,1.2), ylim=(-1.2,1.2)) + # ax.set(xlabel="x", ylabel="y", aspect="equal", xlim=(-2.5,2.5), ylim=(-2.5,2.5)) + ax.set_title(r"m$_{\rm sat}=0$, e=%.1f, n/n$_s$=%s (a0=%.3f)"%(ecc0,resonance,separation(resonance=resonance)), pad=20) + # plt.savefig(f"resonance{resonance}_e{ecc}.png", dpi=200) + fig.tight_layout() + + fig2, ax2 = plt.subplots(figsize=(6,6)) + ax2.scatter(theta0, (rpl0-np.sqrt(planet0[1]**2 + planet0[2]**2))/(rpl0), c="b", s=200) + ax2.scatter(theta1, (rpl1-np.sqrt(planet1[1]**2 + planet1[2]**2))/(rpl1), c="g", s=200) + + plt.show() + +rpl0_sim = np.sqrt(planet0[1]**2 + planet0[2]**2) +rpl1_sim = np.sqrt(planet1[1]**2 + planet1[2]**2) + +mean0 = np.mean(100*(rpl0-rpl0_sim)/rpl0) +mean1 = np.mean(100*(rpl1-rpl1_sim)/rpl1) +std0 = np.std(100*(rpl0-rpl0_sim)/rpl0) +std1 = np.std(100*(rpl1-rpl1_sim)/rpl1) +error_mean0 = mean0 +error_mean1 = mean1 +error_dispersion0 = np.max([abs(mean0+std0),abs(mean0-std0)]) +error_dispersion1 = np.max([abs(mean1+std1),abs(mean1-std1)]) + +print("Mean Error (planet0)=%s"%error_mean0) +print("Mean Error (planet1)=%s"%error_mean1) +print("max(mean +/- std) (planet0)=%s"%error_dispersion0) +print("max(mean +/- std) (planet1)=%s"%error_dispersion1) +if error_mean0<1.0e-10 and error_dispersion0<1.0e-9 and error_mean1<1.0e-10 and error_dispersion1<1.0e-9: + print("SUCCESS!") + sys.exit(0) +else: + print("FAILURE!") + sys.exit(1) diff --git a/test/Planet/Planet3Body/setup.cpp b/test/Planet/Planet3Body/setup.cpp new file mode 100644 index 00000000..fa39da3e --- /dev/null +++ b/test/Planet/Planet3Body/setup.cpp @@ -0,0 +1,433 @@ +#include +#include +#include "idefix.hpp" +#include "setup.hpp" +#include "planet.hpp" + +real wkzMinGlob; +real wkzMaxGlob; +real wkzDampingGlob; +real sigma0Glob; +real sigmaSlopeGlob; +real h0Glob; +real flaringIndexGlob; +real alphaGlob; +real densityFloorGlob; +real masstaperGlob; +real omegaGlob; + +/*********************************************/ +/** +Customized random number generator +Allow one to have consistant random numbers +generators on different architectures. +**/ +/*********************************************/ +real randm(void) { + const int a = 16807; + const int m = 2147483647; + static int in0 = 13763 + 2417*idfx::prank; + int q; + + /* find random number */ + q= (int) fmod((double) a * in0, m); + in0=q; + + return((real) ((double) q/(double)m)); +} + +void MySoundSpeed(DataBlock &data, const real t, IdefixArray3D &cs) { + real h0 = h0Glob; + real flaringIndex = flaringIndexGlob; + IdefixArray1D x1=data.x[IDIR]; + idefix_for("MySoundSpeed",0,data.np_tot[KDIR],0,data.np_tot[JDIR],0,data.np_tot[IDIR], + KOKKOS_LAMBDA (int k, int j, int i) { + real R = x1(i); + cs(k,j,i) = h0*pow(R,flaringIndex-0.5); + }); +} + +void Damping(Hydro *hydro, const real t, const real dtin) { + auto *data = hydro->data; + IdefixArray4D Vc = hydro->Vc; + IdefixArray4D Uc = hydro->Uc; + IdefixArray1D x1 = data->x[IDIR]; + IdefixArray1D x2 = data->x[JDIR]; + + real h0 = h0Glob; + real sigma0 = sigma0Glob; + real sigmaSlope = sigmaSlopeGlob; + real flaringIndex = flaringIndexGlob; + real omega = omegaGlob; + real gamma = hydro->eos->GetGamma(); + real dt = dtin; + bool isFargo = data->haveFargo; + + real rmin{data->mygrid->xbeg[0]}; + real wkzMin{wkzMinGlob}; + real rmax{data->mygrid->xend[0]}; + real wkzMax{wkzMaxGlob}; + real wkzDamping{wkzDampingGlob}; + idefix_for("Damping", + 0, data->np_tot[KDIR], + 0, data->np_tot[JDIR], + 0, data->np_tot[IDIR], + KOKKOS_LAMBDA (int k, int j, int i) { + real R = x1(i); + real Vk = 1.0/sqrt(R); + + /* + // Cooling function + real cs2 = h0*h0*pow(R,2*flaringIndex-1.0); + + // cooling time in local Keplerian units + real tau = 0.1*pow(R,1.5); + real Ptarget = cs2*Vc(RHO,k,j,i); + + Uc(ENG,k,j,i) += -dt*(Vc(PRS,k,j,i)-Ptarget)/(tau*(gamma-1.0)); + */ + + real lambda = 0.0; + + // Damping function for poloidal velocity field in damping zones + // Damp whatever is at RwkzMax + if (RwkzMax) { + lambda = 1.0/(wkzDamping*2.0*M_PI*pow(rmax,1.5))*pow(sin(M_PI*( (R-wkzMax) / (rmax-wkzMax) )/2.0),2.0); + } + + real rhoTarget = sigma0*pow(R,-sigmaSlope) ; + real vx2Target = 0.0; + if(!isFargo) { + vx2Target = Vk*sqrt(1.0-(1.0+sigmaSlope-2*flaringIndex)*h0*h0*pow(R,2*flaringIndex)) - omega * R; + } + + // relaxation + real drho = lambda*(Vc(RHO,k,j,i)-rhoTarget); + real dvx1 = lambda*Vc(RHO,k,j,i)*Vc(VX1,k,j,i); + real dvx2 = lambda*Vc(RHO,k,j,i)*(Vc(VX2,k,j,i)-vx2Target); + // real dvx3 = lambda*Vc(RHO,k,j,i)*Vc(VX3,k,j,i); + + real dmx1 = dvx1 + Vc(VX1,k,j,i) * drho; + real dmx2 = dvx2 + Vc(VX2,k,j,i) * drho; + // real dmx3 = dvx3 + Vc(VX3,k,j,i) * drho; + + // Kinetic energy fluctuations due to above damping + // must be compensated in total energy conservation + /* + real deng = 0.5*(Vc(VX1,k,j,i)*Vc(VX1,k,j,i) + +Vc(VX2,k,j,i)*Vc(VX2,k,j,i))*drho + +Vc(VX3,k,j,i)*Vc(VX3,k,j,i))*drho + + Vc(RHO,k,j,i) * ( + Vc(VX1,k,j,i)*dvx1 + + Vc(VX2,k,j,i)*dvx2 + + Vc(VX3,k,j,i)*dvx3 + ); + */ + + Uc(RHO,k,j,i) += -drho*dt; + Uc(MX1,k,j,i) += -dmx1*dt; + Uc(MX2,k,j,i) += -dmx2*dt; + // Uc(MX3,k,j,i) += -dmx3*dt; + // Uc(ENG,k,j,i) += -deng*dt; + +}); + +} + +// User-defined boundaries +void UserdefBoundary(Hydro *hydro, int dir, BoundarySide side, real t) { + auto *data = hydro->data; + IdefixArray4D Vc = hydro->Vc; + IdefixArray1D x1 = data->x[IDIR]; + real sigmaSlope=sigmaSlopeGlob; + real omega = omegaGlob; + + if(dir==IDIR) { + int ighost,ibeg,iend,sign; + if(side == left) { + ighost = data->beg[IDIR]; + ibeg = 0; + iend = data->beg[IDIR]; + sign=-1; + //return; + } + else if(side==right) { + ighost = data->end[IDIR]-1; + ibeg=data->end[IDIR]; + iend=data->np_tot[IDIR]; + sign=1; + } + + + idefix_for("UserDefBoundary", + 0, data->np_tot[KDIR], + 0, data->np_tot[JDIR], + ibeg, iend, + KOKKOS_LAMBDA (int k, int j, int i) { + real R=x1(i); + real R0=x1(ighost); + real Vk = 1.0/sqrt(R); + + Vc(RHO,k,j,i) = Vc(RHO,k,j,ighost)*pow(R/R0,-sigmaSlope) ; + Vc(VX1,k,j,i) = - Vc(VX1,k,j,2*ighost-i+sign); + Vc(VX2,k,j,i) = Vc(VX2,k,j,ighost)*pow(R/R0,-0.5) - omega*R ; + // Vc(VX3,k,j,i) = Vc(VX3,k,j,2*ighost-i+sign); + }); + } +} + +void FargoVelocity(DataBlock &data, IdefixArray2D &Vphi) { + IdefixArray1D x1 = data.x[IDIR]; + real omega = omegaGlob; + + idefix_for("FargoVphi",0,data.np_tot[KDIR], 0, data.np_tot[IDIR], + KOKKOS_LAMBDA (int k, int i) { + real R = x1(i); + Vphi(k,i) = 1.0/sqrt(R) - omega*R; + }); +} + +// Analyse data to produce an ascii output +void Analysis(DataBlock & data) { + // Mirror data on Host + DataBlockHost d(data); + + // Sync it + d.SyncFromDevice(); + + for(int ip=0; ip < data.planetarySystem->nbp ; ip++) { + // Get the orbital parameters + real timeStep = data.dt; + real xp = data.planetarySystem->planet[ip].getXp(); + real yp = data.planetarySystem->planet[ip].getYp(); + real zp = data.planetarySystem->planet[ip].getZp(); + real vxp = data.planetarySystem->planet[ip].getVxp(); + real vyp = data.planetarySystem->planet[ip].getVyp(); + real vzp = data.planetarySystem->planet[ip].getVzp(); + real qp = data.planetarySystem->planet[ip].getMp(); + real time = data.t; + + std::string planetName, tqwkName; + std::stringstream pName, tName; + pName << "planet" << ip << ".dat"; + tName << "tqwk" << ip << ".dat"; + planetName = pName.str(); + tqwkName = tName.str(); + // Write the data in ascii to our file + if(idfx::prank==0) { + std::ofstream f; + f.open(planetName,std::ios::app); + f.precision(10); + f << std::scientific << timeStep << " " << xp << " " << yp << " " << zp << " " << vxp << " " << vyp << " " << vzp << " " << qp << " " << time << std::endl; + f.close(); + } + + // Force force = {{0.0,0.0,0.0},{0.0,0.0,0.0},{0.0,0.0,0.0},{0.0,0.0,0.0}}; + Force &force = data.planetarySystem->planet[ip].m_force; + bool isp = true; + data.planetarySystem->planet[ip].computeForce(data,isp); + + // Get the torque and work + // real fxi = force.f_inner[0]; + // real fyi = force.f_inner[1]; + // real fzi = force.f_inner[2]; + // real fxo = force.f_outer[0]; + // real fyo = force.f_outer[1]; + // real fzo = force.f_outer[2]; + // real fxhi = force.f_ex_inner[0]; + // real fyhi = force.f_ex_inner[1]; + // real fzhi = force.f_ex_inner[2]; + // real fxho = force.f_ex_outer[0]; + // real fyho = force.f_ex_outer[1]; + // real fzho = force.f_ex_outer[2]; + + real tq_inner = xp*force.f_inner[1]-yp*force.f_inner[0]; + real tq_outer = xp*force.f_outer[1]-yp*force.f_outer[0]; + real tq_ex_inner = xp*force.f_ex_inner[1]-yp*force.f_ex_inner[0]; + real tq_ex_outer = xp*force.f_ex_outer[1]-yp*force.f_ex_outer[0]; + real wk_inner = vxp*force.f_inner[0]+vyp*force.f_inner[1]; + real wk_outer = vxp*force.f_outer[0]+vyp*force.f_outer[1]; + real wk_ex_inner = vxp*force.f_ex_inner[0]+vyp*force.f_ex_inner[1]; + real wk_ex_outer = vxp*force.f_ex_outer[0]+vyp*force.f_ex_outer[1]; + + // Write the data in ascii to our file + if(idfx::prank==0) { + std::ofstream ft; + ft.open(tqwkName,std::ios::app); + ft.precision(10); + // ft << std::scientific << timeStep << " " << fxi << " " << fyi << " " << fzi << " " << fxo << " " << fyo << " " << fzo << " " << fxhi << " " << fyhi << " " << fzhi << " " << fxho << " " << fyho << " " << fzho << " " << time << std::endl; + ft << std::scientific << timeStep << " " << tq_inner << " " << tq_outer << " " << tq_ex_inner << " " << tq_ex_outer << " " << wk_inner << " " << wk_outer << " " << wk_ex_inner << " " << wk_ex_outer << " " << time << std::endl; + ft.close(); + } + } +} + +// Compute user variables which will be written in vtk files +void ComputeUserVars(DataBlock & data, UserDefVariablesContainer &variables) { + // Mirror data on Host + real h0=h0Glob; + real flaringIndex=flaringIndexGlob; + DataBlockHost d(data); + + // Sync it + d.SyncFromDevice(); + + // The model being isothermal, we define ourselves PRS +#undef PRS + // Make references to the user-defined arrays (variables is a container of IdefixHostArray3D) + // Note that the labels should match the variable names in the input file + IdefixHostArray3D PRS = variables["PRS"]; + + for(int k = 0; k < d.np_tot[KDIR] ; k++) { + for(int j = 0; j < d.np_tot[JDIR] ; j++) { + for(int i = 0; i < d.np_tot[IDIR] ; i++) { + real R = d.x[IDIR](i); + real cs2 = h0*h0*pow(R,2*flaringIndex-1.); + PRS(k,j,i) = cs2*d.Vc(RHO,k,j,i); + } + } + } +} + +// Default constructor +// Initialisation routine. Can be used to allocate +// Arrays or variables which are used later on +Setup::Setup(Input &input, Grid &grid, DataBlock &data, Output &output)// : m_planet(0)//, Planet &planet) +{ + // Set the function for userdefboundary + data.hydro->EnrollUserDefBoundary(&UserdefBoundary); + data.hydro->EnrollUserSourceTerm(&Damping); + data.hydro->EnrollIsoSoundSpeed(&MySoundSpeed); + + if(data.haveFargo) + data.fargo->EnrollVelocity(&FargoVelocity); + if(data.hydro->haveRotation) { + omegaGlob = data.hydro->OmegaZ; + } else { + omegaGlob = 0.0; + } + // Enroll the analysis function + output.EnrollAnalysis(&Analysis); + // Enroll our user-defined variables + output.EnrollUserDefVariables(&ComputeUserVars); + wkzMinGlob = input.Get("Setup","wkzMin",0); + wkzMaxGlob = input.Get("Setup","wkzMax",0); + wkzDampingGlob = input.Get("Setup","wkzDamping",0); + sigma0Glob = input.Get("Setup","sigma0",0); + sigmaSlopeGlob = input.Get("Setup","sigmaSlope",0); + h0Glob = input.Get("Setup","h0",0); + flaringIndexGlob = input.Get("Setup","flaringIndex",0); + densityFloorGlob = input.Get("Setup","densityFloor",0); + masstaperGlob = input.Get("Planet","masstaper",0); + // delete file planet0.dat at initialization if we do not restart the simulation. + for(int ip=0; ip < data.planetarySystem->nbp ; ip++) { + std::string planetName, tqwkName; + std::stringstream pName, tName; + pName << "planet" << ip << ".dat"; + tName << "tqwk" << ip << ".dat"; + planetName = pName.str(); + tqwkName = tName.str(); + if (!(input.restartRequested)) { + if(idfx::prank==0) { + std::ofstream f; + f.open(planetName,std::ios::out); + f.close(); + std::ofstream ft; + ft.open(tqwkName,std::ios::out); + ft.close(); + } + } + /* + else { + if(idfx::prank==0) { + char ch; + int nline = input.restartFileNumber*input.Get("Output","dmp",0)/input.Get("Output","analysis",0); + int line = 1; + + // planet0.dat + std::ifstream fin(planetName); + std::ofstream fout; + fout.open("temp.dat", std::ios::out); + while(fin.get(ch)) + { + if(ch == '\n') + line++; + + if(line <= nline) // content not to be deleted + fout<("Output","dmp",0)/input.Get("Output","analysis",0); + line = 1; + while(fint.get(ch)) + { + if(ch == '\n') + line++; + + if(line <= nline) // content not to be deleted + foutt< +#include +#include "idefix.hpp" +#include "setup.hpp" +#include "planet.hpp" + +real wkzMinGlob; +real wkzMaxGlob; +real wkzDampingGlob; +real sigma0Glob; +real sigmaSlopeGlob; +real h0Glob; +real flaringIndexGlob; +real alphaGlob; +real densityFloorGlob; +real omegaGlob; + +/*********************************************/ +/** +Customized random number generator +Allow one to have consistant random numbers +generators on different architectures. +**/ +/*********************************************/ +real randm(void) { + const int a = 16807; + const int m = 2147483647; + static int in0 = 13763 + 2417*idfx::prank; + int q; + + /* find random number */ + q= (int) fmod((double) a * in0, m); + in0=q; + + return((real) ((double) q/(double)m)); +} + +void MySoundSpeed(DataBlock &data, const real t, IdefixArray3D &cs) { + real h0 = h0Glob; + real flaringIndex = flaringIndexGlob; + IdefixArray1D x1=data.x[IDIR]; + idefix_for("MySoundSpeed",0,data.np_tot[KDIR],0,data.np_tot[JDIR],0,data.np_tot[IDIR], + KOKKOS_LAMBDA (int k, int j, int i) { + real R = x1(i); + cs(k,j,i) = h0*pow(R,flaringIndex-0.5); + }); +} + +void MyViscosity(DataBlock &data, const real t, IdefixArray3D &eta1, IdefixArray3D &eta2) { + IdefixArray4D Vc=data.hydro->Vc; + IdefixArray1D x1=data.x[IDIR]; + real h0 = h0Glob; + real flaringIndex = flaringIndexGlob; + real alpha = alphaGlob; + idefix_for("MyViscosity",0,data.np_tot[KDIR],0,data.np_tot[JDIR],0,data.np_tot[IDIR], + KOKKOS_LAMBDA (int k, int j, int i) { + real R = x1(i); + real cs = h0*pow(R,flaringIndex-0.5); + eta1(k,j,i) = alpha*cs*h0*pow(R,flaringIndex+1)*Vc(RHO,k,j,i); + eta2(k,j,i) = ZERO_F; + }); + +} + +void Damping(Hydro *hydro, const real t, const real dtin) { + auto *data = hydro->data; + IdefixArray4D Vc = hydro->Vc; + IdefixArray4D Uc = hydro->Uc; + IdefixArray1D x1 = data->x[IDIR]; + IdefixArray1D x2 = data->x[JDIR]; + + real h0 = h0Glob; + real sigma0 = sigma0Glob; + real sigmaSlope = sigmaSlopeGlob; + real flaringIndex = flaringIndexGlob; + real omega = omegaGlob; + real gamma = hydro->eos->GetGamma(); + real dt = dtin; + bool isFargo = data->haveFargo; + + real rmin{data->mygrid->xbeg[0]}; + real wkzMin{wkzMinGlob}; + real rmax{data->mygrid->xend[0]}; + real wkzMax{wkzMaxGlob}; + real wkzDamping{wkzDampingGlob}; + idefix_for("MySourceTerm", + 0, data->np_tot[KDIR], + 0, data->np_tot[JDIR], + 0, data->np_tot[IDIR], + KOKKOS_LAMBDA (int k, int j, int i) { + real R = x1(i); + real Vk = 1.0/sqrt(R); + + /* + // Cooling function + real cs2 = h0*h0*pow(R,2*flaringIndex-1.0); + + // cooling time in local Keplerian units + real tau = 0.1*pow(R,1.5); + real Ptarget = cs2*Vc(RHO,k,j,i); + + Uc(ENG,k,j,i) += -dt*(Vc(PRS,k,j,i)-Ptarget)/(tau*(gamma-1.0)); + */ + + real lambda = 0.0; + + // Damping function for poloidal velocity field in damping zones + // Damp whatever is at RwkzMax + if (RwkzMax) { + lambda = 1.0/(wkzDamping*2.0*M_PI*pow(rmax,1.5))*pow(sin(M_PI*( (R-wkzMax) / (rmax-wkzMax) )/2.0),2.0); + } + + real rhoTarget = sigma0*pow(R,-sigmaSlope) ; + real vx2Target = 0.0; + if(!isFargo) { + vx2Target = Vk*sqrt(1.0-(1.0+sigmaSlope-2*flaringIndex)*h0*h0*pow(R,2*flaringIndex)) - omega * R; + } + + // relaxation + real drho = lambda*(Vc(RHO,k,j,i)-rhoTarget); + real dvx1 = lambda*Vc(RHO,k,j,i)*Vc(VX1,k,j,i); + real dvx2 = lambda*Vc(RHO,k,j,i)*(Vc(VX2,k,j,i)-vx2Target); + real dvx3 = lambda*Vc(RHO,k,j,i)*Vc(VX3,k,j,i); + + real dmx1 = dvx1 + Vc(VX1,k,j,i) * drho; + real dmx2 = dvx2 + Vc(VX2,k,j,i) * drho; + real dmx3 = dvx3 + Vc(VX3,k,j,i) * drho; + + // Kinetic energy fluctuations due to above damping + // must be compensated in total energy conservation + /* + real deng = 0.5*(Vc(VX1,k,j,i)*Vc(VX1,k,j,i) + +Vc(VX2,k,j,i)*Vc(VX2,k,j,i))*drho + +Vc(VX3,k,j,i)*Vc(VX3,k,j,i))*drho + + Vc(RHO,k,j,i) * ( + Vc(VX1,k,j,i)*dvx1 + + Vc(VX2,k,j,i)*dvx2 + + Vc(VX3,k,j,i)*dvx3 + ); + */ + + Uc(RHO,k,j,i) += -drho*dt; + Uc(MX1,k,j,i) += -dmx1*dt; + Uc(MX2,k,j,i) += -dmx2*dt; + Uc(MX3,k,j,i) += -dmx3*dt; + // Uc(ENG,k,j,i) += -deng*dt; + +}); + +} + +// User-defined boundaries +void UserdefBoundary(Hydro *hydro, int dir, BoundarySide side, real t) { + auto *data = hydro->data; + IdefixArray4D Vc = hydro->Vc; + IdefixArray1D x1 = data->x[IDIR]; + real sigmaSlope=sigmaSlopeGlob; + real omega = omegaGlob; + + if(dir==IDIR) { + int ighost,ibeg,iend,sign; + if(side == left) { + ighost = data->beg[IDIR]; + ibeg = 0; + iend = data->beg[IDIR]; + sign=-1; + //return; + } + else if(side==right) { + ighost = data->end[IDIR]-1; + ibeg=data->end[IDIR]; + iend=data->np_tot[IDIR]; + sign=1; + } + + + idefix_for("UserDefBoundary", + 0, data->np_tot[KDIR], + 0, data->np_tot[JDIR], + ibeg, iend, + KOKKOS_LAMBDA (int k, int j, int i) { + real R=x1(i); + real R0=x1(ighost); + real Vk = 1.0/sqrt(R); + + Vc(RHO,k,j,i) = Vc(RHO,k,j,ighost)*pow(R/R0,-sigmaSlope) ; + Vc(VX1,k,j,i) = - Vc(VX1,k,j,2*ighost-i+sign); + Vc(VX2,k,j,i) = Vc(VX2,k,j,ighost)*pow(R/R0,-0.5) - omega*R ; + Vc(VX3,k,j,i) = Vc(VX3,k,j,2*ighost-i+sign); + }); + } +} + +void FargoVelocity(DataBlock &data, IdefixArray2D &Vphi) { + IdefixArray1D x1 = data.x[IDIR]; + real omega = omegaGlob; + + idefix_for("FargoVphi",0,data.np_tot[KDIR], 0, data.np_tot[IDIR], + KOKKOS_LAMBDA (int k, int i) { + real R = x1(i); + Vphi(k,i) = 1.0/sqrt(R) - omega*R; + }); +} + +// Analyse data to produce an ascii output +void Analysis(DataBlock & data) { + // Mirror data on Host + DataBlockHost d(data); + + // Sync it + d.SyncFromDevice(); + +// data.DumpToFile("totou"); + + for(int ip=0; ip < data.planetarySystem->nbp ; ip++) { + // Get the orbital parameters + real timeStep = data.dt; + real xp = data.planetarySystem->planet[ip].getXp(); + real yp = data.planetarySystem->planet[ip].getYp(); + real zp = data.planetarySystem->planet[ip].getZp(); + real vxp = data.planetarySystem->planet[ip].getVxp(); + real vyp = data.planetarySystem->planet[ip].getVyp(); + real vzp = data.planetarySystem->planet[ip].getVzp(); + real qp = data.planetarySystem->planet[ip].getMp(); + real time = data.t; + + std::string planetName, tqwkName; + std::stringstream pName, tName; + pName << "planet" << ip << ".dat"; + tName << "tqwk" << ip << ".dat"; + planetName = pName.str(); + tqwkName = tName.str(); + // Write the data in ascii to our file + if(idfx::prank==0) { + std::ofstream f; + f.open(planetName,std::ios::app); + f.precision(10); + f << std::scientific << timeStep << " " << xp << " " << yp << " " << zp << " " << vxp << " " << vyp << " " << vzp << " " << qp << " " << time << std::endl; + f.close(); + } + + // Force force = {{0.0,0.0,0.0},{0.0,0.0,0.0},{0.0,0.0,0.0},{0.0,0.0,0.0}}; + Force &force = data.planetarySystem->planet[ip].m_force; + bool isp = true; + data.planetarySystem->planet[ip].computeForce(data,isp); + + // Get the torque and work + // real fxi = force.f_inner[0]; + // real fyi = force.f_inner[1]; + // real fzi = force.f_inner[2]; + // real fxo = force.f_outer[0]; + // real fyo = force.f_outer[1]; + // real fzo = force.f_outer[2]; + // real fxhi = force.f_ex_inner[0]; + // real fyhi = force.f_ex_inner[1]; + // real fzhi = force.f_ex_inner[2]; + // real fxho = force.f_ex_outer[0]; + // real fyho = force.f_ex_outer[1]; + // real fzho = force.f_ex_outer[2]; + + real tq_inner = xp*force.f_inner[1]-yp*force.f_inner[0]; + real tq_outer = xp*force.f_outer[1]-yp*force.f_outer[0]; + real tq_ex_inner = xp*force.f_ex_inner[1]-yp*force.f_ex_inner[0]; + real tq_ex_outer = xp*force.f_ex_outer[1]-yp*force.f_ex_outer[0]; + real wk_inner = vxp*force.f_inner[0]+vyp*force.f_inner[1]; + real wk_outer = vxp*force.f_outer[0]+vyp*force.f_outer[1]; + real wk_ex_inner = vxp*force.f_ex_inner[0]+vyp*force.f_ex_inner[1]; + real wk_ex_outer = vxp*force.f_ex_outer[0]+vyp*force.f_ex_outer[1]; + + // Write the data in ascii to our file + if(idfx::prank==0) { + std::ofstream ft; + ft.open(tqwkName,std::ios::app); + ft.precision(10); + // ft << std::scientific << timeStep << " " << fxi << " " << fyi << " " << fzi << " " << fxo << " " << fyo << " " << fzo << " " << fxhi << " " << fyhi << " " << fzhi << " " << fxho << " " << fyho << " " << fzho << " " << time << std::endl; + ft << std::scientific << timeStep << " " << tq_inner << " " << tq_outer << " " << tq_ex_inner << " " << tq_ex_outer << " " << wk_inner << " " << wk_outer << " " << wk_ex_inner << " " << wk_ex_outer << " " << time << std::endl; + ft.close(); + } + } +} + + +// Compute user variables which will be written in vtk files +void ComputeUserVars(DataBlock & data, UserDefVariablesContainer &variables) { + // Mirror data on Host + real h0=h0Glob; + real flaringIndex=flaringIndexGlob; + DataBlockHost d(data); + + // Sync it + d.SyncFromDevice(); + // The model being isothermal, we define ourselves PRS +#undef PRS + // Make references to the user-defined arrays (variables is a container of IdefixHostArray3D) + // Note that the labels should match the variable names in the input file + IdefixHostArray3D PRS = variables["PRS"]; + + for(int k = 0; k < d.np_tot[KDIR] ; k++) { + for(int j = 0; j < d.np_tot[JDIR] ; j++) { + for(int i = 0; i < d.np_tot[IDIR] ; i++) { + real R = d.x[IDIR](i); + real cs2 = h0*h0*pow(R,2*flaringIndex-1.); + PRS(k,j,i) = cs2*d.Vc(RHO,k,j,i); + } + } + } +} + +// Default constructor +// Initialisation routine. Can be used to allocate +// Arrays or variables which are used later on +Setup::Setup(Input &input, Grid &grid, DataBlock &data, Output &output)// : m_planet(0)//, Planet &planet) +{ + // Set the function for userdefboundary + data.hydro->EnrollUserDefBoundary(&UserdefBoundary); + data.hydro->EnrollUserSourceTerm(&Damping); + data.hydro->EnrollIsoSoundSpeed(&MySoundSpeed); + + if(data.hydro->viscosityStatus.status) { + alphaGlob = input.Get("Setup","alpha",0); + data.hydro->viscosity->EnrollViscousDiffusivity(&MyViscosity); + } + + if(data.haveFargo) + data.fargo->EnrollVelocity(&FargoVelocity); + if(data.hydro->haveRotation) { + omegaGlob = data.hydro->OmegaZ; + } else { + omegaGlob = 0.0; + } + // Enroll the analysis function + output.EnrollAnalysis(&Analysis); + // Enroll our user-defined variables + output.EnrollUserDefVariables(&ComputeUserVars); + wkzMinGlob = input.Get("Setup","wkzMin",0); + wkzMaxGlob = input.Get("Setup","wkzMax",0); + wkzDampingGlob = input.Get("Setup","wkzDamping",0); + sigma0Glob = input.Get("Setup","sigma0",0); + sigmaSlopeGlob = input.Get("Setup","sigmaSlope",0); + h0Glob = input.Get("Setup","h0",0); + flaringIndexGlob = input.Get("Setup","flaringIndex",0); + densityFloorGlob = input.Get("Setup","densityFloor",0); + // delete file planet0.dat at initialization if we do not restart the simulation. + for(int ip=0; ip < data.planetarySystem->nbp ; ip++) { + std::string planetName, tqwkName; + std::stringstream pName, tName; + pName << "planet" << ip << ".dat"; + tName << "tqwk" << ip << ".dat"; + planetName = pName.str(); + tqwkName = tName.str(); + if (!(input.restartRequested)) { + if(idfx::prank==0) { + std::ofstream f; + f.open(planetName,std::ios::out); + f.close(); + std::ofstream ft; + ft.open(tqwkName,std::ios::out); + ft.close(); + } + } +/* + else { + if(idfx::prank==0) { + char ch; + int nline = input.restartFileNumber*input.Get("Output","dmp",0)/input.Get("Output","analysis",0); + int line = 1; + + // planet0.dat + std::ifstream fin(planetName); + std::ofstream fout; + fout.open("temp.dat", std::ios::out); + while(fin.get(ch)) + { + if(ch == '\n') + line++; + + if(line <= nline) // content not to be deleted + fout<("Output","dmp",0)/input.Get("Output","analysis",0); + line = 1; + while(fint.get(ch)) + { + if(ch == '\n') + line++; + + if(line <= nline) // content not to be deleted + foutt< +#include +#include "idefix.hpp" +#include "setup.hpp" +#include "planet.hpp" + +real wkzMinGlob; +real wkzMaxGlob; +real wkzDampingGlob; +real sigma0Glob; +real sigmaSlopeGlob; +real h0Glob; +real flaringIndexGlob; +real alphaGlob; +real densityFloorGlob; +real omegaGlob; + +/*********************************************/ +/** +Customized random number generator +Allow one to have consistant random numbers +generators on different architectures. +**/ +/*********************************************/ +real randm(void) { + const int a = 16807; + const int m = 2147483647; + static int in0 = 13763 + 2417*idfx::prank; + int q; + + /* find random number */ + q= (int) fmod((double) a * in0, m); + in0=q; + + return((real) ((double) q/(double)m)); +} + +void MySoundSpeed(DataBlock &data, const real t, IdefixArray3D &cs) { + real h0 = h0Glob; + real flaringIndex = flaringIndexGlob; + IdefixArray1D x1=data.x[IDIR]; + idefix_for("MySoundSpeed",0,data.np_tot[KDIR],0,data.np_tot[JDIR],0,data.np_tot[IDIR], + KOKKOS_LAMBDA (int k, int j, int i) { + real R = x1(i); + cs(k,j,i) = h0*pow(R,flaringIndex-0.5); + }); +} + +void Damping(Hydro *hydro, const real t, const real dtin) { + auto *data = hydro->data; + IdefixArray4D Vc = hydro->Vc; + IdefixArray4D Uc = hydro->Uc; + IdefixArray1D x1 = data->x[IDIR]; + IdefixArray1D x2 = data->x[JDIR]; + + real h0 = h0Glob; + real sigma0 = sigma0Glob; + real sigmaSlope = sigmaSlopeGlob; + real flaringIndex = flaringIndexGlob; + real omega = omegaGlob; + real gamma = hydro->eos->GetGamma(); + real dt = dtin; + bool isFargo = data->haveFargo; + + real rmin{data->mygrid->xbeg[0]}; + real wkzMin{wkzMinGlob}; + real rmax{data->mygrid->xend[0]}; + real wkzMax{wkzMaxGlob}; + real wkzDamping{wkzDampingGlob}; + idefix_for("MySourceTerm", + 0, data->np_tot[KDIR], + 0, data->np_tot[JDIR], + 0, data->np_tot[IDIR], + KOKKOS_LAMBDA (int k, int j, int i) { + real R = x1(i); + real Vk = 1.0/sqrt(R); + + /* + // Cooling function + real cs2 = h0*h0*pow(R,2*flaringIndex-1.0); + + // cooling time in local Keplerian units + real tau = 0.1*pow(R,1.5); + real Ptarget = cs2*Vc(RHO,k,j,i); + + Uc(ENG,k,j,i) += -dt*(Vc(PRS,k,j,i)-Ptarget)/(tau*(gamma-1.0)); + */ + + real lambda = 0.0; + + // Damping function for poloidal velocity field in damping zones + // Damp whatever is at RwkzMax + if (RwkzMax) { + lambda = 1.0/(wkzDamping*2.0*M_PI*pow(rmax,1.5))*pow(sin(M_PI*( (R-wkzMax) / (rmax-wkzMax) )/2.0),2.0); + } + + real rhoTarget = sigma0*pow(R,-sigmaSlope) ; + real vx2Target = 0.0; + if(!isFargo) { + vx2Target = Vk*sqrt(1.0-(1.0+sigmaSlope-2*flaringIndex)*h0*h0*pow(R,2*flaringIndex)) - omega * R; + } + + // relaxation + real drho = lambda*(Vc(RHO,k,j,i)-rhoTarget); + real dvx1 = lambda*Vc(RHO,k,j,i)*Vc(VX1,k,j,i); + real dvx2 = lambda*Vc(RHO,k,j,i)*(Vc(VX2,k,j,i)-vx2Target); + real dvx3 = lambda*Vc(RHO,k,j,i)*Vc(VX3,k,j,i); + + real dmx1 = dvx1 + Vc(VX1,k,j,i) * drho; + real dmx2 = dvx2 + Vc(VX2,k,j,i) * drho; + real dmx3 = dvx3 + Vc(VX3,k,j,i) * drho; + + // Kinetic energy fluctuations due to above damping + // must be compensated in total energy conservation + /* + real deng = 0.5*(Vc(VX1,k,j,i)*Vc(VX1,k,j,i) + +Vc(VX2,k,j,i)*Vc(VX2,k,j,i))*drho + +Vc(VX3,k,j,i)*Vc(VX3,k,j,i))*drho + + Vc(RHO,k,j,i) * ( + Vc(VX1,k,j,i)*dvx1 + + Vc(VX2,k,j,i)*dvx2 + + Vc(VX3,k,j,i)*dvx3 + ); + */ + + Uc(RHO,k,j,i) += -drho*dt; + Uc(MX1,k,j,i) += -dmx1*dt; + Uc(MX2,k,j,i) += -dmx2*dt; + Uc(MX3,k,j,i) += -dmx3*dt; + // Uc(ENG,k,j,i) += -deng*dt; + +}); + +} + +// User-defined boundaries +void UserdefBoundary(Hydro *hydro, int dir, BoundarySide side, real t) { + auto *data = hydro->data; + IdefixArray4D Vc = hydro->Vc; + IdefixArray1D x1 = data->x[IDIR]; + real sigmaSlope=sigmaSlopeGlob; + real omega = omegaGlob; + + if(dir==IDIR) { + int ighost,ibeg,iend,sign; + if(side == left) { + ighost = data->beg[IDIR]; + ibeg = 0; + iend = data->beg[IDIR]; + sign=-1; + //return; + } + else if(side==right) { + ighost = data->end[IDIR]-1; + ibeg=data->end[IDIR]; + iend=data->np_tot[IDIR]; + sign=1; + } + + + idefix_for("UserDefBoundary", + 0, data->np_tot[KDIR], + 0, data->np_tot[JDIR], + ibeg, iend, + KOKKOS_LAMBDA (int k, int j, int i) { + real R=x1(i); + real R0=x1(ighost); + real Vk = 1.0/sqrt(R); + + Vc(RHO,k,j,i) = Vc(RHO,k,j,ighost)*pow(R/R0,-sigmaSlope) ; + Vc(VX1,k,j,i) = - Vc(VX1,k,j,2*ighost-i+sign); + Vc(VX2,k,j,i) = Vc(VX2,k,j,ighost)*pow(R/R0,-0.5) - omega*R ; + Vc(VX3,k,j,i) = Vc(VX3,k,j,2*ighost-i+sign); + }); + } +} + +void FargoVelocity(DataBlock &data, IdefixArray2D &Vphi) { + IdefixArray1D x1 = data.x[IDIR]; + real omega = omegaGlob; + + idefix_for("FargoVphi",0,data.np_tot[KDIR], 0, data.np_tot[IDIR], + KOKKOS_LAMBDA (int k, int i) { + real R = x1(i); + Vphi(k,i) = 1.0/sqrt(R) - omega*R; + }); +} + +// Analyse data to produce an ascii output +void Analysis(DataBlock & data) { + // Mirror data on Host + DataBlockHost d(data); + + // Sync it + d.SyncFromDevice(); + +// data.DumpToFile("totou"); + + for(int ip=0; ip < data.planetarySystem->nbp ; ip++) { + // Get the orbital parameters + real timeStep = data.dt; + real xp = data.planetarySystem->planet[ip].getXp(); + real yp = data.planetarySystem->planet[ip].getYp(); + real zp = data.planetarySystem->planet[ip].getZp(); + real vxp = data.planetarySystem->planet[ip].getVxp(); + real vyp = data.planetarySystem->planet[ip].getVyp(); + real vzp = data.planetarySystem->planet[ip].getVzp(); + real qp = data.planetarySystem->planet[ip].getMp(); + real time = data.t; + + std::string planetName, tqwkName; + std::stringstream pName, tName; + pName << "planet" << ip << ".dat"; + tName << "tqwk" << ip << ".dat"; + planetName = pName.str(); + tqwkName = tName.str(); + // Write the data in ascii to our file + if(idfx::prank==0) { + std::ofstream f; + f.open(planetName,std::ios::app); + f.precision(10); + f << std::scientific << timeStep << " " << xp << " " << yp << " " << zp << " " << vxp << " " << vyp << " " << vzp << " " << qp << " " << time << std::endl; + f.close(); + } + + // Force force = {{0.0,0.0,0.0},{0.0,0.0,0.0},{0.0,0.0,0.0},{0.0,0.0,0.0}}; + Force &force = data.planetarySystem->planet[ip].m_force; + bool isp = true; + data.planetarySystem->planet[ip].computeForce(data,isp); + + // Get the torque and work + // real fxi = force.f_inner[0]; + // real fyi = force.f_inner[1]; + // real fzi = force.f_inner[2]; + // real fxo = force.f_outer[0]; + // real fyo = force.f_outer[1]; + // real fzo = force.f_outer[2]; + // real fxhi = force.f_ex_inner[0]; + // real fyhi = force.f_ex_inner[1]; + // real fzhi = force.f_ex_inner[2]; + // real fxho = force.f_ex_outer[0]; + // real fyho = force.f_ex_outer[1]; + // real fzho = force.f_ex_outer[2]; + + real tq_inner = xp*force.f_inner[1]-yp*force.f_inner[0]; + real tq_outer = xp*force.f_outer[1]-yp*force.f_outer[0]; + real tq_ex_inner = xp*force.f_ex_inner[1]-yp*force.f_ex_inner[0]; + real tq_ex_outer = xp*force.f_ex_outer[1]-yp*force.f_ex_outer[0]; + real wk_inner = vxp*force.f_inner[0]+vyp*force.f_inner[1]; + real wk_outer = vxp*force.f_outer[0]+vyp*force.f_outer[1]; + real wk_ex_inner = vxp*force.f_ex_inner[0]+vyp*force.f_ex_inner[1]; + real wk_ex_outer = vxp*force.f_ex_outer[0]+vyp*force.f_ex_outer[1]; + + // Write the data in ascii to our file + if(idfx::prank==0) { + std::ofstream ft; + ft.open(tqwkName,std::ios::app); + ft.precision(10); + // ft << std::scientific << timeStep << " " << fxi << " " << fyi << " " << fzi << " " << fxo << " " << fyo << " " << fzo << " " << fxhi << " " << fyhi << " " << fzhi << " " << fxho << " " << fyho << " " << fzho << " " << time << std::endl; + ft << std::scientific << time << " " << timeStep << " " << tq_inner << " " << tq_outer << " " << tq_ex_inner << " " << tq_ex_outer << " " << wk_inner << " " << wk_outer << " " << wk_ex_inner << " " << wk_ex_outer << " " << time << std::endl; + ft.close(); + } + } +} + + +// Compute user variables which will be written in vtk files +void ComputeUserVars(DataBlock & data, UserDefVariablesContainer &variables) { + // Mirror data on Host + real h0=h0Glob; + real flaringIndex=flaringIndexGlob; + DataBlockHost d(data); + + // Sync it + d.SyncFromDevice(); + + // The model being isothermal, we define ourselves PRS +#undef PRS + // Make references to the user-defined arrays (variables is a container of IdefixHostArray3D) + // Note that the labels should match the variable names in the input file + IdefixHostArray3D PRS = variables["PRS"]; + + for(int k = 0; k < d.np_tot[KDIR] ; k++) { + for(int j = 0; j < d.np_tot[JDIR] ; j++) { + for(int i = 0; i < d.np_tot[IDIR] ; i++) { + real R = d.x[IDIR](i); + real cs2 = h0*h0*pow(R,2*flaringIndex-1.); + PRS(k,j,i) = cs2*d.Vc(RHO,k,j,i); + } + } + } +} + +// Default constructor +// Initialisation routine. Can be used to allocate +// Arrays or variables which are used later on +Setup::Setup(Input &input, Grid &grid, DataBlock &data, Output &output)// : m_planet(0)//, Planet &planet) +{ + // Set the function for userdefboundary + data.hydro->EnrollUserDefBoundary(&UserdefBoundary); + data.hydro->EnrollUserSourceTerm(&Damping); + data.hydro->EnrollIsoSoundSpeed(&MySoundSpeed); + + if(data.haveFargo) + data.fargo->EnrollVelocity(&FargoVelocity); + if(data.hydro->haveRotation) { + omegaGlob = data.hydro->OmegaZ; + } else { + omegaGlob = 0.0; + } + // Enroll the analysis function + output.EnrollAnalysis(&Analysis); + // Enroll our user-defined variables + output.EnrollUserDefVariables(&ComputeUserVars); + wkzMinGlob = input.Get("Setup","wkzMin",0); + wkzMaxGlob = input.Get("Setup","wkzMax",0); + wkzDampingGlob = input.Get("Setup","wkzDamping",0); + sigma0Glob = input.Get("Setup","sigma0",0); + sigmaSlopeGlob = input.Get("Setup","sigmaSlope",0); + h0Glob = input.Get("Setup","h0",0); + flaringIndexGlob = input.Get("Setup","flaringIndex",0); + densityFloorGlob = input.Get("Setup","densityFloor",0); + // delete file planet0.dat at initialization if we do not restart the simulation. + for(int ip=0; ip < data.planetarySystem->nbp ; ip++) { + std::string planetName, tqwkName; + std::stringstream pName, tName; + pName << "planet" << ip << ".dat"; + tName << "tqwk" << ip << ".dat"; + planetName = pName.str(); + tqwkName = tName.str(); + if (!(input.restartRequested)) { + if(idfx::prank==0) { + std::ofstream f; + f.open(planetName,std::ios::out); + f.close(); + std::ofstream ft; + ft.open(tqwkName,std::ios::out); + ft.close(); + } + } +/* + else { + if(idfx::prank==0) { + char ch; + int nline = input.restartFileNumber*input.Get("Output","dmp",0)/input.Get("Output","analysis",0); + int line = 1; + + // planet0.dat + std::ifstream fin(planetName); + std::ofstream fout; + fout.open("temp.dat", std::ios::out); + while(fin.get(ch)) + { + if(ch == '\n') + line++; + + if(line <= nline) // content not to be deleted + fout<("Output","dmp",0)/input.Get("Output","analysis",0); + line = 1; + while(fint.get(ch)) + { + if(ch == '\n') + line++; + + if(line <= nline) // content not to be deleted + foutt< 0: + integ = ( + np.nansum( + (data * dR)[k : kk + 1], + dtype="float64", + ) + ) + else: + integ = -( + np.nansum( + (data * dR)[kk : k + 1], + dtype="float64", + ) + ) + return integ + +# Phase equation for m,n +#(phi-wavenumber,order of the spiral) +def phaseEquation( + RR, + *, + resonance=-1, + m=mdom, + n=0, + Rp=Rp, + info=False, +): + phieq = -np.sign(RR-Rp)*np.pi/4/m + 2*np.pi*n/m -_integral( + RR, + resonance=resonance, + m=mdom, + Rp=Rp, + h0=h0, + flaring=flaring, + info=info, + ) + return phieq + +plot = False + +RwkzMin = find_nearest(Rmed, 0.7) +RwkzMax = find_nearest(Rmed, 1.35) + +spiralR = [] +spiralP = [] +spiralP_theo = [] +for ir in range(RwkzMin, RwkzMax+1): + if (Rmed[ir] < Rmm): + rho = ds.data["RHO"][ir,:,0] + spiralR.append(Rmed[ir]) + spiralP.append(phimed[find_nearest(rho,rho.max())]) + spiralP_theo.append(phaseEquation(Rmed[ir],resonance=-1)) + if (Rmed[ir] > Rmp): + rho = ds.data["RHO"][ir,:,0] + spiralR.append(Rmed[ir]) + spiralP.append(phimed[find_nearest(rho,rho.max())]) + spiralP_theo.append(phaseEquation(Rmed[ir],resonance=+1)) + +spiralR_theo = spiralR +spiralP = np.array(spiralP) +spiralP_theo = (np.array(spiralP_theo)/(np.pi))%2*np.pi-np.pi + +if plot: + fig, ax = plt.subplots(figsize=(5,4)) + ax.axvline(x=1.0, ls="--", c="k") + ax.scatter(spiralR_theo,spiralP_theo, c="r",marker="+", label="theoretical") + ax.scatter(spiralR,spiralP,c="k",marker="x", label=r"max $\rho$ simulation") + fig.tight_layout() + + ax.legend(frameon=False) + + fig2, ax2 = plt.subplots() + ax2.scatter(spiralR, 100*(spiralP-spiralP_theo)/spiralP_theo) + fig2.tight_layout() + + plt.show() + +mean = np.mean(100*(spiralP-spiralP_theo)/spiralP_theo) +std = np.std(100*(spiralP-spiralP_theo)/spiralP_theo) +error_mean = mean +error_dispersion = np.max([abs(mean+std),abs(mean-std)]) + +print("Mean Error=%.2f"%error_mean) +print("max(mean +/- std)=%.2f"%error_dispersion) +if error_mean<1.0 and error_dispersion<5.0: + print("SUCCESS!") + sys.exit(0) +else: + print("FAILURE!") + sys.exit(1) diff --git a/test/Planet/PlanetSpiral2D/setup.cpp b/test/Planet/PlanetSpiral2D/setup.cpp new file mode 100644 index 00000000..8d88670e --- /dev/null +++ b/test/Planet/PlanetSpiral2D/setup.cpp @@ -0,0 +1,474 @@ +#include +#include +#include "idefix.hpp" +#include "setup.hpp" +#include "planet.hpp" + +real wkzMinGlob; +real wkzMaxGlob; +real wkzDampingGlob; +real sigma0Glob; +real sigmaSlopeGlob; +real h0Glob; +real flaringIndexGlob; +real alphaGlob; +real densityFloorGlob; +real masstaperGlob; +real omegaGlob; + +/*********************************************/ +/** +Customized random number generator +Allow one to have consistant random numbers +generators on different architectures. +**/ +/*********************************************/ +real randm(void) { + const int a = 16807; + const int m = 2147483647; + static int in0 = 13763 + 2417*idfx::prank; + int q; + + /* find random number */ + q= (int) fmod((double) a * in0, m); + in0=q; + + return((real) ((double) q/(double)m)); +} + + +void MySoundSpeed(DataBlock &data, const real t, IdefixArray3D &cs) { + real h0 = h0Glob; + real flaringIndex = flaringIndexGlob; + IdefixArray1D x1=data.x[IDIR]; + idefix_for("MySoundSpeed",0,data.np_tot[KDIR],0,data.np_tot[JDIR],0,data.np_tot[IDIR], + KOKKOS_LAMBDA (int k, int j, int i) { + real R = x1(i); + cs(k,j,i) = h0*pow(R,flaringIndex-0.5); + }); +} + + +void MyViscosity(DataBlock &data, const real t, IdefixArray3D &eta1, IdefixArray3D &eta2) { + IdefixArray4D Vc=data.hydro->Vc; + IdefixArray1D x1=data.x[IDIR]; + real h0 = h0Glob; + real flaringIndex = flaringIndexGlob; + real alpha = alphaGlob; + idefix_for("MyViscosity",0,data.np_tot[KDIR],0,data.np_tot[JDIR],0,data.np_tot[IDIR], + KOKKOS_LAMBDA (int k, int j, int i) { + real R = x1(i); + real cs = h0*pow(R,flaringIndex-0.5); + eta1(k,j,i) = alpha*cs*h0*pow(R,flaringIndex+1)*Vc(RHO,k,j,i); + eta2(k,j,i) = ZERO_F; + }); + +} + +void Damping(Hydro *hydro, const real t, const real dtin) { + auto *data = hydro->data; + IdefixArray4D Vc = hydro->Vc; + IdefixArray4D Uc = hydro->Uc; + IdefixArray1D x1 = data->x[IDIR]; + IdefixArray1D x2 = data->x[JDIR]; + + real h0 = h0Glob; + real sigma0 = sigma0Glob; + real sigmaSlope = sigmaSlopeGlob; + real flaringIndex = flaringIndexGlob; + real omega = omegaGlob; + real gamma = hydro->eos->GetGamma(); + real dt = dtin; + bool isFargo = data->haveFargo; + + real rmin{data->mygrid->xbeg[0]}; + real wkzMin{wkzMinGlob}; + real rmax{data->mygrid->xend[0]}; + real wkzMax{wkzMaxGlob}; + real wkzDamping{wkzDampingGlob}; + idefix_for("MySourceTerm", + 0, data->np_tot[KDIR], + 0, data->np_tot[JDIR], + 0, data->np_tot[IDIR], + KOKKOS_LAMBDA (int k, int j, int i) { + real R = x1(i); + real Vk = 1.0/sqrt(R); + + /* + // Cooling function + real cs2 = h0*h0*pow(R,2*flaringIndex-1.0); + + // cooling time in local Keplerian units + real tau = 0.1*pow(R,1.5); + real Ptarget = cs2*Vc(RHO,k,j,i); + + Uc(ENG,k,j,i) += -dt*(Vc(PRS,k,j,i)-Ptarget)/(tau*(gamma-1.0)); + */ + + real lambda = 0.0; + + // Damping function for poloidal velocity field in damping zones + // Damp whatever is at RwkzMax + if (RwkzMax) { + lambda = 1.0/(wkzDamping*2.0*M_PI*pow(rmax,1.5))*pow(sin(M_PI*( (R-wkzMax) / (rmax-wkzMax) )/2.0),2.0); + } + + real rhoTarget = sigma0*pow(R,-sigmaSlope) ; + real vx2Target = 0.0; + if(!isFargo) { + vx2Target = Vk*sqrt(1.0-(1.0+sigmaSlope-2*flaringIndex)*h0*h0*pow(R,2*flaringIndex)) - omega * R; + } + + // relaxation + real drho = lambda*(Vc(RHO,k,j,i)-rhoTarget); + real dvx1 = lambda*Vc(RHO,k,j,i)*Vc(VX1,k,j,i); + real dvx2 = lambda*Vc(RHO,k,j,i)*(Vc(VX2,k,j,i)-vx2Target); + real dvx3 = lambda*Vc(RHO,k,j,i)*Vc(VX3,k,j,i); + + real dmx1 = dvx1 + Vc(VX1,k,j,i) * drho; + real dmx2 = dvx2 + Vc(VX2,k,j,i) * drho; + real dmx3 = dvx3 + Vc(VX3,k,j,i) * drho; + + // Kinetic energy fluctuations due to above damping + // must be compensated in total energy conservation + /* + real deng = 0.5*(Vc(VX1,k,j,i)*Vc(VX1,k,j,i) + +Vc(VX2,k,j,i)*Vc(VX2,k,j,i))*drho + +Vc(VX3,k,j,i)*Vc(VX3,k,j,i))*drho + + Vc(RHO,k,j,i) * ( + Vc(VX1,k,j,i)*dvx1 + + Vc(VX2,k,j,i)*dvx2 + + Vc(VX3,k,j,i)*dvx3 + ); + */ + + Uc(RHO,k,j,i) += -drho*dt; + Uc(MX1,k,j,i) += -dmx1*dt; + Uc(MX2,k,j,i) += -dmx2*dt; + Uc(MX3,k,j,i) += -dmx3*dt; + // Uc(ENG,k,j,i) += -deng*dt; + +}); + +} + +// User-defined boundaries +void UserdefBoundary(Hydro *hydro, int dir, BoundarySide side, real t) { + auto *data = hydro->data; + IdefixArray4D Vc = hydro->Vc; + IdefixArray1D x1 = data->x[IDIR]; + real sigmaSlope=sigmaSlopeGlob; + real omega = omegaGlob; + + if(dir==IDIR) { + int ighost,ibeg,iend,sign; + if(side == left) { + ighost = data->beg[IDIR]; + ibeg = 0; + iend = data->beg[IDIR]; + sign=-1; + //return; + } + else if(side==right) { + ighost = data->end[IDIR]-1; + ibeg=data->end[IDIR]; + iend=data->np_tot[IDIR]; + sign=1; + } + + + idefix_for("UserDefBoundary", + 0, data->np_tot[KDIR], + 0, data->np_tot[JDIR], + ibeg, iend, + KOKKOS_LAMBDA (int k, int j, int i) { + real R=x1(i); + real R0=x1(ighost); + real Vk = 1.0/sqrt(R); + // real cs2 = h0*h0*pow(R,2*flaringIndex-1.0); + + Vc(RHO,k,j,i) = Vc(RHO,k,j,ighost)*pow(R/R0,-sigmaSlope) ; + // Vc(PRS,k,j,i) = cs2*Vc(RHO,k,j,i); + + if( sign*Vc(VX1,k,j,ighost) > 0.0) { + Vc(VX1,k,j,i) = Vc(VX1,k,j,ighost); + } else { + Vc(VX1,k,j,i) = - Vc(VX1,k,j,2*ighost-i+sign); + } + + Vc(VX2,k,j,i) = Vc(VX2,k,j,ighost)*pow(R/R0,-0.5) - omega*R ; + Vc(VX3,k,j,i) = Vc(VX3,k,j,ighost); + + // Vc(RHO,k,j,i) = sigma0*pow(R,-sigmaSlope) ; + // Vc(VX2,k,j,i) = Vk*sqrt(1.0-(1.0+sigmaSlope-2*flaringIndex)*h0*h0*pow(R,2*flaringIndex)) - omega * R; + // Vc(VX3,k,j,i) = Vc(VX3,k,j,2*ighost-i+sign); + }); + } +} + +void FargoVelocity(DataBlock &data, IdefixArray2D &Vphi) { + IdefixArray1D x1 = data.x[IDIR]; + real sigmaSlope=sigmaSlopeGlob; + real h0=h0Glob; + real flaringIndex=flaringIndexGlob; + real omega = omegaGlob; + + idefix_for("FargoVphi",0,data.np_tot[KDIR], 0, data.np_tot[IDIR], + KOKKOS_LAMBDA (int k, int i) { + real R = x1(i); + real Vk = 1.0/sqrt(R); + // Vphi(k,i) = 1.0/sqrt(R) - omega*R; + Vphi(k,i) = Vk*sqrt(1.0-(1.0+sigmaSlope-2*flaringIndex)*h0*h0*pow(R,2*flaringIndex)) - omega * R; + }); +} + +// Analyse data to produce an ascii output +void Analysis(DataBlock & data) { + // Mirror data on Host + DataBlockHost d(data); + + // Sync it + d.SyncFromDevice(); + +// data.DumpToFile("totou"); + + for(int ip=0; ip < data.planetarySystem->nbp ; ip++) { + // Get the orbital parameters + real timeStep = data.dt; + real xp = data.planetarySystem->planet[ip].getXp(); + real yp = data.planetarySystem->planet[ip].getYp(); + real zp = data.planetarySystem->planet[ip].getZp(); + real vxp = data.planetarySystem->planet[ip].getVxp(); + real vyp = data.planetarySystem->planet[ip].getVyp(); + real vzp = data.planetarySystem->planet[ip].getVzp(); + real qp = data.planetarySystem->planet[ip].getMp(); + real time = data.t; + + std::string planetName, tqwkName; + std::stringstream pName, tName; + pName << "planet" << ip << ".dat"; + tName << "tqwk" << ip << ".dat"; + planetName = pName.str(); + tqwkName = tName.str(); + // Write the data in ascii to our file + if(idfx::prank==0) { + std::ofstream f; + f.open(planetName,std::ios::app); + f.precision(10); + f << std::scientific << timeStep << " " << xp << " " << yp << " " << zp << " " << vxp << " " << vyp << " " << vzp << " " << qp << " " << time << std::endl; + f.close(); + } + + // Force force = {{0.0,0.0,0.0},{0.0,0.0,0.0},{0.0,0.0,0.0},{0.0,0.0,0.0}}; + Force &force = data.planetarySystem->planet[ip].m_force; + bool isp = true; + data.planetarySystem->planet[ip].computeForce(data,isp); + + // Get the torque and work + // real fxi = force.f_inner[0]; + // real fyi = force.f_inner[1]; + // real fzi = force.f_inner[2]; + // real fxo = force.f_outer[0]; + // real fyo = force.f_outer[1]; + // real fzo = force.f_outer[2]; + // real fxhi = force.f_ex_inner[0]; + // real fyhi = force.f_ex_inner[1]; + // real fzhi = force.f_ex_inner[2]; + // real fxho = force.f_ex_outer[0]; + // real fyho = force.f_ex_outer[1]; + // real fzho = force.f_ex_outer[2]; + + real tq_inner = xp*force.f_inner[1]-yp*force.f_inner[0]; + real tq_outer = xp*force.f_outer[1]-yp*force.f_outer[0]; + real tq_ex_inner = xp*force.f_ex_inner[1]-yp*force.f_ex_inner[0]; + real tq_ex_outer = xp*force.f_ex_outer[1]-yp*force.f_ex_outer[0]; + real wk_inner = vxp*force.f_inner[0]+vyp*force.f_inner[1]; + real wk_outer = vxp*force.f_outer[0]+vyp*force.f_outer[1]; + real wk_ex_inner = vxp*force.f_ex_inner[0]+vyp*force.f_ex_inner[1]; + real wk_ex_outer = vxp*force.f_ex_outer[0]+vyp*force.f_ex_outer[1]; + + // Write the data in ascii to our file + if(idfx::prank==0) { + std::ofstream ft; + ft.open(tqwkName,std::ios::app); + ft.precision(10); + // ft << std::scientific << timeStep << " " << fxi << " " << fyi << " " << fzi << " " << fxo << " " << fyo << " " << fzo << " " << fxhi << " " << fyhi << " " << fzhi << " " << fxho << " " << fyho << " " << fzho << " " << time << std::endl; + ft << std::scientific << timeStep << " " << tq_inner << " " << tq_outer << " " << tq_ex_inner << " " << tq_ex_outer << " " << wk_inner << " " << wk_outer << " " << wk_ex_inner << " " << wk_ex_outer << " " << time << std::endl; + ft.close(); + } + } +} + +/* +// Compute user variables which will be written in vtk files +void ComputeUserVars(DataBlock & data, UserDefVariablesContainer &variables) { + // Mirror data on Host + real h0=h0Glob; + real flaringIndex=flaringIndexGlob; + DataBlockHost d(data); + + // Sync it + d.SyncFromDevice(); + + // Make references to the user-defined arrays (variables is a container of IdefixHostArray3D) + // Note that the labels should match the variable names in the input file + IdefixHostArray3D PRS = variables["PRS"]; + + for(int k = 0; k < d.np_tot[KDIR] ; k++) { + for(int j = 0; j < d.np_tot[JDIR] ; j++) { + for(int i = 0; i < d.np_tot[IDIR] ; i++) { + real R = d.x[IDIR](i); + real cs2 = h0*h0*pow(R,2*flaringIndex-1.); + PRS(k,j,i) = cs2*d.Vc(RHO,k,j,i); + } + } + } +} +*/ + +// Default constructor +// Initialisation routine. Can be used to allocate +// Arrays or variables which are used later on +Setup::Setup(Input &input, Grid &grid, DataBlock &data, Output &output)// : m_planet(0)//, Planet &planet) +{ + // Set the function for userdefboundary + data.hydro->EnrollUserDefBoundary(&UserdefBoundary); + data.hydro->EnrollUserSourceTerm(&Damping); + data.hydro->EnrollIsoSoundSpeed(&MySoundSpeed); + + if(data.haveFargo) + data.fargo->EnrollVelocity(&FargoVelocity); + if(data.hydro->haveRotation) { + omegaGlob = data.hydro->OmegaZ; + } else { + omegaGlob = 0.0; + } + // Enroll the analysis function + output.EnrollAnalysis(&Analysis); + // Enroll our user-defined variables + // output.EnrollUserDefVariables(&ComputeUserVars); + + wkzMinGlob = input.Get("Setup","wkzMin",0); + wkzMaxGlob = input.Get("Setup","wkzMax",0); + wkzDampingGlob = input.Get("Setup","wkzDamping",0); + sigma0Glob = input.Get("Setup","sigma0",0); + sigmaSlopeGlob = input.Get("Setup","sigmaSlope",0); + h0Glob = input.Get("Setup","h0",0); + flaringIndexGlob = input.Get("Setup","flaringIndex",0); + densityFloorGlob = input.Get("Setup","densityFloor",0); + masstaperGlob = input.Get("Planet","masstaper",0); + // delete file planet0.dat at initialization if we do not restart the simulation. + for(int ip=0; ip < data.planetarySystem->nbp ; ip++) { + std::string planetName, tqwkName; + std::stringstream pName, tName; + pName << "planet" << ip << ".dat"; + tName << "tqwk" << ip << ".dat"; + planetName = pName.str(); + tqwkName = tName.str(); + if (!(input.restartRequested)) { + if(idfx::prank==0) { + std::ofstream f; + f.open(planetName,std::ios::out); + f.close(); + std::ofstream ft; + ft.open(tqwkName,std::ios::out); + ft.close(); + } + } + /* + else { + if(idfx::prank==0) { + char ch; + int nline = input.restartFileNumber*input.Get("Output","dmp",0)/input.Get("Output","analysis",0); + int line = 1; + + // planet0.dat + std::ifstream fin(planetName); + std::ofstream fout; + fout.open("temp.dat", std::ios::out); + while(fin.get(ch)) + { + if(ch == '\n') + line++; + + if(line <= nline) // content not to be deleted + fout<("Output","dmp",0)/input.Get("Output","analysis",0); + line = 1; + while(fint.get(ch)) + { + if(ch == '\n') + line++; + + if(line <= nline) // content not to be deleted + foutt< +#include +#include "idefix.hpp" +#include "setup.hpp" +#include "planet.hpp" + +real wkzMinGlob; +real wkzMaxGlob; +real wkzDampingGlob; +real sigma0Glob; +real sigmaSlopeGlob; +real h0Glob; +real flaringIndexGlob; +real alphaGlob; +real densityFloorGlob; +real masstaperGlob; +real omegaGlob; + +/*********************************************/ +/** +Customized random number generator +Allow one to have consistant random numbers +generators on different architectures. +**/ +/*********************************************/ +real randm(void) { + const int a = 16807; + const int m = 2147483647; + static int in0 = 13763 + 2417*idfx::prank; + int q; + + /* find random number */ + q= (int) fmod((double) a * in0, m); + in0=q; + + return((real) ((double) q/(double)m)); +} + +void MySoundSpeed(DataBlock &data, const real t, IdefixArray3D &cs) { + real h0 = h0Glob; + real flaringIndex = flaringIndexGlob; + IdefixArray1D x1=data.x[IDIR]; + IdefixArray1D x2=data.x[JDIR]; + idefix_for("MySoundSpeed",0,data.np_tot[KDIR],0,data.np_tot[JDIR],0,data.np_tot[IDIR], + KOKKOS_LAMBDA (int k, int j, int i) { + real R = x1(i)*sin(x2(j)); + cs(k,j,i) = h0*pow(R,flaringIndex-0.5); + }); +} + +void Damping(Hydro *hydro, const real t, const real dtin) { + auto *data = hydro->data; + IdefixArray4D Vc = hydro->Vc; + IdefixArray4D Uc = hydro->Uc; + IdefixArray1D x1 = data->x[IDIR]; + IdefixArray1D x2 = data->x[JDIR]; + + real h0 = h0Glob; + real sigma0 = sigma0Glob; + real sigmaSlope = sigmaSlopeGlob; + real flaringIndex = flaringIndexGlob; + real omega = omegaGlob; + real gamma = hydro->eos->GetGamma(); + real pexp = -sigmaSlope-flaringIndex-1; + real qexp = 2*flaringIndex-1; + real dt = dtin; + bool isFargo = data->haveFargo; + + real rmin{data->mygrid->xbeg[0]}; + real wkzMin{wkzMinGlob}; + real rmax{data->mygrid->xend[0]}; + real wkzMax{wkzMaxGlob}; + real wkzDamping{wkzDampingGlob}; + idefix_for("Damping", + 0, data->np_tot[KDIR], + 0, data->np_tot[JDIR], + 0, data->np_tot[IDIR], + KOKKOS_LAMBDA (int k, int j, int i) { + real r = x1(i); + real th = x2(j); + real R = r*sin(th); + real cs2 = h0*h0*pow(R,2*flaringIndex-1); + real Rmin = rmin*sin(th); + real Rmax = rmax*sin(th); + real wkzRmin = wkzMin*sin(th); + real wkzRmax = wkzMax*sin(th); + real Vk = 1.0/sqrt(R); + + /* + // Cooling function + real cs2 = h0*h0*pow(R,2*flaringIndex-1.0); + + // cooling time in local Keplerian units + real tau = 0.1*pow(R,1.5); + real Ptarget = cs2*Vc(RHO,k,j,i); + + Uc(ENG,k,j,i) += -dt*(Vc(PRS,k,j,i)-Ptarget)/(tau*(gamma-1.0)); + */ + + real lambda = 0.0; + + // Damping function for poloidal velocity field in damping zones + // Damp whatever is at RwkzRmax + if (RwkzRmax) { + lambda = 1.0/(wkzDamping*2.0*M_PI*pow(Rmax,1.5))*pow(sin(M_PI*( (R-wkzRmax) / (Rmax-wkzRmax) )/2.0),2.0); + } + + real rhoTarget = sigma0/sqrt(2.0*M_PI)/(h0*pow(R,flaringIndex+1))*pow(R,-sigmaSlope) * exp(1.0/ (cs2) * (1.0/r-1.0/R)) ; + real vx3Target = 0.0; + if(!isFargo) { + vx3Target = Vk*sqrt((pexp+qexp)*h0*h0*pow(R,2*flaringIndex)+(1+qexp)-qexp*R/r) - omega * R; + } + + // relaxation + real drho = lambda*(Vc(RHO,k,j,i)-rhoTarget); + real dvx1 = lambda*Vc(RHO,k,j,i)*Vc(VX1,k,j,i); + real dvx2 = lambda*Vc(RHO,k,j,i)*Vc(VX2,k,j,i); + real dvx3 = lambda*Vc(RHO,k,j,i)*(Vc(VX3,k,j,i)-vx3Target); + + real dmx1 = dvx1 + Vc(VX1,k,j,i) * drho; + real dmx2 = dvx2 + Vc(VX2,k,j,i) * drho; + real dmx3 = dvx3 + Vc(VX3,k,j,i) * drho; + + // Kinetic energy fluctuations due to above damping + // must be compensated in total energy conservation + /* + real deng = 0.5*(Vc(VX1,k,j,i)*Vc(VX1,k,j,i) + +Vc(VX2,k,j,i)*Vc(VX2,k,j,i))*drho + +Vc(VX3,k,j,i)*Vc(VX3,k,j,i))*drho + + Vc(RHO,k,j,i) * ( + Vc(VX1,k,j,i)*dvx1 + + Vc(VX2,k,j,i)*dvx2 + + Vc(VX3,k,j,i)*dvx3 + ); + */ + + Uc(RHO,k,j,i) += -drho*dt; + Uc(MX1,k,j,i) += -dmx1*dt; + Uc(MX2,k,j,i) += -dmx2*dt; + Uc(MX3,k,j,i) += -dmx3*dt; + // Uc(ENG,k,j,i) += -deng*dt; + +}); + +} + +// User-defined boundaries +void UserdefBoundary(Hydro *hydro, int dir, BoundarySide side, real t) { + auto *data = hydro->data; + IdefixArray4D Vc = hydro->Vc; + IdefixArray1D x1 = data->x[IDIR]; + IdefixArray1D x2 = data->x[JDIR]; + real h0=h0Glob; + real flaringIndex=flaringIndexGlob; + real sigmaSlope=sigmaSlopeGlob; + real omega = omegaGlob; + real pexp = -sigmaSlope-flaringIndex-1; + real qexp = 2*flaringIndex-1; + + if(dir==IDIR) { + int ighost,ibeg,iend; + if(side == left) { + ighost = data->beg[IDIR]; + ibeg = 0; + iend = data->beg[IDIR]; + //return; + } + else if(side==right) { + ighost = data->end[IDIR]-1; + ibeg=data->end[IDIR]; + iend=data->np_tot[IDIR]; + } + + + idefix_for("UserDefBoundary", + 0, data->np_tot[KDIR], + 0, data->np_tot[JDIR], + ibeg, iend, + KOKKOS_LAMBDA (int k, int j, int i) { + real r = x1(i); + real rghost = x1(ighost); + real R = x1(i)*sin(x2(j)); + real Rghost = x1(ighost)*sin(x2(j)); + real Vk = 1.0/sqrt(R); + real cs2 = h0*h0*pow(R,2*flaringIndex-1); + + Vc(RHO,k,j,i) = Vc(RHO,k,j,ighost) * pow(R/Rghost,-sigmaSlope-flaringIndex-1) * exp(1.0/cs2 * (1.0/r - 1.0/rghost + 1.0/Rghost - 1.0/R)) ; + Vc(VX1,k,j,i) = ZERO_F;//- Vc(VX1,k,j,2*ighost-i+sign); + Vc(VX2,k,j,i) = ZERO_F;//Vc(VX2,k,j,2*ighost-i+sign); + Vc(VX3,k,j,i) = Vk*sqrt((pexp+qexp)*h0*h0*pow(R,2*flaringIndex)+(1+qexp)-qexp*R/r) - omega * R; + +// Vc(RHO,k,j,i) = sigma0/sqrt(2.0*M_PI)/(h0)*pow(R,-sigmaSlope-1.5) * exp(1.0/cs2 * (1.0/r - 1.0/R)) ; +// Vc(VX1,k,j,i) = - Vc(VX1,k,j,2*ighost-i+1); +// Vc(VX2,k,j,i) = ZERO_F; +// Vc(VX3,k,j,i) = Vk*sqrt(1 - (1.5+sigmaSlope)*cs2*R); + + }); + } + + if( dir==JDIR) { + IdefixArray4D Vc = hydro->Vc; + int jghost; + int jbeg,jend; + // UPPER LAYER + if(side == left) { + jghost = data->beg[JDIR]; + jbeg = 0; + jend = data->beg[JDIR]; + //return; + idefix_for("UserDefBoundary", + 0, data->np_tot[KDIR], + jbeg, jend, + 0, data->np_tot[IDIR], + KOKKOS_LAMBDA (int k, int j, int i) { + real r = x1(i); + real R = x1(i)*sin(x2(j)); + real Rghost = x1(i)*sin(x2(jghost)); + real Vk = 1.0/sqrt(R); + real cs2 = h0*h0*pow(R,2*flaringIndex-1); + + Vc(RHO,k,j,i) = Vc(RHO,k,jghost,i) * pow(R/Rghost,-sigmaSlope-flaringIndex-1) * exp(1.0/(cs2)*(1.0/Rghost - 1.0/R)) ; + // Vc(RHO,k,j,i) = sigma0/sqrt(2.0*M_PI)/(h0)*pow(R,-sigmaSlope-1.5) * exp(1.0/(cs2) * (1.0/r - 1.0/R)) ; + Vc(VX1,k,j,i) = ZERO_F; //Vc(VX1,k,2*jghost-j-1,i); + Vc(VX2,k,j,i) = ZERO_F; + Vc(VX3,k,j,i) = Vk*sqrt((pexp+qexp)*h0*h0*pow(R,2*flaringIndex)+(1+qexp)-qexp*R/r) - omega * R; + }); + } + // MIDPLANE + else if(side==right) { + jghost = data->end[JDIR]-1; + jbeg = data->end[JDIR]; + jend = data->np_tot[JDIR]; + idefix_for("UserDefBoundary", + 0, data->np_tot[KDIR], + jbeg, jend, + 0, data->np_tot[IDIR], + KOKKOS_LAMBDA (int k, int j, int i) { + real r = x1(i); + real R = x1(i)*sin(x2(j)); + real Vk = 1.0/sqrt(R); + + Vc(RHO,k,j,i) = Vc(RHO,k,2*jghost-j+1,i); + Vc(VX1,k,j,i) = Vc(VX1,k,2*jghost-j+1,i); + Vc(VX2,k,j,i) = - Vc(VX2,k,2*jghost-j+1,i); + Vc(VX3,k,j,i) = Vc(VX3,k,2*jghost-j+1,i); + }); + } + } + +} + +void FargoVelocity(DataBlock &data, IdefixArray2D &Vphi) { + IdefixArray1D x1 = data.x[IDIR]; + IdefixArray1D x2 = data.x[JDIR]; + real h0=h0Glob; + real flaringIndex=flaringIndexGlob; + real sigmaSlope=sigmaSlopeGlob; + real pexp = -sigmaSlope-flaringIndex-1; + real qexp = 2*flaringIndex-1; + real omega = omegaGlob; + idefix_for("FargoVphi",0,data.np_tot[JDIR], 0, data.np_tot[IDIR], + KOKKOS_LAMBDA (int j, int i) { + real r = x1(i); + real R = x1(i)*sin(x2(j)); + real Vk = 1.0/sqrt(R); + Vphi(j,i) = Vk*sqrt((pexp+qexp)*h0*h0*pow(R,2*flaringIndex)+(1+qexp)-qexp*R/r) - omega * R; + }); +} + +// Analyse data to produce an ascii output +void Analysis(DataBlock & data) { + // Mirror data on Host + DataBlockHost d(data); + + // Sync it + d.SyncFromDevice(); + + for(int ip=0; ip < data.planetarySystem->nbp ; ip++) { + // Get the orbital parameters + real timeStep = data.dt; + real xp = data.planetarySystem->planet[ip].getXp(); + real yp = data.planetarySystem->planet[ip].getYp(); + real zp = data.planetarySystem->planet[ip].getZp(); + real vxp = data.planetarySystem->planet[ip].getVxp(); + real vyp = data.planetarySystem->planet[ip].getVyp(); + real vzp = data.planetarySystem->planet[ip].getVzp(); + real qp = data.planetarySystem->planet[ip].getMp(); + real time = data.t; + + std::string planetName, tqwkName; + std::stringstream pName, tName; + pName << "planet" << ip << ".dat"; + tName << "tqwk" << ip << ".dat"; + planetName = pName.str(); + tqwkName = tName.str(); + // Write the data in ascii to our file + if(idfx::prank==0) { + std::ofstream f; + f.open(planetName,std::ios::app); + f.precision(10); + f << std::scientific << timeStep << " " << xp << " " << yp << " " << zp << " " << vxp << " " << vyp << " " << vzp << " " << qp << " " << time << std::endl; + f.close(); + } + + // Force force = {{0.0,0.0,0.0},{0.0,0.0,0.0},{0.0,0.0,0.0},{0.0,0.0,0.0}}; + Force &force = data.planetarySystem->planet[ip].m_force; + bool isp = true; + data.planetarySystem->planet[ip].computeForce(data,isp); + + // Get the torque and work + // real fxi = force.f_inner[0]; + // real fyi = force.f_inner[1]; + // real fzi = force.f_inner[2]; + // real fxo = force.f_outer[0]; + // real fyo = force.f_outer[1]; + // real fzo = force.f_outer[2]; + // real fxhi = force.f_ex_inner[0]; + // real fyhi = force.f_ex_inner[1]; + // real fzhi = force.f_ex_inner[2]; + // real fxho = force.f_ex_outer[0]; + // real fyho = force.f_ex_outer[1]; + // real fzho = force.f_ex_outer[2]; + + real tq_inner = xp*force.f_inner[1]-yp*force.f_inner[0]; + real tq_outer = xp*force.f_outer[1]-yp*force.f_outer[0]; + real tq_ex_inner = xp*force.f_ex_inner[1]-yp*force.f_ex_inner[0]; + real tq_ex_outer = xp*force.f_ex_outer[1]-yp*force.f_ex_outer[0]; + real wk_inner = vxp*force.f_inner[0]+vyp*force.f_inner[1]; + real wk_outer = vxp*force.f_outer[0]+vyp*force.f_outer[1]; + real wk_ex_inner = vxp*force.f_ex_inner[0]+vyp*force.f_ex_inner[1]; + real wk_ex_outer = vxp*force.f_ex_outer[0]+vyp*force.f_ex_outer[1]; + + // Write the data in ascii to our file + if(idfx::prank==0) { + std::ofstream ft; + ft.open(tqwkName,std::ios::app); + ft.precision(10); + // ft << std::scientific << timeStep << " " << fxi << " " << fyi << " " << fzi << " " << fxo << " " << fyo << " " << fzo << " " << fxhi << " " << fyhi << " " << fzhi << " " << fxho << " " << fyho << " " << fzho << " " << time << std::endl; + ft << std::scientific << timeStep << " " << tq_inner << " " << tq_outer << " " << tq_ex_inner << " " << tq_ex_outer << " " << wk_inner << " " << wk_outer << " " << wk_ex_inner << " " << wk_ex_outer << " " << time << std::endl; + ft.close(); + } + } +} + +// Default constructor +// Initialisation routine. Can be used to allocate +// Arrays or variables which are used later on +Setup::Setup(Input &input, Grid &grid, DataBlock &data, Output &output)// : m_planet(0)//, Planet &planet) +{ + // Set the function for userdefboundary + data.hydro->EnrollUserDefBoundary(&UserdefBoundary); + data.hydro->EnrollUserSourceTerm(&Damping); + data.hydro->EnrollIsoSoundSpeed(&MySoundSpeed); + if(data.haveFargo) + data.fargo->EnrollVelocity(&FargoVelocity); + if(data.hydro->haveRotation) { + omegaGlob = data.hydro->OmegaZ; + } else { + omegaGlob = 0.0; + } + // Enroll the analysis function + output.EnrollAnalysis(&Analysis); + wkzMinGlob = input.Get("Setup","wkzMin",0); + wkzMaxGlob = input.Get("Setup","wkzMax",0); + wkzDampingGlob = input.Get("Setup","wkzDamping",0); + sigma0Glob = input.Get("Setup","sigma0",0); + sigmaSlopeGlob = input.Get("Setup","sigmaSlope",0); + h0Glob = input.Get("Setup","h0",0); + flaringIndexGlob = input.Get("Setup","flaringIndex",0); + densityFloorGlob = input.Get("Setup","densityFloor",0); + masstaperGlob = input.Get("Planet","masstaper",0); + // delete file planet0.dat at initialization if we do not restart the simulation. + for(int ip=0; ip < data.planetarySystem->nbp ; ip++) { + std::string planetName, tqwkName; + std::stringstream pName, tName; + pName << "planet" << ip << ".dat"; + tName << "tqwk" << ip << ".dat"; + planetName = pName.str(); + tqwkName = tName.str(); + if (!(input.restartRequested)) { + if(idfx::prank==0) { + std::ofstream f; + f.open(planetName,std::ios::out); + f.close(); + std::ofstream ft; + ft.open(tqwkName,std::ios::out); + ft.close(); + } + } + /* + else { + if(idfx::prank==0) { + char ch; + int nline = input.restartFileNumber*input.Get("Output","dmp",0)/input.Get("Output","analysis",0); + int line = 1; + + // planet0.dat + std::ifstream fin(planetName); + std::ofstream fout; + fout.open("temp.dat", std::ios::out); + while(fin.get(ch)) + { + if(ch == '\n') + line++; + + if(line <= nline) // content not to be deleted + fout<("Output","dmp",0)/input.Get("Output","analysis",0); + line = 1; + while(fint.get(ch)) + { + if(ch == '\n') + line++; + + if(line <= nline) // content not to be deleted + foutt< +#include +#include "idefix.hpp" +#include "setup.hpp" +#include "planet.hpp" + +real wkzMinGlob; +real wkzMaxGlob; +real wkzDampingGlob; +real sigma0Glob; +real sigmaSlopeGlob; +real h0Glob; +real flaringIndexGlob; +real alphaGlob; +real densityFloorGlob; +real omegaGlob; + +/*********************************************/ +/** +Customized random number generator +Allow one to have consistant random numbers +generators on different architectures. +**/ +/*********************************************/ +real randm(void) { + const int a = 16807; + const int m = 2147483647; + static int in0 = 13763 + 2417*idfx::prank; + int q; + + /* find random number */ + q= (int) fmod((double) a * in0, m); + in0=q; + + return((real) ((double) q/(double)m)); +} + +void MySoundSpeed(DataBlock &data, const real t, IdefixArray3D &cs) { + real h0 = h0Glob; + real flaringIndex = flaringIndexGlob; + IdefixArray1D x1=data.x[IDIR]; + idefix_for("MySoundSpeed",0,data.np_tot[KDIR],0,data.np_tot[JDIR],0,data.np_tot[IDIR], + KOKKOS_LAMBDA (int k, int j, int i) { + real R = x1(i); + cs(k,j,i) = h0*pow(R,flaringIndex-0.5); + }); +} + +void Damping(Hydro *hydro, const real t, const real dtin) { + auto *data = hydro->data; + IdefixArray4D Vc = hydro->Vc; + IdefixArray4D Uc = hydro->Uc; + IdefixArray1D x1 = data->x[IDIR]; + IdefixArray1D x2 = data->x[JDIR]; + + real h0 = h0Glob; + real sigma0 = sigma0Glob; + real sigmaSlope = sigmaSlopeGlob; + real flaringIndex = flaringIndexGlob; + real omega = omegaGlob; + real gamma = hydro->eos->GetGamma(); + real dt = dtin; + bool isFargo = data->haveFargo; + + real rmin{data->mygrid->xbeg[0]}; + real wkzMin{wkzMinGlob}; + real rmax{data->mygrid->xend[0]}; + real wkzMax{wkzMaxGlob}; + real wkzDamping{wkzDampingGlob}; + idefix_for("MySourceTerm", + 0, data->np_tot[KDIR], + 0, data->np_tot[JDIR], + 0, data->np_tot[IDIR], + KOKKOS_LAMBDA (int k, int j, int i) { + real R = x1(i); + real Vk = 1.0/sqrt(R); + + /* + // Cooling function + real cs2 = h0*h0*pow(R,2*flaringIndex-1.0); + + // cooling time in local Keplerian units + real tau = 0.1*pow(R,1.5); + real Ptarget = cs2*Vc(RHO,k,j,i); + + Uc(ENG,k,j,i) += -dt*(Vc(PRS,k,j,i)-Ptarget)/(tau*(gamma-1.0)); + */ + + real lambda = 0.0; + + // Damping function for poloidal velocity field in damping zones + // Damp whatever is at RwkzMax + if (RwkzMax) { + lambda = 1.0/(wkzDamping*2.0*M_PI*pow(rmax,1.5))*pow(sin(M_PI*( (R-wkzMax) / (rmax-wkzMax) )/2.0),2.0); + } + + real rhoTarget = sigma0*pow(R,-sigmaSlope) ; + real vx2Target = 0.0; + if(!isFargo) { + vx2Target = Vk*sqrt(1.0-(1.0+sigmaSlope-2*flaringIndex)*h0*h0*pow(R,2*flaringIndex)) - omega * R; + } + + // relaxation + real drho = lambda*(Vc(RHO,k,j,i)-rhoTarget); + real dvx1 = lambda*Vc(RHO,k,j,i)*Vc(VX1,k,j,i); + real dvx2 = lambda*Vc(RHO,k,j,i)*(Vc(VX2,k,j,i)-vx2Target); + real dvx3 = lambda*Vc(RHO,k,j,i)*Vc(VX3,k,j,i); + + real dmx1 = dvx1 + Vc(VX1,k,j,i) * drho; + real dmx2 = dvx2 + Vc(VX2,k,j,i) * drho; + real dmx3 = dvx3 + Vc(VX3,k,j,i) * drho; + + // Kinetic energy fluctuations due to above damping + // must be compensated in total energy conservation + /* + real deng = 0.5*(Vc(VX1,k,j,i)*Vc(VX1,k,j,i) + +Vc(VX2,k,j,i)*Vc(VX2,k,j,i))*drho + +Vc(VX3,k,j,i)*Vc(VX3,k,j,i))*drho + + Vc(RHO,k,j,i) * ( + Vc(VX1,k,j,i)*dvx1 + + Vc(VX2,k,j,i)*dvx2 + + Vc(VX3,k,j,i)*dvx3 + ); + */ + + Uc(RHO,k,j,i) += -drho*dt; + Uc(MX1,k,j,i) += -dmx1*dt; + Uc(MX2,k,j,i) += -dmx2*dt; + Uc(MX3,k,j,i) += -dmx3*dt; + // Uc(ENG,k,j,i) += -deng*dt; + +}); + +} + +// User-defined boundaries +void UserdefBoundary(Hydro *hydro, int dir, BoundarySide side, real t) { + DataBlock &data = *hydro->data; + IdefixArray4D Vc = hydro->Vc; + IdefixArray1D x1 = data.x[IDIR]; + real sigmaSlope=sigmaSlopeGlob; + real omega = omegaGlob; + + if(dir==IDIR) { + int ighost,ibeg,iend,sign; + if(side == left) { + ighost = data.beg[IDIR]; + ibeg = 0; + iend = data.beg[IDIR]; + sign=-1; + //return; + } + else if(side==right) { + ighost = data.end[IDIR]-1; + ibeg=data.end[IDIR]; + iend=data.np_tot[IDIR]; + sign=1; + } + + + idefix_for("UserDefBoundary",0,data.np_tot[KDIR],0,data.np_tot[JDIR],ibeg,iend, + KOKKOS_LAMBDA (int k, int j, int i) { + real R=x1(i); + real R0=x1(ighost); + real Vk = 1.0/sqrt(R); + + Vc(RHO,k,j,i) = Vc(RHO,k,j,ighost)*pow(R/R0,-sigmaSlope) ; + Vc(VX1,k,j,i) = - Vc(VX1,k,j,2*ighost-i+sign); + Vc(VX2,k,j,i) = Vc(VX2,k,j,ighost)*pow(R/R0,-0.5) - omega*R ; + Vc(VX3,k,j,i) = Vc(VX3,k,j,2*ighost-i+sign); + }); + } +} + +void FargoVelocity(DataBlock &data, IdefixArray2D &Vphi) { + IdefixArray1D x1 = data.x[IDIR]; + real omega = omegaGlob; + + idefix_for("FargoVphi",0,data.np_tot[KDIR], 0, data.np_tot[IDIR], + KOKKOS_LAMBDA (int k, int i) { + real R = x1(i); + Vphi(k,i) = 1.0/sqrt(R) - omega*R; + }); +} + +// Analyse data to produce an ascii output +void Analysis(DataBlock & data) { + // Mirror data on Host + DataBlockHost d(data); + + // Sync it + d.SyncFromDevice(); + +// data.DumpToFile("totou"); + + for(int ip=0; ip < data.planetarySystem->nbp ; ip++) { + // Get the orbital parameters + real timeStep = data.dt; + real xp = data.planetarySystem->planet[ip].getXp(); + real yp = data.planetarySystem->planet[ip].getYp(); + real zp = data.planetarySystem->planet[ip].getZp(); + real vxp = data.planetarySystem->planet[ip].getVxp(); + real vyp = data.planetarySystem->planet[ip].getVyp(); + real vzp = data.planetarySystem->planet[ip].getVzp(); + real qp = data.planetarySystem->planet[ip].getMp(); + real time = data.t; + bool isActive = data.planetarySystem->planet[ip].getIsActive(); + // idfx::cout << "isactive: " << isActive << std::endl; + + std::string isActiveName; + std::stringstream iaName; + iaName << "isactive" << ip << ".dat"; + isActiveName = iaName.str(); + // Write the data in ascii to our file + if(idfx::prank==0) { + std::ofstream fi; + fi.open(isActiveName,std::ios::app); + fi.precision(10); + fi << std::scientific << time << " " << isActive << std::endl; + fi.close(); + } + + std::string planetName, tqwkName; + std::stringstream pName, tName; + pName << "planet" << ip << ".dat"; + tName << "tqwk" << ip << ".dat"; + planetName = pName.str(); + tqwkName = tName.str(); + // Write the data in ascii to our file + if(idfx::prank==0) { + std::ofstream f; + f.open(planetName,std::ios::app); + f.precision(10); + f << std::scientific << timeStep << " " << xp << " " << yp << " " << zp << " " << vxp << " " << vyp << " " << vzp << " " << qp << " " << time << std::endl; + f.close(); + } + + // Force force = {{0.0,0.0,0.0},{0.0,0.0,0.0},{0.0,0.0,0.0},{0.0,0.0,0.0}}; + Force &force = data.planetarySystem->planet[ip].m_force; + bool isp = true; + data.planetarySystem->planet[ip].computeForce(data,isp); + + // Get the torque and work + // real fxi = force.f_inner[0]; + // real fyi = force.f_inner[1]; + // real fzi = force.f_inner[2]; + // real fxo = force.f_outer[0]; + // real fyo = force.f_outer[1]; + // real fzo = force.f_outer[2]; + // real fxhi = force.f_ex_inner[0]; + // real fyhi = force.f_ex_inner[1]; + // real fzhi = force.f_ex_inner[2]; + // real fxho = force.f_ex_outer[0]; + // real fyho = force.f_ex_outer[1]; + // real fzho = force.f_ex_outer[2]; + + real tq_inner = xp*force.f_inner[1]-yp*force.f_inner[0]; + real tq_outer = xp*force.f_outer[1]-yp*force.f_outer[0]; + real tq_ex_inner = xp*force.f_ex_inner[1]-yp*force.f_ex_inner[0]; + real tq_ex_outer = xp*force.f_ex_outer[1]-yp*force.f_ex_outer[0]; + real wk_inner = vxp*force.f_inner[0]+vyp*force.f_inner[1]; + real wk_outer = vxp*force.f_outer[0]+vyp*force.f_outer[1]; + real wk_ex_inner = vxp*force.f_ex_inner[0]+vyp*force.f_ex_inner[1]; + real wk_ex_outer = vxp*force.f_ex_outer[0]+vyp*force.f_ex_outer[1]; + + // Write the data in ascii to our file + if(idfx::prank==0) { + std::ofstream ft; + ft.open(tqwkName,std::ios::app); + ft.precision(10); + // ft << std::scientific << timeStep << " " << fxi << " " << fyi << " " << fzi << " " << fxo << " " << fyo << " " << fzo << " " << fxhi << " " << fyhi << " " << fzhi << " " << fxho << " " << fyho << " " << fzho << " " << time << std::endl; + ft << std::scientific << timeStep << " " << tq_inner << " " << tq_outer << " " << tq_ex_inner << " " << tq_ex_outer << " " << wk_inner << " " << wk_outer << " " << wk_ex_inner << " " << wk_ex_outer << " " << time << std::endl; + ft.close(); + } + } +} + + +// Compute user variables which will be written in vtk files +void ComputeUserVars(DataBlock & data, UserDefVariablesContainer &variables) { + // Mirror data on Host + real h0=h0Glob; + real flaringIndex=flaringIndexGlob; + DataBlockHost d(data); + + // Sync it + d.SyncFromDevice(); + + // Make references to the user-defined arrays (variables is a container of IdefixHostArray3D) + // Note that the labels should match the variable names in the input file + IdefixHostArray3D press = variables["PRS"]; + + for(int k = 0; k < d.np_tot[KDIR] ; k++) { + for(int j = 0; j < d.np_tot[JDIR] ; j++) { + for(int i = 0; i < d.np_tot[IDIR] ; i++) { + real R = d.x[IDIR](i); + real cs2 = h0*h0*pow(R,2*flaringIndex-1.); + press(k,j,i) = cs2*d.Vc(RHO,k,j,i); + } + } + } +} + +// Default constructor +// Initialisation routine. Can be used to allocate +// Arrays or variables which are used later on +Setup::Setup(Input &input, Grid &grid, DataBlock &data, Output &output)// : m_planet(0)//, Planet &planet) +{ + // Set the function for userdefboundary + data.hydro->EnrollUserDefBoundary(&UserdefBoundary); + data.hydro->EnrollUserSourceTerm(&Damping); + data.hydro->EnrollIsoSoundSpeed(&MySoundSpeed); + + if(data.haveFargo) + data.fargo->EnrollVelocity(&FargoVelocity); + if(data.hydro->haveRotation) { + omegaGlob = data.hydro->OmegaZ; + } else { + omegaGlob = 0.0; + } + // Enroll the analysis function + output.EnrollAnalysis(&Analysis); + // Enroll our user-defined variables + output.EnrollUserDefVariables(&ComputeUserVars); + wkzMinGlob = input.Get("Setup","wkzMin",0); + wkzMaxGlob = input.Get("Setup","wkzMax",0); + wkzDampingGlob = input.Get("Setup","wkzDamping",0); + sigma0Glob = input.Get("Setup","sigma0",0); + sigmaSlopeGlob = input.Get("Setup","sigmaSlope",0); + h0Glob = input.Get("Setup","h0",0); + flaringIndexGlob = input.Get("Setup","flaringIndex",0); + densityFloorGlob = input.Get("Setup","densityFloor",0); + // delete file planet0.dat at initialization if we do not restart the simulation. + for(int ip=0; ip < data.planetarySystem->nbp ; ip++) { + std::string planetName, tqwkName, isActiveName; + std::stringstream pName, tName, iaName; + pName << "planet" << ip << ".dat"; + tName << "tqwk" << ip << ".dat"; + iaName << "isactive" << ip << ".dat"; + planetName = pName.str(); + tqwkName = tName.str(); + isActiveName = iaName.str(); + if (!(input.restartRequested)) { + if(idfx::prank==0) { + std::ofstream f; + f.open(planetName,std::ios::out); + f.close(); + std::ofstream ft; + ft.open(tqwkName,std::ios::out); + ft.close(); + std::ofstream fia; + fia.open(isActiveName,std::ios::out); + fia.close(); + } + } +/* + else { + if(idfx::prank==0) { + char ch; + int nline = input.restartFileNumber*input.Get("Output","dmp",0)/input.Get("Output","analysis",0); + int line = 1; + + // planet0.dat + std::ifstream fin(planetName); + std::ofstream fout; + fout.open("temp.dat", std::ios::out); + while(fin.get(ch)) + { + if(ch == '\n') + line++; + + if(line <= nline) // content not to be deleted + fout<("Output","dmp",0)/input.Get("Output","analysis",0); + line = 1; + while(fint.get(ch)) + { + if(ch == '\n') + line++; + + if(line <= nline) // content not to be deleted + foutt<a2[i] = g_gamma*p->v[i][PRS]/p->v[i][RHO]; + #elif EOS == ISOTHERMAL + { + int j,k; /* -- used as multidimensional indices -- */ + double *x1, *x2, *x3; + + x1 = grid->x[IDIR]; + x2 = grid->x[JDIR]; + x3 = grid->x[KDIR]; + + i = g_i; + j = g_j; + k = g_k; + + if (g_dir == IDIR) { + double R; + + x1 = (pos == FACE_CENTER ? grid->xr[IDIR] : grid->x[IDIR]); + for (i = beg; i <= end; i++){ + #if GEOMETRY == POLAR + R = x1[i]; + #elif GEOMETRY == SPHERICAL + R = x1[i]*sin(x2[j]); + #endif + p->a2[i] = g_isoSoundSpeed*g_isoSoundSpeed/R; + } + + }else if (g_dir == JDIR){ + double R; + + x2 = (pos == FACE_CENTER ? grid->xr[JDIR] : grid->x[JDIR]); + for (j = beg; j <= end; j++) { + #if GEOMETRY == POLAR + R = x1[i]; + #elif GEOMETRY == SPHERICAL + R = x1[i]*sin(x2[j]); + #endif + p->a2[j] = g_isoSoundSpeed*g_isoSoundSpeed/R; + } + }else if (g_dir == KDIR){ + double R; + + x3 = (pos == FACE_CENTER ? grid->xr[KDIR] : grid->x[KDIR]); + for (k = beg; k <= end; k++){ + #if GEOMETRY == POLAR + R = x1[i]; + #elif GEOMETRY == SPHERICAL + R = x1[i]*sin(x2[j]); + #endif + p->a2[k] = g_isoSoundSpeed*g_isoSoundSpeed/R; + } + } + } + #else + print ("! SoundSpeed2: not defined for this EoS\n"); + QUIT_PLUTO(1); + #endif +} + + +/* *************************************************************** */ +void Enthalpy (real **uprim, real *h, int beg, int end) +/* + * + * + * + ***************************************************************** */ +{ + int i; + double g_gammar; + + #if EOS == IDEAL + g_gammar = g_gamma/(g_gamma - 1.0); + for (i = beg; i <= end; i++){ + h[i] = g_gammar*uprim[i][PRS]/uprim[i][RHO]; + } + #elif EOS == ISOTHERMAL + print (" Enthalpy not defined for isothermal EoS\n"); + QUIT_PLUTO(1); + #endif +} +/* *************************************************************** */ +void ENTROPY (real **v, real *s, int is, int ie) +/* + * + * + * + ***************************************************************** */ +{ + int i; + double rho; + + #if EOS == IDEAL + for (i = is; i <= ie; i++){ + rho = v[i][RHO]; + s[i] = v[i][PRS]/pow(rho,g_gamma); + } + #elif EOS == ISOTHERMAL || EOS == BAROTROPIC + print (" Entropy not defined in isothermal or barotropic MHD\n"); + QUIT_PLUTO(1); + #endif +} diff --git a/test/Pluto/HD/PlanetDisk/init.c b/test/Pluto/HD/PlanetDisk/init.c new file mode 100644 index 00000000..bd35a262 --- /dev/null +++ b/test/Pluto/HD/PlanetDisk/init.c @@ -0,0 +1,274 @@ +/* ///////////////////////////////////////////////////////////////////// */ +/*! + \file + \brief Disk-Planet interaction problem. + + Simulate the interaction of a planet embedded in a disk as described + in section 3.4 of Mignone et al., A&A (2012) 545, A152. + This test is a nice benchmark for the FARGO module and the + \c ROTATING_FRAME switch. + For testing-purposes no viscosity is used here. + The initial condition consists of a locally isothermal configuration + with temperature profile \f$\propto T^{-1}\f$ yielding a disk vertical + height to radius of \c 0.05. + The gravitational potential due to the presence of the star and the + planet is defined in BodyForcePotential() function. + + The conventions used throught the implementation are the following: + + - \c r = spherical radius + - \c R = cylindrical radius + - \c z = cylindrical height + - \c th = meridional angle + + The test can be carried out in polar (2D or 3D) or spherical (3D) + coordinates and the following parameters determine the initial configuration: + + -# g_inputParam[Mstar]: controls the star mass (in solar masses) + -# g_inputParam[Mdisk]: controls the disk mass (in solar masses) + -# g_inputParam[Mplanet]: sets the planet mass (in earth masses) + -# g_inputParam[Viscosity]: sets the amount of viscosity + + + Computation can be carried in the rotating or in the observer's frame + of reference (\c ROTATING_FRAME to \c YES or \c NO, respectively). + In particular: + + - Configurations #01 and #02 are in 2D polar coordinates without and with + FARGO, in the rotating frame. + - Configurations #03, #04 and #05 are in spherical 3D coordinates with + and without FARGO in the rotating frame + - Configurations #06 and #07 are in 2D polar coordinates but in the + observer's frame + - Configuration #08 employs static AMR (grid levels are spatial + dependent but not dependent on time) in the rotating frame. + + + \image html hd_disk_planet.08.png "Density map for configuration #08 using AMR" width=1cm + + \author A. Mignone (mignone@ph.unito.it) + \date Aug 16, 2012 + + \b References: + - "A Conservative orbital advection scheme for simulations + of magnetized shear flows with the PLUTO Code" + Mignone et al, A&A (2012) +*/ +/* ///////////////////////////////////////////////////////////////////// */ +#include "pluto.h" + +#define MIN_DENSITY 1e-8 + +static void NormalizeDensity (const Data *d, Grid *g); +#if ROTATING_FRAME == NO + #define g_OmegaZ 0.0 +#endif + +/* ********************************************************************* */ +void Init (double *us, double x1, double x2, double x3) +/* + * + * + * + *********************************************************************** */ +{ + double r, th, R, z, H, OmegaK, cs; + double scrh; + double sigma0=0.125; + double sigmaSlope=0.5; + double h0=0.05; + + #if EOS == IDEAL + g_gamma = 1.01; + #endif + + + + #if GEOMETRY == POLAR + R = x1; + #if DIMENSIONS == 2 + z = 0.0; + r = R; + th = 0.5*CONST_PI; + #else + z = x3; + r = sqrt(R*R + z*z); + th = atan2(R,z); + #endif + #endif + + H = h0*R; + OmegaK = 1.0/(R*sqrt(R)); + cs = H*OmegaK; + + scrh = (0.5*CONST_PI - th)*r/H; + // us[RHO] = sigma0/sqrt(2.0*M_PI)/H*pow(R,-sigmaSlope) ; + us[RHO] = sigma0*pow(R,-sigmaSlope) ; + + us[VX1] = us[VX2] = us[VX3] = 0.0; + + us[iVPHI] = R*(OmegaK - g_OmegaZ)*sqrt(1-(1.0+sigmaSlope)*h0*h0); + #if EOS == IDEAL + us[PRS] = us[RHO]*cs*cs; + #elif EOS == ISOTHERMAL + // g_isoSoundSpeed = cs; + g_isoSoundSpeed = h0; + #endif + + +} + +/* ********************************************************************* */ +void InitDomain (Data *d, Grid *grid) +/*! + * Assign initial condition by looping over the computational domain. + * Called after the usual Init() function to assign initial conditions + * on primitive variables. + * Value assigned here will overwrite those prescribed during Init(). + * + * + *********************************************************************** */ +{ +} + +/* ********************************************************************* */ +void Analysis (const Data *d, Grid *grid) +/* + * + * + *********************************************************************** */ +{ + +} +/* ********************************************************************* */ +void UserDefBoundary (const Data *d, RBox *box, int side, Grid *grid) +/* + * + *********************************************************************** */ +{ + int i, j, k, nv; + double *x1, *x2, *x3, R, OmegaK, v[256]; + static int do_once = 1; + + x1 = grid->x[IDIR]; + x2 = grid->x[JDIR]; + x3 = grid->x[KDIR]; + + if (side == X1_BEG){ + X1_BEG_LOOP(k,j,i){ + NVAR_LOOP(nv) d->Vc[nv][k][j][i] = d->Vc[nv][k][j][2*IBEG - i - 1]; + d->Vc[VX1][k][j][i] *= -1.0; + #if GEOMETRY == POLAR + R = x1[i]; + #elif GEOMETRY == SPHERICAL + R = x1[i]*sin(x2[j]); + #endif + OmegaK = 1.0/(R*sqrt(R)); + d->Vc[iVPHI][k][j][i] = R*(OmegaK - g_OmegaZ); +#if DUST == YES +// NDUST_LOOP(nv) d->Vc[nv][k][j][i] = 0.0; + d->Vc[VX2_D][k][j][i] = d->Vc[iVPHI][k][j][i]; +#endif + } + } + + if (side == X1_END){ + X1_END_LOOP(k,j,i){ + NVAR_LOOP(nv) d->Vc[nv][k][j][i] = d->Vc[nv][k][j][IEND]; + #if GEOMETRY == POLAR + R = x1[i]; +// d->Vc[iVR][k][j][i] = 0.0; + #elif GEOMETRY == SPHERICAL + R = x1[i]*sin(x2[j]); + d->Vc[iVR][k][j][i] = 0.0; + d->Vc[iVTH][k][j][i] = 0.0; + #endif + OmegaK = 1.0/(R*sqrt(R)); + d->Vc[iVPHI][k][j][i] = R*(OmegaK - g_OmegaZ); +#if DUST == YES + d->Vc[VX2_D][k][j][i] = d->Vc[iVPHI][k][j][i]; +#endif + + } + } +} + + + +#if (BODY_FORCE & VECTOR) +/* ************************************************************************ */ +void BodyForceVector(double *v, double *g, double x1, double x2, double x3) +/* + * + * + * + *************************************************************************** */ +{ + g[IDIR] = 0.0; + g[JDIR] = 0.0; + g[KDIR] = 0.0; +} +#endif + +#if (BODY_FORCE & POTENTIAL) +/* ************************************************************************ */ +double BodyForcePotential(double x1, double x2, double x3) +/* + * + * + * + *************************************************************************** */ +{ + double d, R, r, z, th, x, y, phiplanet, rsm; + double xp, yp, t, phi; + double h0=0.05; + double smoothing = 0.6; + + #if GEOMETRY == POLAR + R = x1; + #if DIMENSIONS == 2 + z = 0.0; + r = R; + th = 0.5*CONST_PI; + #else + z = x3; + r = sqrt(R*R + z*z); + th = atan2(R,z); + #endif + x = R*cos(x2); + y = R*sin(x2); + #elif (GEOMETRY == SPHERICAL) + r = x1; + th = x2; + R = r*sin(th); + z = r*cos(th); + x = r*sin(th)*cos(x3); + y = r*sin(th)*sin(x3); + #endif + +/* --------------------------------------------- + planet position + --------------------------------------------- */ + + + double OmegaZ; + t = g_time; + // This is not present in Idefix yet + //if (g_stepNumber == 2) t += g_dt; + OmegaZ = sqrt(1.0 + g_inputParam[Mplanet]/g_inputParam[Mstar]); + + xp = cos(OmegaZ*t); + yp = sin(OmegaZ*t); + + + d = (x-xp)*(x-xp) + (y-yp)*(y-yp) + z*z; + rsm = h0*R*smoothing; + phiplanet = g_inputParam[Mplanet]/g_inputParam[Mstar]/sqrt(d+rsm*rsm); + + phi = - 1/r; + phi += -phiplanet; + phi += g_inputParam[Mplanet]/g_inputParam[Mstar]*(x*xp+y*yp); // /Rp^3 with Rp=1 + + return phi; +} +#endif diff --git a/test/Pluto/HD/PlanetDisk/pluto.ini b/test/Pluto/HD/PlanetDisk/pluto.ini new file mode 100644 index 00000000..3a5edbd4 --- /dev/null +++ b/test/Pluto/HD/PlanetDisk/pluto.ini @@ -0,0 +1,52 @@ +[Grid] +X1-grid 1 0.4 256 l+ 2.5 +X2-grid 1 0.0 256 u 6.283185307 +X3-grid 1 -1.0 1 u 1.0 + +[Chombo Refinement] +Levels 4 +Ref_ratio 2 2 2 2 2 +Regrid_interval 2 2 2 2 +Refine_thresh 0.3 +Tag_buffer_size 3 +Block_factor 8 +Max_grid_size 64 +Fill_ratio 0.75 + +[Time] +CFL 0.5 +CFL_max_var 1.1 +tstop 100.0 +first_dt 1.e-4 + +[Solver] +Solver hllc + +[Boundary] +X1-beg userdef +X1-end userdef +X2-beg periodic +X2-end periodic +X3-beg periodic +X3-end periodic + +[Static Grid Output] +uservar 0 +dbl -10.0 -200 single_file +flt -1.0 -1 single_file +vtk 6.283185307 -1 single_file +tab -1.0 -1 +ppm -1.0 -1 +png -1.0 -1 +log 100 +analysis -1.0 100 + +[Chombo HDF5 output] +Checkpoint_interval -1.0 0 +Plot_interval 1.0 0 + +[Parameters] +Mstar 1.0 +Mdisk 0.01 +Mplanet 1.0e-3 +Viscosity 0.0 diff --git a/test/Pluto/HD/PlanetDisk3D/definitions.h b/test/Pluto/HD/PlanetDisk3D/definitions.h new file mode 100644 index 00000000..5ad7a315 --- /dev/null +++ b/test/Pluto/HD/PlanetDisk3D/definitions.h @@ -0,0 +1,42 @@ +#define PHYSICS HD +#define DIMENSIONS 3 +#define COMPONENTS 3 +#define GEOMETRY POLAR +#define BODY_FORCE POTENTIAL +#define FORCED_TURB NO +#define COOLING NO +#define RECONSTRUCTION LINEAR +#define TIME_STEPPING RK2 +#define DIMENSIONAL_SPLITTING NO +#define NTRACER 0 +#define USER_DEF_PARAMETERS 8 + +/* -- physics dependent declarations -- */ + +#define EOS ISOTHERMAL +#define ENTROPY_SWITCH NO +#define THERMAL_CONDUCTION NO +#define VISCOSITY NO +#define ROTATING_FRAME NO +#define DUST NO + +/* -- user-defined parameters (labels) -- */ + +#define Mstar 0 +#define Mdisk 1 +#define Mplanet 2 +#define Viscosity 3 +#define H0 4 +#define SIGMA0 5 +#define SIGMA_SLOPE 6 +#define THICKNESS_SMOOTHING 7 + +/* [Beg] user-defined constants (do not change this line) */ + +#define LIMITER VANLEER_LIM +#define UNIT_LENGTH (5.2*CONST_au) +#define UNIT_DENSITY (CONST_Msun/(UNIT_LENGTH*UNIT_LENGTH*UNIT_LENGTH)) +#define UNIT_VELOCITY (sqrt(CONST_G*g_inputParam[Mstar]*CONST_Msun/UNIT_LENGTH)/(2.*CONST_PI)) +#define PRINT_TO_SCREEN YES + +/* [End] user-defined constants (do not change this line) */ diff --git a/test/Pluto/HD/PlanetDisk3D/eos.c b/test/Pluto/HD/PlanetDisk3D/eos.c new file mode 100644 index 00000000..61845c17 --- /dev/null +++ b/test/Pluto/HD/PlanetDisk3D/eos.c @@ -0,0 +1,116 @@ +#include "pluto.h" + +/* **************************************************************** */ +void SoundSpeed2 (const State *p, int beg, int end, + int pos, Grid *grid) +/* + * + * Define the square of the sound speed for different EOS + * + ****************************************************************** */ +{ + int i; + + #if EOS == IDEAL + for (i = beg; i <= end; i++) p->a2[i] = g_gamma*p->v[i][PRS]/p->v[i][RHO]; + #elif EOS == ISOTHERMAL + { + int j,k; /* -- used as multidimensional indices -- */ + double *x1, *x2, *x3; + + x1 = grid->x[IDIR]; + x2 = grid->x[JDIR]; + x3 = grid->x[KDIR]; + + i = g_i; + j = g_j; + k = g_k; + + if (g_dir == IDIR) { + double R; + + x1 = (pos == FACE_CENTER ? grid->xr[IDIR] : grid->x[IDIR]); + for (i = beg; i <= end; i++){ + #if GEOMETRY == POLAR + R = x1[i]; + #elif GEOMETRY == SPHERICAL + R = x1[i]*sin(x2[j]); + #endif + p->a2[i] = g_isoSoundSpeed*g_isoSoundSpeed/R; + } + + }else if (g_dir == JDIR){ + double R; + + x2 = (pos == FACE_CENTER ? grid->xr[JDIR] : grid->x[JDIR]); + for (j = beg; j <= end; j++) { + #if GEOMETRY == POLAR + R = x1[i]; + #elif GEOMETRY == SPHERICAL + R = x1[i]*sin(x2[j]); + #endif + p->a2[j] = g_isoSoundSpeed*g_isoSoundSpeed/R; + } + }else if (g_dir == KDIR){ + double R; + + x3 = (pos == FACE_CENTER ? grid->xr[KDIR] : grid->x[KDIR]); + for (k = beg; k <= end; k++){ + #if GEOMETRY == POLAR + R = x1[i]; + #elif GEOMETRY == SPHERICAL + R = x1[i]*sin(x2[j]); + #endif + p->a2[k] = g_isoSoundSpeed*g_isoSoundSpeed/R; + } + } + } + #else + print ("! SoundSpeed2: not defined for this EoS\n"); + QUIT_PLUTO(1); + #endif +} + + +/* *************************************************************** */ +void Enthalpy (real **uprim, real *h, int beg, int end) +/* + * + * + * + ***************************************************************** */ +{ + int i; + double g_gammar; + + #if EOS == IDEAL + g_gammar = g_gamma/(g_gamma - 1.0); + for (i = beg; i <= end; i++){ + h[i] = g_gammar*uprim[i][PRS]/uprim[i][RHO]; + } + #elif EOS == ISOTHERMAL + print (" Enthalpy not defined for isothermal EoS\n"); + QUIT_PLUTO(1); + #endif +} +/* *************************************************************** */ +void ENTROPY (real **v, real *s, int is, int ie) +/* + * + * + * + ***************************************************************** */ +{ + int i; + double rho; + + #if EOS == IDEAL + for (i = is; i <= ie; i++){ + rho = v[i][RHO]; + s[i] = v[i][PRS]/pow(rho,g_gamma); + } + #elif EOS == ISOTHERMAL || EOS == BAROTROPIC + print (" Entropy not defined in isothermal or barotropic MHD\n"); + QUIT_PLUTO(1); + #endif +} diff --git a/test/Pluto/HD/PlanetDisk3D/init.c b/test/Pluto/HD/PlanetDisk3D/init.c new file mode 100644 index 00000000..39577c77 --- /dev/null +++ b/test/Pluto/HD/PlanetDisk3D/init.c @@ -0,0 +1,301 @@ +/* ///////////////////////////////////////////////////////////////////// */ +/*! + \file + \brief Disk-Planet interaction problem. + + Simulate the interaction of a planet embedded in a disk as described + in section 3.4 of Mignone et al., A&A (2012) 545, A152. + This test is a nice benchmark for the FARGO module and the + \c ROTATING_FRAME switch. + For testing-purposes no viscosity is used here. + The initial condition consists of a locally isothermal configuration + with temperature profile \f$\propto T^{-1}\f$ yielding a disk vertical + height to radius of \c 0.05. + The gravitational potential due to the presence of the star and the + planet is defined in BodyForcePotential() function. + + The conventions used throught the implementation are the following: + + - \c r = spherical radius + - \c R = cylindrical radius + - \c z = cylindrical height + - \c th = meridional angle + + The test can be carried out in polar (2D or 3D) or spherical (3D) + coordinates and the following parameters determine the initial configuration: + + -# g_inputParam[Mstar]: controls the star mass (in solar masses) + -# g_inputParam[Mdisk]: controls the disk mass (in solar masses) + -# g_inputParam[Mplanet]: sets the planet mass (in earth masses) + -# g_inputParam[Viscosity]: sets the amount of viscosity + + + Computation can be carried in the rotating or in the observer's frame + of reference (\c ROTATING_FRAME to \c YES or \c NO, respectively). + In particular: + + - Configurations #01 and #02 are in 2D polar coordinates without and with + FARGO, in the rotating frame. + - Configurations #03, #04 and #05 are in spherical 3D coordinates with + and without FARGO in the rotating frame + - Configurations #06 and #07 are in 2D polar coordinates but in the + observer's frame + - Configuration #08 employs static AMR (grid levels are spatial + dependent but not dependent on time) in the rotating frame. + + + \image html hd_disk_planet.08.png "Density map for configuration #08 using AMR" width=1cm + + \author A. Mignone (mignone@ph.unito.it) + \date Aug 16, 2012 + + \b References: + - "A Conservative orbital advection scheme for simulations + of magnetized shear flows with the PLUTO Code" + Mignone et al, A&A (2012) +*/ +/* ///////////////////////////////////////////////////////////////////// */ +#include "pluto.h" + +#define MIN_DENSITY 1e-8 + +static void NormalizeDensity (const Data *d, Grid *g); +#if ROTATING_FRAME == NO + #define g_OmegaZ 0.0 +#endif + +/* ********************************************************************* */ +void Init (double *us, double x1, double x2, double x3) +/* + * + * + * + *********************************************************************** */ +{ + double r, th, R, z, H, OmegaK, cs; + double scrh; + double sigma0=g_inputParam[SIGMA0]; + double sigmaSlope=g_inputParam[SIGMA_SLOPE]; + double h0=g_inputParam[H0]; + + #if EOS == IDEAL + g_gamma = 1.01; + #endif + + + + #if GEOMETRY == POLAR + R = x1; + #if DIMENSIONS == 2 + z = 0.0; + r = R; + th = 0.5*CONST_PI; + #else + z = x3; + r = sqrt(R*R + z*z); + th = atan2(R,z); + #endif + #endif + + H = h0*R; + OmegaK = 1.0/(R*sqrt(R)); + cs = H*OmegaK; + + scrh = (0.5*CONST_PI - th)*r/H; + // us[RHO] = sigma0/sqrt(2.0*M_PI)/H*pow(R,-sigmaSlope) ; + us[RHO] = sigma0/sqrt(2.0*M_PI)/(h0*R)*pow(R,-sigmaSlope) * exp(1.0/ (cs*cs) * (1.0/sqrt(R*R+z*z)-1.0/R)) ; + + us[VX1] = us[VX2] = us[VX3] = 0.0; + + us[iVPHI] = R*OmegaK*sqrt( R / sqrt(R*R + z*z) -(2.0+sigmaSlope)*h0*h0); + + if(us[RHO]<1e-6) us[RHO] = 1e-6; + + #if EOS == IDEAL + us[PRS] = us[RHO]*cs*cs; + #elif EOS == ISOTHERMAL + // g_isoSoundSpeed = cs; + g_isoSoundSpeed = h0; + #endif + + +} + +/* ********************************************************************* */ +void InitDomain (Data *d, Grid *grid) +/*! + * Assign initial condition by looping over the computational domain. + * Called after the usual Init() function to assign initial conditions + * on primitive variables. + * Value assigned here will overwrite those prescribed during Init(). + * + * + *********************************************************************** */ +{ +} + +/* ********************************************************************* */ +void Analysis (const Data *d, Grid *grid) +/* + * + * + *********************************************************************** */ +{ + +} +/* ********************************************************************* */ +void UserDefBoundary (const Data *d, RBox *box, int side, Grid *grid) +/* + * + *********************************************************************** */ +{ + int i, j, k, nv; + double *x1, *x2, *x3, R, z, OmegaK, v[256]; + static int do_once = 1; + + double h0 = g_inputParam[H0]; + double sigmaSlope = g_inputParam[SIGMA_SLOPE]; + double sigma0 = g_inputParam[SIGMA0]; + + x1 = grid->x[IDIR]; + x2 = grid->x[JDIR]; + x3 = grid->x[KDIR]; + + if (side == X1_BEG){ + X1_BEG_LOOP(k,j,i){ + R=x1[i]; + z=x3[k]; + double Vk = 1/sqrt(R); + double cs2=h0*h0*Vk*Vk; + + d->Vc[RHO][k][j][i] = sigma0/sqrt(2.0*M_PI)/(h0*R)*pow(R,-sigmaSlope) * exp(1.0/(cs2) * (1.0/sqrt(R*R+z*z)-1.0/R)) ; + d->Vc[VX1][k][j][i] = -d->Vc[VX1][k][j][IBEG]; + d->Vc[VX2][k][j][i] = Vk*sqrt(R/sqrt(R*R + z*z)-(2.0+sigmaSlope)*h0*h0); + d->Vc[VX3][k][j][i] = 0.0; + } + } + + if (side == X1_END){ + X1_END_LOOP(k,j,i){ + NVAR_LOOP(nv) d->Vc[nv][k][j][i] = d->Vc[nv][k][j][IEND]; + #if GEOMETRY == POLAR + R = x1[i]; +// d->Vc[iVR][k][j][i] = 0.0; + #elif GEOMETRY == SPHERICAL + R = x1[i]*sin(x2[j]); + d->Vc[iVR][k][j][i] = 0.0; + d->Vc[iVTH][k][j][i] = 0.0; + #endif + OmegaK = 1.0/(R*sqrt(R)); + d->Vc[iVPHI][k][j][i] = R*(OmegaK - g_OmegaZ); + + } + } + + if (side == X3_BEG){ + X3_BEG_LOOP(k,j,i){ + R=x1[i]; + z=x3[k]; + double Vk = 1/sqrt(R); + double cs2=h0*h0*Vk*Vk; + d->Vc[RHO][k][j][i] = d->Vc[RHO][KBEG][j][i]*exp(1.0/(cs2) * (1.0/sqrt(R*R+z*z) - 1.0/sqrt(R*R+x3[KBEG]*x3[KBEG]))); + d->Vc[VX1][k][j][i] = d->Vc[VX1][KBEG][j][i]; + d->Vc[VX2][k][j][i] = Vk*sqrt(R/sqrt(R*R + z*z)-(1+sigmaSlope)*h0*h0); + d->Vc[VX3][k][j][i] = -d->Vc[VX3][KBEG][j][i]; + } + } + if (side == X3_END){ + X3_END_LOOP(k,j,i){ + R=x1[i]; + z=x3[k]; + double Vk = 1/sqrt(R); + double cs2=h0*h0*Vk*Vk; + d->Vc[RHO][k][j][i] = d->Vc[RHO][KEND][j][i]*exp(1.0/(cs2) * (1.0/sqrt(R*R+z*z) - 1.0/sqrt(R*R+x3[KEND]*x3[KEND]))); + d->Vc[VX1][k][j][i] = d->Vc[VX1][KEND][j][i]; + d->Vc[VX2][k][j][i] = Vk*sqrt(R/sqrt(R*R + z*z)-(1+sigmaSlope)*h0*h0); + d->Vc[VX3][k][j][i] = -d->Vc[VX3][KEND][j][i]; + } + } + + +} + + + +#if (BODY_FORCE & VECTOR) +/* ************************************************************************ */ +void BodyForceVector(double *v, double *g, double x1, double x2, double x3) +/* + * + * + * + *************************************************************************** */ +{ + g[IDIR] = 0.0; + g[JDIR] = 0.0; + g[KDIR] = 0.0; +} +#endif + +#if (BODY_FORCE & POTENTIAL) +/* ************************************************************************ */ +double BodyForcePotential(double x1, double x2, double x3) +/* + * + * + * + *************************************************************************** */ +{ + double d, R, r, z, th, x, y, phiplanet, rsm; + double xp, yp, t, phi; + double h0=g_inputParam[H0]; + double thicknessSmoothing = g_inputParam[THICKNESS_SMOOTHING]; + double smoothing = h0*thicknessSmoothing; + + #if GEOMETRY == POLAR + R = x1; + #if DIMENSIONS == 2 + z = 0.0; + r = R; + th = 0.5*CONST_PI; + #else + z = x3; + r = sqrt(R*R + z*z); + th = atan2(R,z); + #endif + x = R*cos(x2); + y = R*sin(x2); + #elif (GEOMETRY == SPHERICAL) + r = x1; + th = x2; + R = r*sin(th); + z = r*cos(th); + x = r*sin(th)*cos(x3); + y = r*sin(th)*sin(x3); + #endif + +/* --------------------------------------------- + planet position + --------------------------------------------- */ + + + double OmegaZ; + t = g_time; + // This is not present in Idefix yet + //if (g_stepNumber == 2) t += g_dt; + OmegaZ = sqrt(1.0 + g_inputParam[Mplanet]/g_inputParam[Mstar]); + + xp = cos(OmegaZ*t); + yp = sin(OmegaZ*t); + + + d = (x-xp)*(x-xp) + (y-yp)*(y-yp) + z*z; + phiplanet = g_inputParam[Mplanet]/g_inputParam[Mstar]/sqrt(d+smoothing*smoothing); + + phi = - 1/r; + phi += -phiplanet; + phi += g_inputParam[Mplanet]/g_inputParam[Mstar]*(x*xp+y*yp); // /Rp^3 with Rp=1 + + return phi; +} +#endif diff --git a/test/Pluto/HD/PlanetDisk3D/pluto.ini b/test/Pluto/HD/PlanetDisk3D/pluto.ini new file mode 100644 index 00000000..77114b1a --- /dev/null +++ b/test/Pluto/HD/PlanetDisk3D/pluto.ini @@ -0,0 +1,56 @@ +[Grid] +X1-grid 1 0.4 64 l+ 2.5 +X2-grid 1 0.0 64 u 6.283185307 +X3-grid 1 -0.1 32 u 0.1 + +[Chombo Refinement] +Levels 4 +Ref_ratio 2 2 2 2 2 +Regrid_interval 2 2 2 2 +Refine_thresh 0.3 +Tag_buffer_size 3 +Block_factor 8 +Max_grid_size 64 +Fill_ratio 0.75 + +[Time] +CFL 0.16666666 +CFL_max_var 1.1 +tstop 12.6 +first_dt 1.e-4 + +[Solver] +Solver hllc + +[Boundary] +X1-beg userdef +X1-end userdef +X2-beg periodic +X2-end periodic +X3-beg userdef +X3-end userdef + +[Static Grid Output] +uservar 0 +dbl -10.0 -200 single_file +flt -1.0 -1 single_file +vtk 0.314159265359 -1 single_file +tab -1.0 -1 +ppm -1.0 -1 +png -1.0 -1 +log 100 +analysis -1.0 100 + +[Chombo HDF5 output] +Checkpoint_interval -1.0 0 +Plot_interval 1.0 0 + +[Parameters] +Mstar 1.0 +Mdisk 0.01 +Mplanet 1.0e-3 +Viscosity 0.0 +H0 0.05 +SIGMA0 0.125 +SIGMA_SLOPE 0.5 +THICKNESS_SMOOTHING 0.6 diff --git a/test/SelfGravity/JeansInstability/definitions.hpp b/test/SelfGravity/JeansInstability/definitions.hpp new file mode 100644 index 00000000..f64d67e9 --- /dev/null +++ b/test/SelfGravity/JeansInstability/definitions.hpp @@ -0,0 +1,6 @@ +#define COMPONENTS 1 +#define DIMENSIONS 1 + +#define GEOMETRY CARTESIAN +// Order of the scheme. 1=donnor cell, 2= linear reconstruction +#define ORDER 2 diff --git a/test/SelfGravity/JeansInstability/idefix-cg.ini b/test/SelfGravity/JeansInstability/idefix-cg.ini new file mode 100644 index 00000000..97f1e6c9 --- /dev/null +++ b/test/SelfGravity/JeansInstability/idefix-cg.ini @@ -0,0 +1,43 @@ +[Grid] +X1-grid 1 0.0 1000 u 10.0 +X2-grid 1 0.0 100 u 10.0 +X3-grid 1 0.0 100 u 10.0 + +[TimeIntegrator] +CFL 0.8 +CFL_max_var 1.1 +tstop 1.0 +first_dt 1.e-4 +nstages 2 + +[Hydro] +solver hll +gamma 1.66666666667 + +[Gravity] +potential selfgravity +gravCst 3.141592654 + +[SelfGravity] +maxIter 1000 +solver CG +targetError 1e-6 +boundary-X1-beg periodic +boundary-X1-end periodic +boundary-X2-beg periodic +boundary-X2-end periodic +boundary-X3-beg periodic +boundary-X3-end periodic + +[Boundary] +X1-beg periodic +X1-end periodic +X2-beg periodic +X2-end periodic +X3-beg periodic +X3-end periodic + +[Output] +vtk 0.1 +dmp 1.0 +log 10 diff --git a/test/SelfGravity/JeansInstability/idefix.ini b/test/SelfGravity/JeansInstability/idefix.ini new file mode 100644 index 00000000..47101406 --- /dev/null +++ b/test/SelfGravity/JeansInstability/idefix.ini @@ -0,0 +1,43 @@ +[Grid] +X1-grid 1 0.0 1000 u 10.0 +X2-grid 1 0.0 100 u 10.0 +X3-grid 1 0.0 100 u 10.0 + +[TimeIntegrator] +CFL 0.8 +CFL_max_var 1.1 +tstop 1.0 +first_dt 1.e-4 +nstages 2 + +[Hydro] +solver hll +gamma 1.66666666667 + +[Gravity] +potential selfgravity +gravCst 3.141592654 + +[SelfGravity] +solver BICGSTAB +targetError 1e-6 +# skip 2 +boundary-X1-beg periodic +boundary-X1-end periodic +boundary-X2-beg periodic +boundary-X2-end periodic +boundary-X3-beg periodic +boundary-X3-end periodic + +[Boundary] +X1-beg periodic +X1-end periodic +X2-beg periodic +X2-end periodic +X3-beg periodic +X3-end periodic + +[Output] +vtk 0.1 +dmp 1.0 +log 10 diff --git a/test/SelfGravity/JeansInstability/python/testidefix.py b/test/SelfGravity/JeansInstability/python/testidefix.py new file mode 100755 index 00000000..80d6ee62 --- /dev/null +++ b/test/SelfGravity/JeansInstability/python/testidefix.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +Created on Thu Mar 5 11:29:41 2020 + +@author: glesur +""" + +import os +import sys +sys.path.append(os.getenv("IDEFIX_DIR")) +from pytools.vtk_io import readVTK +import numpy as np +from scipy.fftpack import fft, fftfreq + +# Arguments +gravCst = np.pi # grav cst G=pi +gamma = 5/3 +rho_bg = 3.0 # background density + +# Parameters (calculated from the arguments) +prs_bg = 1.0/gamma +cs02 = gamma*prs_bg/rho_bg # squared bg speed velocity +lbdJ = np.sqrt(cs02*np.pi/(gravCst*rho_bg)) # Jean's wavelength +kJ = 2*np.pi/lbdJ # Jean's wavevector + +# Collect vtk files (we ignore those before vtkRef) +Vmax=readVTK('../data.0004.vtk') # Best time vtk (for gr fitting) +Vref=readVTK('../data.0003.vtk') # Ref vtk (after unwanted modes are quenched) + +# Collect data +x1 = np.squeeze(Vref.x) +tref, tmax = Vref.t[0], Vmax.t[0] +uref, umax = np.squeeze(Vref.data['VX1']), np.squeeze(Vmax.data['VX1']) + +# Compute fft +N1 = x1.size +dx1 = x1[1]-x1[0] +k1 = 2*np.pi*fftfreq(N1, dx1) +furef, fumax = np.abs(fft(uref)), np.abs(fft(umax)) + +# Compute the numerical/theoretical growth rate of the second non-zero unstable mode +mode = 2 +gr_num = np.log(fumax/furef)[mode]/(tmax-tref) +gr_th = np.sqrt(np.abs(k1[mode]**2*cs02-4.*np.pi*gravCst*rho_bg)) +error = np.abs((gr_num-gr_th)/gr_th) + +# Print the result of the test +print("Error=%e"%error) +if error<2.3e-3: + print("SUCCESS!") + sys.exit(0) +else: + print("FAILURE!") + sys.exit(1) diff --git a/test/SelfGravity/JeansInstability/setup.cpp b/test/SelfGravity/JeansInstability/setup.cpp new file mode 100644 index 00000000..78ee7a15 --- /dev/null +++ b/test/SelfGravity/JeansInstability/setup.cpp @@ -0,0 +1,65 @@ +#include + +#include "idefix.hpp" +#include "setup.hpp" + +real lengthA, xc, sigma; +real rho0, gammaGlob, prs0, A; + +// Default constructor + + +// Initialisation routine. Can be used to allocate +// Arrays or variables which are used later on +Setup::Setup(Input &input, Grid &grid, DataBlock &data, Output &output) { + lengthA = grid.xend[IDIR]-grid.xbeg[JDIR]; // Length of the active total domain + xc = 0.5 * lengthA; // gaussian's center + sigma = 0.1 * lengthA; // gaussian's variance + + // Parameters of the physical setup + rho0 = 3.0; // Background density + gammaGlob=data.hydro->eos->GetGamma(); // Input gamma + prs0 = 1.0/gammaGlob; // Background pressure + A = 0.0001; // Perturbation amplitude +} + +// This routine initialize the flow +// Note that data is on the device. +// One can therefore define locally +// a datahost and sync it, if needed +void Setup::InitFlow(DataBlock &data) { + // Create a host copy + DataBlockHost d(data); + + // Grid parameters + IdefixHostArray1D x1 = d.x[IDIR]; // Position on the dataBlock + + // Mapping position wrt 0. and ensuring periodicity + IdefixHostArray1D y1 = IdefixHostArray1D ("CenteredPosition", d.np_tot[IDIR]); // Position wrt to the new center 0. + + for(int i = 0; i < d.np_tot[IDIR] ; i++) { + if((x1(i) - xc) > lengthA/2.) { + y1(i) = x1(i) - xc - lengthA; + } + else if((x1(i) - xc) < -1.*lengthA/2.) { + y1(i) = x1(i) - xc + lengthA; + } + else { + y1(i) = x1(i) - xc; + } + } + + // Filling initial profiles + for(int k = 0; k < d.np_tot[KDIR] ; k++) { + for(int j = 0; j < d.np_tot[JDIR] ; j++) { + for(int i = 0; i < d.np_tot[IDIR] ; i++) { + d.Vc(RHO,k,j,i) = rho0 * (1. + A * exp(-1.*y1(i)*y1(i)/(2.*sigma*sigma))); + d.Vc(VX1,k,j,i) = 0.; + d.Vc(PRS,k,j,i) = prs0 * (1. + gammaGlob * A * exp(-1.*y1(i)*y1(i)/(2.*sigma*sigma))); + } + } + } + + // Send it all, if needed + d.SyncToDevice(); +} diff --git a/test/SelfGravity/JeansInstability/testme.py b/test/SelfGravity/JeansInstability/testme.py new file mode 100755 index 00000000..435156d1 --- /dev/null +++ b/test/SelfGravity/JeansInstability/testme.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python3 + +""" + +@author: glesur +""" +import os +import sys +sys.path.append(os.getenv("IDEFIX_DIR")) + +import pytools.idfx_test as tst + +name="dump.0001.dmp" + + +tolerance=1e-12 + +def testMe(test): + test.configure() + test.compile() + inifiles=["idefix.ini","idefix-cg.ini"] + + # loop on all the ini files for this test + for ini in inifiles: + test.run(inputFile=ini) + test.standardTest() + + +test=tst.idfxTest() + +if not test.all: + if(test.check): + test.standardTest() + else: + testMe(test) +else: + test.noplot = True + test.vectPot=False + test.single=False + test.reconstruction=2 + test.mpi=False + testMe(test) diff --git a/test/SelfGravity/RandomSphere/definitions.hpp b/test/SelfGravity/RandomSphere/definitions.hpp new file mode 100644 index 00000000..1abadf73 --- /dev/null +++ b/test/SelfGravity/RandomSphere/definitions.hpp @@ -0,0 +1,7 @@ +#define COMPONENTS 3 +#define DIMENSIONS 3 + +#define GEOMETRY SPHERICAL +// Order of the scheme. 1=donnor cell, 2= linear reconstruction +#define ORDER 2 +#define ISOTHERMAL diff --git a/test/SelfGravity/RandomSphere/idefix-cg.ini b/test/SelfGravity/RandomSphere/idefix-cg.ini new file mode 100644 index 00000000..9b22df66 --- /dev/null +++ b/test/SelfGravity/RandomSphere/idefix-cg.ini @@ -0,0 +1,41 @@ +[Grid] +X1-grid 1 1.0 64 u 10.0 +X2-grid 3 0.0 16 s+ 1.2707963267948965 32 u 1.8707963267948966 16 s- 3.141592653589793 +X3-grid 1 0.0 64 u 6.283185307179586 + +[TimeIntegrator] +CFL 0.8 +CFL_max_var 1.1 +tstop 0.0 +first_dt 1.e-4 +nstages 2 + +[Hydro] +solver roe +csiso constant 1.0 + +[Gravity] +potential selfgravity +gravCst 1.0 + +[SelfGravity] +solver PCG +targetError 1e-4 +boundary-X1-beg origin +boundary-X1-end nullpot +boundary-X2-beg axis +boundary-X2-end axis +boundary-X3-beg periodic +boundary-X3-end periodic + +[Boundary] +X1-beg outflow +X1-end outflow +X2-beg axis +X2-end axis +X3-beg periodic +X3-end periodic + +[Output] +vtk 1.e-4 +uservar phiP diff --git a/test/SelfGravity/RandomSphere/idefix-minres.ini b/test/SelfGravity/RandomSphere/idefix-minres.ini new file mode 100644 index 00000000..807508e7 --- /dev/null +++ b/test/SelfGravity/RandomSphere/idefix-minres.ini @@ -0,0 +1,41 @@ +[Grid] +X1-grid 1 1.0 64 u 10.0 +X2-grid 3 0.0 16 s+ 1.2707963267948965 32 u 1.8707963267948966 16 s- 3.141592653589793 +X3-grid 1 0.0 64 u 6.283185307179586 + +[TimeIntegrator] +CFL 0.8 +CFL_max_var 1.1 +tstop 0.0 +first_dt 1.e-4 +nstages 2 + +[Hydro] +solver roe +csiso constant 1.0 + +[Gravity] +potential selfgravity +gravCst 1.0 + +[SelfGravity] +solver PMINRES +targetError 1e-4 +boundary-X1-beg origin +boundary-X1-end nullpot +boundary-X2-beg axis +boundary-X2-end axis +boundary-X3-beg periodic +boundary-X3-end periodic + +[Boundary] +X1-beg outflow +X1-end outflow +X2-beg axis +X2-end axis +X3-beg periodic +X3-end periodic + +[Output] +vtk 1.e-4 +uservar phiP diff --git a/test/SelfGravity/RandomSphere/idefix.ini b/test/SelfGravity/RandomSphere/idefix.ini new file mode 100644 index 00000000..e02dfdb1 --- /dev/null +++ b/test/SelfGravity/RandomSphere/idefix.ini @@ -0,0 +1,41 @@ +[Grid] +X1-grid 1 1.0 64 u 10.0 +X2-grid 3 0.0 16 s+ 1.2707963267948965 32 u 1.8707963267948966 16 s- 3.141592653589793 +X3-grid 1 0.0 64 u 6.283185307179586 + +[TimeIntegrator] +CFL 0.8 +CFL_max_var 1.1 +tstop 0.0 +first_dt 1.e-4 +nstages 2 + +[Hydro] +solver roe +csiso constant 1.0 + +[Gravity] +potential selfgravity +gravCst 1.0 + +[SelfGravity] +solver PBICGSTAB +targetError 1e-4 +boundary-X1-beg origin +boundary-X1-end nullpot +boundary-X2-beg axis +boundary-X2-end axis +boundary-X3-beg periodic +boundary-X3-end periodic + +[Boundary] +X1-beg outflow +X1-end outflow +X2-beg axis +X2-end axis +X3-beg periodic +X3-end periodic + +[Output] +vtk 1.e-4 +uservar phiP diff --git a/test/SelfGravity/RandomSphere/python/testidefix.py b/test/SelfGravity/RandomSphere/python/testidefix.py new file mode 100755 index 00000000..444c3034 --- /dev/null +++ b/test/SelfGravity/RandomSphere/python/testidefix.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +Created on Thu Mar 5 11:29:41 2020 + +@author: glesur +""" + +import os +import sys +import numpy as np +import matplotlib.pyplot as plt +import argparse + +sys.path.append(os.getenv("IDEFIX_DIR")) + +from pytools.vtk_io import readVTK + +# Add noplot argument to parser +parser = argparse.ArgumentParser() +parser.add_argument("-noplot", + default=False, + help="disable plotting", + action="store_true") + +args=parser.parse_args() + +# Read vtk +V=readVTK('../data.0000.vtk') + +# Spherical coordinates +nr, nth, nphi = np.squeeze(V.nx), np.squeeze(V.ny), np.squeeze(V.nz) +r = np.squeeze(V.r) +th = np.squeeze(V.theta) +phi = np.squeeze(V.phi) +rg, thg, phig = np.meshgrid(r, th, phi, indexing="ij") + +# Cartesian coordinates +x = rg*np.sin(thg)*np.cos(phig) +y = rg*np.sin(thg)*np.sin(phig) +z = rg*np.cos(thg) + +# Density (for norm) +rho = np.squeeze(V.data['RHO']) + +# Constrained potential +phi_f = np.squeeze(V.data['phiP']) + +# Theoretical parameters +r0 = 3. +delta = 1. +rho0 = 3./(4.*np.pi) +G = 1. +M = 4.*np.pi/3.*delta**3*rho0 + +# Theoretical potential +dr = np.sqrt((x-r0)**2 + y**2 + z**2) +sph = np.where(dr <= delta) # Sphere delimitation +phi_th = -G*M/dr # Out of the sphere +phi_th[sph] = - (G*M)/(2*delta**3)*(3*delta**2 - dr[sph]**2) # In the sphere + +# Recovering gradients to get rid of the integration constant +gth, gf = np.zeros(phi_f.shape), np.zeros(phi_f.shape) +for i in range(nphi): + for j in range(nth): + gth[:,j,i] = -np.gradient(phi_th[:,j,i], r, axis=0) + gf[:,j,i] = -np.gradient(phi_f[:,j,i], r, axis=0) + +# Recovering the norm (density ponderated like the solver) +num = np.linalg.norm(gf-gth) +denom = np.linalg.norm(rho) +norm = num/denom +print(norm) + +# Display a check plot if necessary +if(not args.noplot): + plt.plot(r, gf[:,nth//2-1,0], 'c-', label='cpt') + plt.plot(r, gth[:,nth//2-1,0], 'k--', label='th') + plt.title('Gravitational gradient along the sphere solid angle') + plt.xlabel('r') + plt.ylabel('g') + plt.legend() + plt.show() + +# Print the result of the test +print("Error=%e"%norm) +if norm<1.5e-1: + print("SUCCESS!") + sys.exit(0) +else: + print("FAILURE!") + sys.exit(1) diff --git a/test/SelfGravity/RandomSphere/setup.cpp b/test/SelfGravity/RandomSphere/setup.cpp new file mode 100644 index 00000000..ce545b0a --- /dev/null +++ b/test/SelfGravity/RandomSphere/setup.cpp @@ -0,0 +1,66 @@ +#include + +#include "idefix.hpp" +#include "setup.hpp" + +real delta, r0, rho0; + +// Compute user variables which will be written in vtk files +void ComputeUserVars(DataBlock & data, UserDefVariablesContainer &variables) { + // Force compute the gravity field + idfx::cout << "ComputeUserVars: Computing total gravity field..." << std::endl; + data.gravity->ComputeGravity(0); + idfx::cout << "ComputeUserVars: Done." << std::endl; + + // Make references to the user-defined arrays (variables is a container of IdefixHostArray3D) + // Note that the labels should match the variable names in the input file + // Storing into variables + Kokkos::deep_copy(variables["phiP"],data.gravity->phiP); +} + + +// Default constructor + + +// Initialisation routine. Can be used to allocate +// Arrays or variables which are used later on +Setup::Setup(Input &input, Grid &grid, DataBlock &data, Output &output) { + // Enroll output functions + output.EnrollUserDefVariables(&ComputeUserVars); +} + +// This routine initialize the flow +// Note that data is on the device. +// One can therefore define locally +// a datahost and sync it, if needed +void Setup::InitFlow(DataBlock &data) { + // Create a host copy + DataBlockHost d(data); + + // Grid parameters + IdefixHostArray1D r = d.x[IDIR]; + IdefixHostArray1D th = d.x[JDIR]; + IdefixHostArray1D phi = d.x[KDIR]; + + // Sphere's parameters + r0 = 3.0; // Sphere's center position + delta = 1.0; // Sphere's radius + rho0 = 3.0/(4.*M_PI); // Sphere's density + + // Filling initial profiles + for(int k = 0; k < d.np_tot[KDIR] ; k++) { + for(int j = 0; j < d.np_tot[JDIR] ; j++) { + for(int i = 0; i < d.np_tot[IDIR] ; i++) { + real x, y, z; + x = r(i)*sin(th(j))*cos(phi(k)); + y = r(i)*sin(th(j))*sin(phi(k)); + z = r(i)*cos(th(j)); + real dr = sqrt((x-r0)*(x-r0) + y*y + z*z); + d.Vc(RHO,k,j,i) = (dr<=delta) ? rho0 : ZERO_F; + } + } + } + + // Send it all, if needed + d.SyncToDevice(); +} diff --git a/test/SelfGravity/RandomSphere/testme.py b/test/SelfGravity/RandomSphere/testme.py new file mode 100755 index 00000000..beafdd77 --- /dev/null +++ b/test/SelfGravity/RandomSphere/testme.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python3 + +""" + +@author: glesur +""" +import os +import sys +sys.path.append(os.getenv("IDEFIX_DIR")) + +import pytools.idfx_test as tst + +def testMe(test): + test.configure() + test.compile() + inifiles=["idefix.ini","idefix-cg.ini","idefix-minres.ini"] + + # loop on all the ini files for this test + for ini in inifiles: + test.run(inputFile=ini) + #if test.init: + # test.makeReference(filename=name) + test.standardTest() + # since the gravitationnal potential is not included in .dmp files, we can't perform + # the full non-regression test + #test.nonRegressionTest(filename=name) + + +test=tst.idfxTest() +if not test.all: + testMe(test) +else: + test.noplot=True + testMe(test) diff --git a/test/SelfGravity/RandomSphereCartesian/definitions.hpp b/test/SelfGravity/RandomSphereCartesian/definitions.hpp new file mode 100644 index 00000000..0eebd2c1 --- /dev/null +++ b/test/SelfGravity/RandomSphereCartesian/definitions.hpp @@ -0,0 +1,7 @@ +#define COMPONENTS 3 +#define DIMENSIONS 3 + +#define GEOMETRY CARTESIAN +// Order of the scheme. 1=donnor cell, 2= linear reconstruction +#define ORDER 2 +#define ISOTHERMAL diff --git a/test/SelfGravity/RandomSphereCartesian/idefix-cg.ini b/test/SelfGravity/RandomSphereCartesian/idefix-cg.ini new file mode 100644 index 00000000..ae9e7007 --- /dev/null +++ b/test/SelfGravity/RandomSphereCartesian/idefix-cg.ini @@ -0,0 +1,47 @@ +[Grid] +X1-grid 1 -0.5 64 u 0.5 +X2-grid 1 -0.5 64 u 0.5 +X3-grid 1 -0.5 64 u 0.5 + +[TimeIntegrator] +CFL 0.8 +CFL_max_var 1.1 +tstop 0.0 +first_dt 1.e-4 +nstages 2 + +[Hydro] +solver roe +csiso constant 1.0 + +[Gravity] +potential selfgravity +gravCst 1.0 + +[SelfGravity] +solver CG +targetError 1e-4 +boundary-X1-beg periodic +boundary-X1-end periodic +boundary-X2-beg periodic +boundary-X2-end periodic +boundary-X3-beg periodic +boundary-X3-end periodic + +[Setup] +x0 0.0 +y0 0.0 +z0 0.0 +r0 0.1 + +[Boundary] +X1-beg periodic +X1-end periodic +X2-beg periodic +X2-end periodic +X3-beg periodic +X3-end periodic + +[Output] +vtk 1.e-4 +uservar phiP diff --git a/test/SelfGravity/RandomSphereCartesian/idefix-jacobi.ini b/test/SelfGravity/RandomSphereCartesian/idefix-jacobi.ini new file mode 100644 index 00000000..75d5bde9 --- /dev/null +++ b/test/SelfGravity/RandomSphereCartesian/idefix-jacobi.ini @@ -0,0 +1,48 @@ +[Grid] +X1-grid 1 -0.5 64 u 0.5 +X2-grid 1 -0.5 64 u 0.5 +X3-grid 1 -0.5 64 u 0.5 + +[TimeIntegrator] +CFL 0.8 +CFL_max_var 1.1 +tstop 0.0 +first_dt 1.e-4 +nstages 2 + +[Hydro] +solver roe +csiso constant 1.0 + +[Gravity] +potential selfgravity +gravCst 1.0 + +[SelfGravity] +solver Jacobi +targetError 1e-4 +maxIter 10000 +boundary-X1-beg periodic +boundary-X1-end periodic +boundary-X2-beg periodic +boundary-X2-end periodic +boundary-X3-beg periodic +boundary-X3-end periodic + +[Setup] +x0 0.0 +y0 0.0 +z0 0.0 +r0 0.1 + +[Boundary] +X1-beg periodic +X1-end periodic +X2-beg periodic +X2-end periodic +X3-beg periodic +X3-end periodic + +[Output] +vtk 1.e-4 +uservar phiP diff --git a/test/SelfGravity/RandomSphereCartesian/idefix-minres.ini b/test/SelfGravity/RandomSphereCartesian/idefix-minres.ini new file mode 100644 index 00000000..6188b280 --- /dev/null +++ b/test/SelfGravity/RandomSphereCartesian/idefix-minres.ini @@ -0,0 +1,47 @@ +[Grid] +X1-grid 1 -0.5 64 u 0.5 +X2-grid 1 -0.5 64 u 0.5 +X3-grid 1 -0.5 64 u 0.5 + +[TimeIntegrator] +CFL 0.8 +CFL_max_var 1.1 +tstop 0.0 +first_dt 1.e-4 +nstages 2 + +[Hydro] +solver roe +csiso constant 1.0 + +[Gravity] +potential selfgravity +gravCst 1.0 + +[SelfGravity] +solver MINRES +targetError 1e-4 +boundary-X1-beg periodic +boundary-X1-end periodic +boundary-X2-beg periodic +boundary-X2-end periodic +boundary-X3-beg periodic +boundary-X3-end periodic + +[Setup] +x0 0.0 +y0 0.0 +z0 0.0 +r0 0.1 + +[Boundary] +X1-beg periodic +X1-end periodic +X2-beg periodic +X2-end periodic +X3-beg periodic +X3-end periodic + +[Output] +vtk 1.e-4 +uservar phiP diff --git a/test/SelfGravity/RandomSphereCartesian/idefix.ini b/test/SelfGravity/RandomSphereCartesian/idefix.ini new file mode 100644 index 00000000..dbb3bce2 --- /dev/null +++ b/test/SelfGravity/RandomSphereCartesian/idefix.ini @@ -0,0 +1,47 @@ +[Grid] +X1-grid 1 -0.5 64 u 0.5 +X2-grid 1 -0.5 64 u 0.5 +X3-grid 1 -0.5 64 u 0.5 + +[TimeIntegrator] +CFL 0.8 +CFL_max_var 1.1 +tstop 0.0 +first_dt 1.e-4 +nstages 2 + +[Hydro] +solver roe +csiso constant 1.0 + +[Gravity] +potential selfgravity +gravCst 1.0 + +[SelfGravity] +solver BICGSTAB +targetError 1e-4 +boundary-X1-beg periodic +boundary-X1-end periodic +boundary-X2-beg periodic +boundary-X2-end periodic +boundary-X3-beg periodic +boundary-X3-end periodic + +[Setup] +x0 0.1 +y0 0.05 +z0 -0.15 +r0 0.1 + +[Boundary] +X1-beg periodic +X1-end periodic +X2-beg periodic +X2-end periodic +X3-beg periodic +X3-end periodic + +[Output] +vtk 1.e-4 +uservar phiP diff --git a/test/SelfGravity/RandomSphereCartesian/python/testidefix.py b/test/SelfGravity/RandomSphereCartesian/python/testidefix.py new file mode 100755 index 00000000..7367b3d9 --- /dev/null +++ b/test/SelfGravity/RandomSphereCartesian/python/testidefix.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +Created on Thu Mar 5 11:29:41 2020 + +@author: glesur +""" + +import os +import sys +import numpy as np +import matplotlib.pyplot as plt +import numpy.fft as fft +import argparse + +sys.path.append(os.getenv("IDEFIX_DIR")) + +from pytools.vtk_io import readVTK + +# Compute theoretical psi from spectral solution to Laplace equation + +def getPsiTheoretical(x,y,z,rho): + + n=x.shape[0] + + # Compute fourier transform and k vector + k1D=2.0*np.pi*fft.fftfreq(n,1/n) + [kx,ky,kz]=np.meshgrid(k1D,k1D,k1D,indexing='ij') + k2=(kx**2+ky**2+kz**2) + # avoid divide by 0 + k2[0,0,0]=1.0 + ik2=1.0/k2 + # kill zero frequency + ik2[0,0,0]=0 + + rhof=fft.fft2(rho,axes=(0,1,2)) + psif=-rhof*ik2 + psi=4.0*np.pi*np.real(fft.ifft2(psif,axes=(0,1,2))) + return(psi) + +# Add noplot argument to parser +parser = argparse.ArgumentParser() +parser.add_argument("-noplot", + default=False, + help="disable plotting", + action="store_true") + +args=parser.parse_args() + +# Read vtk +V=readVTK('../data.0000.vtk') + + + +# Cartesian coordinates +x = V.x +y = V.y +z = V.z + +# Density (for norm) +rho = V.data['RHO'] + +# Constrained potential +phi_f = V.data['phiP'] + +phi_th=getPsiTheoretical(x,y,z,rho) + + + +# Recovering gradients to get rid of the integration constant +gx, gy, gz = np.zeros(phi_f.shape), np.zeros(phi_f.shape), np.zeros(phi_f.shape) +gx_f=-np.gradient(phi_f,axis=0) +gx_th=-np.gradient(phi_th,axis=0) +gy_f=-np.gradient(phi_f,axis=1) +gy_th=-np.gradient(phi_th,axis=1) +gz_f=-np.gradient(phi_f,axis=2) +gz_th=-np.gradient(phi_th,axis=2) + +norm=np.sqrt(np.mean((gx_f-gx_th)**2+(gy_f-gy_th)**2+(gz_f-gz_th)**2)/np.mean(gx_th**2+gy_th**2+gz_th**2)) + +# Display a check plot if necessary +if(not args.noplot): + n=x.size//2 + plt.plot(x,gx_f[:,n,n], 'c-', label='cpt') + plt.plot(x, gx_th[:,n,n], 'k--', label='th') + plt.xlabel('r') + plt.ylabel('g') + plt.legend() + plt.show() + +# Print the result of the test +print("Error=%e"%norm) +if norm<1e-2: + print("SUCCESS!") + sys.exit(0) +else: + print("FAILURE!") + sys.exit(1) diff --git a/test/SelfGravity/RandomSphereCartesian/setup.cpp b/test/SelfGravity/RandomSphereCartesian/setup.cpp new file mode 100644 index 00000000..7eaf4411 --- /dev/null +++ b/test/SelfGravity/RandomSphereCartesian/setup.cpp @@ -0,0 +1,64 @@ +#include + +#include "idefix.hpp" +#include "setup.hpp" + +real inx0, iny0, inz0, inr0; + +// Compute user variables which will be written in vtk files +void ComputeUserVars(DataBlock & data, UserDefVariablesContainer &variables) { + // Force compute the gravity field + idfx::cout << "ComputeUserVars: Computing total gravity field..." << std::endl; + data.gravity->ComputeGravity(0); + idfx::cout << "ComputeUserVars: Done." << std::endl; + + // Make references to the user-defined arrays (variables is a container of IdefixHostArray3D) + // Note that the labels should match the variable names in the input file + // Storing into variables + Kokkos::deep_copy(variables["phiP"],data.gravity->phiP); +} + + +// Default constructor + + +// Initialisation routine. Can be used to allocate +// Arrays or variables which are used later on +Setup::Setup(Input &input, Grid &grid, DataBlock &data, Output &output) { + // Enroll output functions + output.EnrollUserDefVariables(&ComputeUserVars); + inx0 = input.Get("Setup","x0",0); + iny0 = input.Get("Setup","y0",0); + inz0 = input.Get("Setup","z0",0); + inr0 = input.Get("Setup","r0",0); +} + +// This routine initialize the flow +// Note that data is on the device. +// One can therefore define locally +// a datahost and sync it, if needed +void Setup::InitFlow(DataBlock &data) { + // Create a host copy + DataBlockHost d(data); + + real rho0 = 1.0; // Sphere's density + real x0=inx0; + real y0=iny0; + real z0=inz0; + real r0=inr0; + // Filling initial profiles + for(int k = 0; k < d.np_tot[KDIR] ; k++) { + for(int j = 0; j < d.np_tot[JDIR] ; j++) { + for(int i = 0; i < d.np_tot[IDIR] ; i++) { + real dx = d.x[IDIR](i)-x0; + real dy = d.x[JDIR](j)-y0; + real dz = d.x[KDIR](k)-z0; + real dr = sqrt(dx*dx+dy*dy+dz*dz); + d.Vc(RHO,k,j,i) = (dr<=r0) ? rho0 : ZERO_F; + } + } + } + + // Send it all, if needed + d.SyncToDevice(); +} diff --git a/test/SelfGravity/RandomSphereCartesian/testme.py b/test/SelfGravity/RandomSphereCartesian/testme.py new file mode 100755 index 00000000..64b4a9ba --- /dev/null +++ b/test/SelfGravity/RandomSphereCartesian/testme.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python3 + +""" + +@author: glesur +""" +import os +import sys +sys.path.append(os.getenv("IDEFIX_DIR")) + +import pytools.idfx_test as tst + +def testMe(test): + test.configure() + test.compile() + inifiles=["idefix.ini","idefix-cg.ini","idefix-minres.ini","idefix-jacobi.ini"] + + # loop on all the ini files for this test + for ini in inifiles: + test.run(inputFile=ini) + #if test.init: + # test.makeReference(filename=name) + test.standardTest() + # since the gravitationnal potential is not included in .dmp files, we can't perform + # the full non-regression test + #test.nonRegressionTest(filename=name) + + +test=tst.idfxTest() +if not test.all: + testMe(test) +else: + test.noplot=True + testMe(test) diff --git a/test/SelfGravity/UniformCollapse/CMakeLists.txt b/test/SelfGravity/UniformCollapse/CMakeLists.txt new file mode 100644 index 00000000..fe954712 --- /dev/null +++ b/test/SelfGravity/UniformCollapse/CMakeLists.txt @@ -0,0 +1 @@ +#replace_idefix_source(main.cpp main.cpp) diff --git a/test/SelfGravity/UniformCollapse/definitions.hpp b/test/SelfGravity/UniformCollapse/definitions.hpp new file mode 100644 index 00000000..923795c2 --- /dev/null +++ b/test/SelfGravity/UniformCollapse/definitions.hpp @@ -0,0 +1,7 @@ +#define COMPONENTS 1 +#define DIMENSIONS 1 +#define ISOTHERMAL + +#define GEOMETRY SPHERICAL +// Order of the scheme. 1=donnor cell, 2= linear reconstruction +#define ORDER 2 diff --git a/test/SelfGravity/UniformCollapse/idefix.ini b/test/SelfGravity/UniformCollapse/idefix.ini new file mode 100644 index 00000000..d5f8e4a4 --- /dev/null +++ b/test/SelfGravity/UniformCollapse/idefix.ini @@ -0,0 +1,45 @@ +[Grid] +X1-grid 1 .01 100 l 1000. +X2-grid 1 0.0 20 u 3.141592653589793 +X3-grid 1 0.0 20 u 6.283185307179586 + +[TimeIntegrator] +CFL 0.8 +CFL_max_var 1.1 +tstop 50.0 +first_dt 1.e-4 +nstages 2 + +[Hydro] +solver hll +csiso constant 0.4 + +[Gravity] +potential selfgravity central +Mcentral 4.188790205e-9 +gravCst 0.07957747155 # 4piG=1.0 + +[SelfGravity] +solver PBICGSTAB +skip 5 +targetError 1e-6 +boundary-X1-beg origin +boundary-X1-end nullpot +boundary-X2-beg periodic +boundary-X2-end periodic +boundary-X3-beg periodic +boundary-X3-end periodic + +[Boundary] +X1-beg userdef +X1-end outflow +X2-beg periodic +X2-end periodic +X3-beg periodic +X3-end periodic + +[Output] +analysis 10. +vtk 10. +dmp 50.0 +uservar phiP diff --git a/test/SelfGravity/UniformCollapse/python/testidefix.py b/test/SelfGravity/UniformCollapse/python/testidefix.py new file mode 100755 index 00000000..e89a296a --- /dev/null +++ b/test/SelfGravity/UniformCollapse/python/testidefix.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +Created on Thu Mar 5 11:29:41 2020 + +@author: glesur +""" + +import os +import sys +sys.path.append(os.getenv("IDEFIX_DIR")) +from pytools.vtk_io import readVTK +import numpy as np + +# Arguments +gravCst = 1./(4.*np.pi) # grav cst 4piG +rho0 = 1e-3 # initial density + +# Parameters (calculated from the arguments) +tff = np.sqrt(3.*np.pi/(32.*rho0*gravCst)) # Theoretical free-fall time to achieve collapse + +# Collecting data +V=readVTK('../data.0005.vtk') +time = V.t[0] +rho = np.squeeze(V.data['RHO']) + +# A function to detect density plateau, its size and position +def get_plateau(rho): + plateau = [rho[0]] + for dsty in rho[1:]: + if np.abs(dsty-plateau[-1])>0.01*plateau[-1]: + return np.mean(plateau) + else: + plateau.append(dsty) + return "Error : the whole density distribution is approximately constant !" + +# Isolating density plateau +plateau=get_plateau(rho) + +# Calculating absolute error +r_th=time/tff # Theoretical ratio +eta = np.arccos((plateau/rho0)**(-1./6.)) +r_num = 2./np.pi*(eta + 1./2.*np.sin(2.*eta)) # Numerical ratio +error = np.abs(r_num-r_th) # Absolute error + +# Print the result of the test +print("Error=%e"%error) +if error<2.0e-3: + print("SUCCESS!") + sys.exit(0) +else: + print("FAILURE!") + sys.exit(1) diff --git a/test/SelfGravity/UniformCollapse/setup.cpp b/test/SelfGravity/UniformCollapse/setup.cpp new file mode 100644 index 00000000..e573bfae --- /dev/null +++ b/test/SelfGravity/UniformCollapse/setup.cpp @@ -0,0 +1,142 @@ +#include + +#include "idefix.hpp" +#include "setup.hpp" + +real RcloudGlob; +real internalAreaGlob; + +// Analyse data to produce an ascii output +void Analysis(DataBlock & data) { + // Get the usefull parameters + real centralMass = data.gravity->centralMass; + real time = data.t; + + if(time==0.){ + // Write the data in ascii to our file + std::ofstream f; + f.open("timevol.dat",std::ios::trunc); + f.precision(15); + f << std::scientific << time << "\t" << centralMass << std::endl; + f.close(); + } else { + // Append the data in ascii to our file + std::ofstream f; + f.open("timevol.dat",std::ios::app); + f.precision(15); + f << std::scientific << time << "\t" << centralMass << std::endl; + f.close(); + } +} + +// Compute user variables which will be written in vtk files +void ComputeUserVars(DataBlock & data, UserDefVariablesContainer &variables) { + // Make references to the user-defined arrays (variables is a container of IdefixHostArray3D) + // Note that the labels should match the variable names in the input file + IdefixHostArray3D phiP = variables["phiP"]; + Kokkos::deep_copy(phiP, data.gravity->phiP); +} + +// User-defined boundaries +void UserdefBoundary(Hydro *hydro, int dir, BoundarySide side, real t) { + auto *data = hydro->data; + if((dir==IDIR) && (side == left)) { + IdefixArray4D Vc = hydro->Vc; + int ighost = data->nghost[IDIR]; + IdefixArray1D r = data->x[IDIR]; + + idefix_for("UserDefBoundary", + 0, data->np_tot[KDIR], + 0, data->np_tot[JDIR], + 0, ighost, + KOKKOS_LAMBDA (int k, int j, int i) { + Vc(RHO,k,j,i) = Vc(RHO,k,j,ighost); + // We cap radial velocity at zero following Xu & Kunz 2021 I. + if(Vc(VX1,k,j,ighost)>=ZERO_F) { + Vc(VX1,k,j,i) = ZERO_F; + } else { + Vc(VX1,k,j,i) = Vc(VX1,k,j,ighost); + } + }); + } +} + +void FluxBoundary(DataBlock & data, int dir, BoundarySide side, const real t) { + if((dir==IDIR) && (side == left)) { + // Loading needed data + IdefixArray4D Flux = data.hydro->FluxRiemann; + real halfDt = data.dt/2.; // RK2, dt is actually half at each flux calculation + int iref = data.nghost[IDIR]; + real rin = data.xbeg[IDIR]; + real internalArea = internalAreaGlob; + + // Storage variables + real fluxTotMass; + + // Setting positive mass flux to zero to avoid mass flowing in the + // domain (Xu & Kunz 2021 I.) + idefix_for("SetZeroOutwardFlux", + data.beg[KDIR], data.end[KDIR], + data.beg[JDIR], data.end[JDIR], + iref, iref+1, + KOKKOS_LAMBDA (int k, int j, int i) { + if(Flux(RHO,k,j,i)>=ZERO_F) { + Flux(RHO,k,j,i)=ZERO_F; + } + }); + + idefix_reduce("ComputeTotalMassFlux", + data.beg[KDIR], data.end[KDIR], + data.beg[JDIR], data.end[JDIR], + iref, iref+1, + KOKKOS_LAMBDA (int k, int j, int i, real &localSum) { + localSum += Flux(RHO,k,j,i); + }, + Kokkos::Sum(fluxTotMass)); + + // We normalize by the considered totalA then extrapolate to the whole + // sphere at given radius (cause we're only 1D) + data.gravity->centralMass -= fluxTotMass*halfDt*4.*M_PI*rin*rin/internalArea; + } +} + +// Default constructor + + +// Initialisation routine. Can be used to allocate +// Arrays or variables which are used later on +Setup::Setup(Input &input, Grid &grid, DataBlock &data, Output &output) { + data.hydro->EnrollUserDefBoundary(&UserdefBoundary); + data.hydro->EnrollFluxBoundary(&FluxBoundary); + output.EnrollUserDefVariables(&ComputeUserVars); + output.EnrollAnalysis(&Analysis); + RcloudGlob = 60.; + internalAreaGlob = grid.xbeg[IDIR]*grid.xbeg[IDIR]; +} + +// This routine initialize the flow +// Note that data is on the device. +// One can therefore define locally +// a datahost and sync it, if needed +void Setup::InitFlow(DataBlock &data) { + // Create a host copy + DataBlockHost d(data); + + // Grid and block parameters + IdefixHostArray4D Vc = d.Vc; + IdefixHostArray1D r = d.x[IDIR]; + real Rcloud = RcloudGlob; + + // Filling initial profiles + for(int k = 0; k < d.np_tot[KDIR] ; k++) { + for(int j = 0; j < d.np_tot[JDIR] ; j++) { + for(int i = 0; i < d.np_tot[IDIR] ; i++) { + Vc(RHO,k,j,i) = (r(i)<=Rcloud)? 0.001 : 0.00001 ; + Vc(VX1,k,j,i) = ZERO_F; + } + } + } + + // Send it all, if needed + d.SyncToDevice(); +} diff --git a/test/SelfGravity/UniformCollapse/testme.py b/test/SelfGravity/UniformCollapse/testme.py new file mode 100755 index 00000000..21debc0e --- /dev/null +++ b/test/SelfGravity/UniformCollapse/testme.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 + +""" + +@author: glesur +""" +import os +import sys +sys.path.append(os.getenv("IDEFIX_DIR")) + +import pytools.idfx_test as tst + +name="dump.0001.dmp" + +tolerance=1e-13 + +def testMe(test): + test.configure() + test.compile() + inifiles=["idefix.ini"] + + # loop on all the ini files for this test + for ini in inifiles: + test.run(inputFile=ini) + test.standardTest() + + +test=tst.idfxTest() + +if not test.all: + if(test.check): + test.standardTest() + else: + testMe(test) +else: + test.noplot = True + test.vectPot=False + test.single=False + test.reconstruction=2 + test.mpi=False + testMe(test) diff --git a/test/checks_examples.sh b/test/checks_examples.sh index d563aac9..4cca1621 100755 --- a/test/checks_examples.sh +++ b/test/checks_examples.sh @@ -2,7 +2,7 @@ # check that the provided examples (which do not have a reference case) compile and run for a few loops -rep_example_list="HD/KHI HD/RWI-cavity HD/VSI MHD/RotorCartesian MHD/RotorPolar MHD/advectionOBlique MHD/AmbipolarShearingBox MHD/AmbipolarWind MHD/FieldLoop MHD/HallDisk MHD/disk MHD/diskSpherical" +rep_example_list="HD/KHI HD/RWI-cavity HD/VSI MHD/RotorCartesian MHD/RotorPolar MHD/advectionOBlique MHD/AmbipolarShearingBox MHD/AmbipolarWind MHD/FieldLoop MHD/HallDisk MHD/disk MHD/diskSpherical Dust/StreamingInstability Dust/FargoPlanet" # refer to the parent dir of this file, wherever this is called from # a python equivalent is e.g. @@ -70,7 +70,7 @@ for rep in $rep_example_list; do echo "***********************************************" echo "Cleaning $rep in $TMP_DIR" echo "***********************************************" - rm -rf *.vtk *.dbl *.dmp *.ini python CMakeLists.txt + rm -rf *.vtk *.dbl *.dmp *.ini python CMakeLists.txt CMakeCache.txt done diff --git a/test/checks_highorder.sh b/test/checks_highorder.sh deleted file mode 100755 index 9b55974d..00000000 --- a/test/checks_highorder.sh +++ /dev/null @@ -1,125 +0,0 @@ -#!/bin/bash - -rep_list="HD/sod-iso HD/sod MHD/sod MHD/sod-iso MHD/LinearWaveTest MHD/OrszagTang" -rep_MPI_list="MHD/OrszagTang3D" -order_list="LimO3 Parabolic" - -# refer to the parent dir of this file, wherever this is called from -# a python equivalent is e.g. -# -# import pathlib -# TEST_DIR = pathlib.Path(__file__).parent -TEST_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" - - -function resolve_path { - # resolve relative paths - # work around the fact that `realpath` is not bundled with every UNIX distro - echo "`cd "$1";pwd`" -} - -target_dir=$(resolve_path $TEST_DIR/..) - -if [ -z ${var+IDEFIX_DIR} ] & [ -d "$IDEFIX_DIR" ] ; then - global_dir=$(resolve_path $IDEFIX_DIR) - if [ $target_dir != $global_dir ] ; then - echo \ - "Warning: IDEFIX_DIR is set globally to $global_dir" \ - ", but is redefined to $target_dir" - fi -fi - -export IDEFIX_DIR=$target_dir - -set -e -options=$@ - -TMP_DIR="$(mktemp -d)" -function finish () -{ - echo $1 - echo "Cleaning directory $TMP_DIR" - cd $TEST_DIR - rm -rf $TMP_DIR - exit 1 -} - -# High order tests -for rep in $rep_list; do - cp -R $TEST_DIR/$rep/* $TMP_DIR - cd $TMP_DIR - echo "***********************************************" - echo "Configuring $rep" - echo "Using $TMP_DIR as working directory" - echo "***********************************************" - for order in $order_list; do - - cmake $IDEFIX_DIR $options -DIdefix_RECONSTRUCTION=$order || finish "!!!!$rep with $order failed during configuration" - echo "***********************************************" - echo "Making $rep with $order" - echo "***********************************************" - make -j 8 || finish "!!!! $rep with $order failed during compilation" - - ini_files=$(ls *.ini) - for ini in $ini_files; do - echo "***********************************************" - echo "Running $rep with $order and $ini" - echo "***********************************************" - ./idefix -i $ini -nolog || finish "!!!! $rep with $order failed running with $ini" - - cd python - echo "***********************************************" - echo "Testing $rep with $order and $ini" - echo "***********************************************" - python3 testidefix.py -noplot || finish "!!!! $rep with $order failed validation with $ini" - cd .. - done - rm -f *.vtk *.dbl *.dmp - done - echo "***********************************************" - echo "Cleaning $rep in $TMP_DIR" - echo "***********************************************" - rm -rf *.vtk *.dbl *.dmp *.ini python CMakeLists.txt -done - -#do it with MPI (only the default .ini files though) -# High order tests -for rep in $rep_MPI_list; do - cp -R $TEST_DIR/$rep/* $TMP_DIR - cd $TMP_DIR - echo "***********************************************" - echo "Configuring $rep" - echo "Using $TMP_DIR as working directory" - echo "***********************************************" - - for order in $order_list; do - - cmake $IDEFIX_DIR $options -DIdefix_RECONSTRUCTION=$order -DIdefix_MPI=ON || finish "!!!!$rep with $order failed during configuration" - echo "***********************************************" - echo "Making $rep with $order" - echo "***********************************************" - make -j 8 || finish "!!!! $rep with $order and MPI failed during compilation with" - - echo "***********************************************" - echo "Running $rep with $order and MPI" - echo "***********************************************" - mpirun -np 4 ./idefix || finish "!!!! $rep with $order and MPI failed running " - - cd python - echo "***********************************************" - echo "Testing $rep with $order and MPI" - echo "***********************************************" - python3 testidefix.py -noplot || finish "!!!! $rep with $order and MPI failed validation" - cd .. - - rm -f *.vtk *.dbl *.dmp - done - echo "***********************************************" - echo "Cleaning $rep in $TMP_DIR" - echo "***********************************************" - rm -rf *.vtk *.dbl *.dmp *.ini python CMakeLists.txt -done - -echo "Test was successfull" -cd $TEST_DIR -rm -rf $TMP_DIR diff --git a/test/checks_hydro.sh b/test/checks_hydro.sh deleted file mode 100755 index 5313dad0..00000000 --- a/test/checks_hydro.sh +++ /dev/null @@ -1,85 +0,0 @@ -#!/bin/bash - -rep_HD_list="sod-iso sod MachReflection ViscousFlowPastCylinder ViscousDisk FargoPlanet ShearingBox thermalDiffusion" - -# refer to the parent dir of this file, wherever this is called from -# a python equivalent is e.g. -# -# import pathlib -# TEST_DIR = pathlib.Path(__file__).parent -TEST_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" - -function resolve_path { - # resolve relative paths - # work around the fact that `realpath` is not bundled with every UNIX distro - echo "`cd "$1";pwd`" -} - -target_dir=$(resolve_path $TEST_DIR/..) - -if [ -z ${var+IDEFIX_DIR} ] & [ -d "$IDEFIX_DIR" ] ; then - global_dir=$(resolve_path $IDEFIX_DIR) - if [ $target_dir != $global_dir ] ; then - echo \ - "Warning: IDEFIX_DIR is set globally to $global_dir" \ - ", but is redefined to $target_dir" - fi -fi - -export IDEFIX_DIR=$target_dir -echo $IDEFIX_DIR - -set -e -options=$@ - -# HD tests -TMP_DIR="$(mktemp -d)" -function finish () -{ - echo $1 - echo "Cleaning directory $TMP_DIR" - cd $TEST_DIR - rm -rf $TMP_DIR - exit 1 -} - -for rep in $rep_HD_list; do - cp -R $TEST_DIR/HD/$rep/* $TMP_DIR - cd $TMP_DIR - echo "***********************************************" - echo "Configuring $rep" - echo "Using $TMP_DIR as working directory" - echo "***********************************************" - def_files=$(ls definitions*.hpp) - for def in $def_files; do - - cmake $IDEFIX_DIR $options -DIdefix_DEFS=$def|| finish "!!!! HD $rep failed during configuration" - echo "***********************************************" - echo "Making $rep with $def" - echo "***********************************************" - make -j 8 || finish "!!!! HD $rep failed during compilation with $def" - - ini_files=$(ls *.ini) - for ini in $ini_files; do - echo "***********************************************" - echo "Running $rep with $ini" - echo "***********************************************" - ./idefix -i $ini -nolog || finish "!!!! HD $rep failed running with $def and $ini" - - cd python - echo "***********************************************" - echo "Testing $rep with $ini and $def" - echo "***********************************************" - python3 testidefix.py -noplot || finish "!!!! HD $rep failed validation with $def and $ini" - cd .. - done - rm -f *.vtk *.dbl *.dmp - done - echo "***********************************************" - echo "Cleaning $rep in $TMP_DIR" - echo "***********************************************" - rm -rf *.vtk *.dbl *.dmp *.ini python CMakeLists.txt -done -echo "Test was successfull" -cd $TEST_DIR -rm -rf $TMP_DIR diff --git a/test/checks_mhd.sh b/test/checks_mhd.sh deleted file mode 100755 index 4748e7ab..00000000 --- a/test/checks_mhd.sh +++ /dev/null @@ -1,83 +0,0 @@ -#!/bin/bash -rep_MHD_list="sod-iso sod AxisFluxTube AmbipolarCshock HallWhistler ResistiveAlfvenWave LinearWaveTest FargoMHDSpherical ShearingBox OrszagTang OrszagTang3D" - -# refer to the parent dir of this file, wherever this is called from -# a python equivalent is e.g. -# -# import pathlib -# TEST_DIR = pathlib.Path(__file__).parent -TEST_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -TMP_DIR="$(mktemp -d)" - -function resolve_path { - # resolve relative paths - # work around the fact that `realpath` is not bundled with every UNIX distro - echo "`cd "$1";pwd`" -} - -target_dir=$(resolve_path $TEST_DIR/..) - -if [ -z ${var+IDEFIX_DIR} ] & [ -d "$IDEFIX_DIR" ] ; then - global_dir=$(resolve_path $IDEFIX_DIR) - if [ $target_dir != $global_dir ] ; then - echo \ - "Warning: IDEFIX_DIR is set globally to $global_dir" \ - ", but is redefined to $target_dir" - fi -fi - -export IDEFIX_DIR=$target_dir -echo $IDEFIX_DIR - -set -e -options=$@ - -function finish () -{ - echo $1 - echo "Cleaning directory $TMP_DIR" - cd $TEST_DIR - rm -rf $TMP_DIR - exit 1 -} - -# MHD tests -for rep in $rep_MHD_list; do - cp -R $TEST_DIR/MHD/$rep/* $TMP_DIR - cd $TMP_DIR - echo "***********************************************" - echo "Configuring $rep" - echo "Using $TMP_DIR/$rep as working directory" - echo "***********************************************" - def_files=$(ls definitions*.hpp) - for def in $def_files; do - cmake $IDEFIX_DIR $options -DIdefix_DEFS=$def|| finish "!!!! MHD $rep failed during configuration" - echo "***********************************************" - echo "Making $rep with $def" - echo "***********************************************" - make -j 8 || finish "!!!! MHD $rep failed during compilation with $def" - - ini_files=$(ls *.ini) - for ini in $ini_files; do - echo "***********************************************" - echo "Running $rep with $ini" - echo "***********************************************" - ./idefix -i $ini -nolog || finish "!!!! MHD $rep failed running with $def and $ini" - - cd python - echo "***********************************************" - echo "Testing $rep with $ini and $def" - echo "***********************************************" - python3 testidefix.py -noplot -i ../$ini || finish "!!!! MHD $rep failed validation with $def and $ini" - cd .. - done - rm -f *.vtk *.dbl *.dmp - done - echo "***********************************************" - echo "Cleaning $TMP_DIR" - echo "***********************************************" - rm -rf *.vtk *.dbl *.dmp *.ini python CMakeLists.txt -done -echo "Test was successfull" -cd $TEST_DIR -rm -rf $TMP_DIR diff --git a/test/checks_mpi.sh b/test/checks_mpi.sh deleted file mode 100755 index 60eef2b3..00000000 --- a/test/checks_mpi.sh +++ /dev/null @@ -1,197 +0,0 @@ -#!/bin/bash - -rep_2D_mpi_list="HD/MachReflection HD/ViscousFlowPastCylinder HD/FargoPlanet MHD/OrszagTang" -rep_3D_mpi_list="MHD/AxisFluxTube MHD/LinearWaveTest MHD/FargoMHDSpherical MHD/OrszagTang3D" -rep_3D_noX3_list="MHD/AmbipolarCshock3D" - -# refer to the parent dir of this file, wherever this is called from -# a python equivalent is e.g. -# -# import pathlib -# TEST_DIR = pathlib.Path(__file__).parent -TEST_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -TMP_DIR="$(mktemp -d)" - -function resolve_path { - # resolve relative paths - # work around the fact that `realpath` is not bundled with every UNIX distro - echo "`cd "$1";pwd`" -} - -function finish () -{ - echo $1 - echo "Cleaning directory $TMP_DIR" - cd $TEST_DIR - rm -rf $TMP_DIR - exit 1 -} - -target_dir=$(resolve_path $TEST_DIR/..) - -if [ -z ${var+IDEFIX_DIR} ] & [ -d "$IDEFIX_DIR" ] ; then - global_dir=$(resolve_path $IDEFIX_DIR) - if [ $target_dir != $global_dir ] ; then - echo \ - "Warning: IDEFIX_DIR is set globally to $global_dir" \ - ", but is redefined to $target_dir" - fi -fi - -export IDEFIX_DIR=$target_dir -echo $IDEFIX_DIR - -set -e -options=$@ - -# 2D MPI tests -for rep in $rep_2D_mpi_list; do - cp -R $TEST_DIR/$rep/* $TMP_DIR - cd $TMP_DIR - echo "***********************************************" - echo "Configuring $rep" - echo "Using $TMP_DIR/$rep as working directory" - echo "***********************************************" - cmake $IDEFIX_DIR -DIdefix_MPI=ON $options || finish "!!!! MPI $rep failed during configuration" - echo "***********************************************" - echo "Making $rep" - echo "***********************************************" - make -j 8 || finish "!!!! MPI $rep failed during compilation" - - ini_files=$(ls *.ini) - for ini in $ini_files; do - echo "***********************************************" - echo "Running $rep with $ini" - echo "***********************************************" - mpirun -np 4 ./idefix -i $ini -dec 2 2 -nolog || finish "!!!! MPI $rep failed during runtime with $ini" - - cd python - echo "***********************************************" - echo "Testing $rep with $ini" - echo "***********************************************" - python3 testidefix.py -noplot -i ../$ini || finish "!!!! MPI $rep failed during validation" - cd .. - done - rm -rf *.vtk *.dbl *.dmp *.ini python CMakeLists.txt -done - -# MHD tests -for rep in $rep_3D_noX3_list; do - cp -R $TEST_DIR/$rep/* $TMP_DIR - cd $TMP_DIR - echo "***********************************************" - echo "Configuring $rep" - echo "Using $TMP_DIR/$rep as working directory" - echo "***********************************************" - cmake $IDEFIX_DIR -DIdefix_MPI=ON $options || finish "!!!! MPI $rep failed during configuration" - echo "***********************************************" - echo "Making $rep" - echo "***********************************************" - make -j 8 || finish "!!!! MPI $rep failed during compilation" - - ini_files=$(ls *.ini) - for ini in $ini_files; do - echo "***********************************************" - echo "Running $rep with $ini" - echo "***********************************************" - mpirun -np 4 ./idefix -i $ini -dec 2 2 1 -nolog || finish "!!!! MPI $rep failed during runtime with $ini" - - cd python - echo "***********************************************" - echo "Testing $rep with $ini" - echo "***********************************************" - python3 testidefix.py -noplot || finish "!!!! MPI $rep failed during validation" - cd .. - done - rm -rf *.vtk *.dbl *.dmp *.ini python CMakeLists.txt - cd $TEST_DIR -done - -for rep in $rep_3D_mpi_list; do - cp -R $TEST_DIR/$rep/* $TMP_DIR - cd $TMP_DIR - echo "***********************************************" - echo "Configuring $rep" - echo "Using $TMP_DIR/$rep as working directory" - echo "***********************************************" - cmake $IDEFIX_DIR -DIdefix_MPI=ON $options || finish "!!!! MPI $rep failed during configuration" - echo "***********************************************" - echo "Making $rep" - echo "***********************************************" - make -j 8 || finish "!!!! MPI $rep failed during compilation" - - ini_files=$(ls *.ini) - for ini in $ini_files; do - echo "***********************************************" - echo "Running $rep with $ini" - echo "***********************************************" - mpirun -np 8 ./idefix -i $ini -dec 2 2 2 -nolog || finish "!!!! MPI $rep failed during runtime with $ini" - - cd python - echo "***********************************************" - echo "Testing $rep with $ini" - echo "***********************************************" - python3 testidefix.py -noplot -i ../$ini || finish "!!!! MPI $rep failed during validation" - cd .. - done - rm -rf *.vtk *.dbl *.dmp *.ini python CMakeLists.txt - cd $TEST_DIR -done - -# Blast wave test -rep="HD/SedovBlastWave" - -## Cartesian blast -cp -R $TEST_DIR/$rep/* $TMP_DIR -cd $TMP_DIR -echo "***********************************************" -echo "Configuring $rep" -echo "Using $TMP_DIR/$rep as working directory" -echo "***********************************************" -cmake $IDEFIX_DIR -DIdefix_MPI=ON -DIdefix_MHD=OFF $options || finish "!!!! MPI $rep failed during configuration" -echo "***********************************************" -echo "Making $rep" -echo "***********************************************" -make -j 8 || finish "!!!! MPI $rep failed during compilation" - -ini="idefix.ini" -echo "***********************************************" -echo "Running $rep with $ini" -echo "***********************************************" -mpirun -np 8 ./idefix -i $ini -dec 2 2 2 -nolog || finish "!!!! MPI $rep failed during runtime with $ini" - -cd python -echo "***********************************************" -echo "Testing $rep with $ini" -echo "***********************************************" -python3 testidefix.py -noplot -i ../$ini || finish "!!!! MPI $rep failed during validation" -cd .. - -## Spherical blast -echo "***********************************************" -echo "Configuring $rep in spherical geometry" -echo "Using $TMP_DIR/$rep as working directory" -echo "***********************************************" - -cmake $IDEFIX_DIR -DIdefix_MPI=ON -DIdefix_MHD=OFF -DIdefix_DEFS=definitions-spherical.hpp $options || finish "!!!! MPI $rep failed during configuration" -echo "***********************************************" -echo "Making $rep" -echo "***********************************************" -make -j 8 || finish "!!!! MPI $rep failed during compilation" - -ini="idefix-spherical.ini" -echo "***********************************************" -echo "Running $rep with $ini" -echo "***********************************************" -mpirun -np 8 ./idefix -i $ini -dec 2 2 2 -nolog || finish "!!!! MPI $rep failed during runtime with $ini" - -cd python -echo "***********************************************" -echo "Testing $rep with $ini" -echo "***********************************************" -python3 testidefix.py -noplot -i ../$ini || finish "!!!! MPI $rep failed during validation" - -## done -echo "Test was successfull" -cd $TEST_DIR -rm -rf $TMP_DIR diff --git a/test/checks_singleprecision.sh b/test/checks_singleprecision.sh deleted file mode 100755 index ef7fe895..00000000 --- a/test/checks_singleprecision.sh +++ /dev/null @@ -1,119 +0,0 @@ -#!/bin/bash - -rep_list="HD/sod-iso HD/sod MHD/sod MHD/sod-iso MHD/OrszagTang" -rep_MPI_list="MHD/OrszagTang3D" - -# refer to the parent dir of this file, wherever this is called from -# a python equivalent is e.g. -# -# import pathlib -# TEST_DIR = pathlib.Path(__file__).parent -TEST_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" - - -function resolve_path { - # resolve relative paths - # work around the fact that `realpath` is not bundled with every UNIX distro - echo "`cd "$1";pwd`" -} - -target_dir=$(resolve_path $TEST_DIR/..) - -if [ -z ${var+IDEFIX_DIR} ] & [ -d "$IDEFIX_DIR" ] ; then - global_dir=$(resolve_path $IDEFIX_DIR) - if [ $target_dir != $global_dir ] ; then - echo \ - "Warning: IDEFIX_DIR is set globally to $global_dir" \ - ", but is redefined to $target_dir" - fi -fi - -export IDEFIX_DIR=$target_dir - - -set -e -options=$@ - -TMP_DIR="$(mktemp -d)" -function finish () -{ - echo $1 - echo "Cleaning directory $TMP_DIR" - cd $TEST_DIR - rm -rf $TMP_DIR - exit 1 -} - -# High order tests -for rep in $rep_list; do - - cp -R $TEST_DIR/$rep/* $TMP_DIR - cd $TMP_DIR - echo "***********************************************" - echo "Configuring $rep" - echo "Using $TMP_DIR as working directory" - echo "***********************************************" - cmake $IDEFIX_DIR $options -DIdefix_PRECISION=Single || finish "!!!!$rep in single precision failed during configuration" - echo "***********************************************" - echo "Making $rep in single precision" - echo "***********************************************" - make -j 8 || finish "!!!! $rep in single precision failed during compilation with" - - ini_files=$(ls *.ini) - for ini in $ini_files; do - echo "***********************************************" - echo "Running $rep in single precision and $ini" - echo "***********************************************" - ./idefix -i $ini -nolog || finish "!!!! $rep in single precision failed running with $ini" - - cd python - echo "***********************************************" - echo "Testing $rep in single precision and $ini" - echo "***********************************************" - python3 testidefix.py -noplot || finish "!!!! $rep in single precision failed validation with $ini" - cd .. - done - - echo "***********************************************" - echo "Cleaning $rep in $TMP_DIR" - echo "***********************************************" - rm -rf *.vtk *.dbl *.dmp *.ini python CMakeLists.txt -done - -#do it with MPI (only the default .ini files though) -# High order tests -for rep in $rep_MPI_list; do - cp -R $TEST_DIR/$rep/* $TMP_DIR - cd $TMP_DIR - echo "***********************************************" - echo "Configuring $rep" - echo "Using $TMP_DIR as working directory" - echo "***********************************************" - cmake $IDEFIX_DIR $options -DIdefix_MPI=ON -DIdefix_PRECISION=Single || finish "!!!!$rep in single precision failed during configuration" - echo "***********************************************" - echo "Making $rep in single precision" - echo "***********************************************" - make -j 8 || finish "!!!! $rep in single precision and MPI failed during compilation with" - - ini_files=$(ls *.ini) - for ini in $ini_files; do - echo "***********************************************" - echo "Running $rep in single precision, $ini and MPI" - echo "***********************************************" - mpirun -np 4 ./idefix -i $ini -nolog || finish "!!!! $rep in single precision and MPI failed running " - - cd python - echo "***********************************************" - echo "Testing $rep in single precision and MPI" - echo "***********************************************" - python3 testidefix.py -noplot || finish "!!!! $rep in single precision and MPI failed validation" - cd .. - done - echo "***********************************************" - echo "Cleaning $rep in $TMP_DIR" - echo "***********************************************" - rm -rf *.vtk *.dbl *.dmp *.ini python CMakeLists.txt -done -echo "Test was successfull" -cd $TEST_DIR -rm -rf $TMP_DIR diff --git a/test/checks_utils.sh b/test/checks_utils.sh deleted file mode 100755 index 30e55166..00000000 --- a/test/checks_utils.sh +++ /dev/null @@ -1,85 +0,0 @@ -#!/bin/bash - -# refer to the parent dir of this file, wherever this is called from -# a python equivalent is e.g. -# -# import pathlib -# TEST_DIR = pathlib.Path(__file__).parent -TEST_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" - -function resolve_path { - # resolve relative paths - # work around the fact that `realpath` is not bundled with every UNIX distro - echo "`cd "$1";pwd`" -} - -target_dir=$(resolve_path $TEST_DIR/..) - -if [ -z ${var+IDEFIX_DIR} ] & [ -d "$IDEFIX_DIR" ] ; then - global_dir=$(resolve_path $IDEFIX_DIR) - if [ $target_dir != $global_dir ] ; then - echo \ - "Warning: IDEFIX_DIR is set globally to $global_dir" \ - ", but is redefined to $target_dir" - fi -fi - -export IDEFIX_DIR=$target_dir -echo $IDEFIX_DIR - -set -e -options=$@ - -# Validate LookupTable -rep="utils/lookupTable" - -TMP_DIR="$(mktemp -d)" -function finish () -{ - echo $1 - echo "Cleaning directory $TMP_DIR" - cd $TEST_DIR - rm -rf $TMP_DIR - exit 1 -} - -cp -R $TEST_DIR/$rep/* $TMP_DIR -cd $TMP_DIR -echo "***********************************************" -echo "Making numpy test files" -echo "***********************************************" -python3 makeNpy.py || finish "!!!! can't make numpy test file" -echo "***********************************************" -echo "Configuring $rep" -echo "Using $TMP_DIR/$rep as working directory" -echo "***********************************************" -cmake $IDEFIX_DIR -DIdefix_MPI=ON $options || finish "!!!! Example $rep failed during configuration" -echo "***********************************************" -echo "Making $rep" -echo "***********************************************" -make -j 8 || finish "!!!! Test $rep failed during compilation" -mpirun -np 4 ./idefix || finish "!!!! Test $rep failed during runtime" -rm -rf *.vtk *.dbl *.dmp *.ini python - -cd $TEST_DIR - - -# Validate DumpImage -rep="utils/dumpImage" - -cp -R $TEST_DIR/$rep/* $TMP_DIR -cd $TMP_DIR -echo "***********************************************" -echo "Configuring $rep" -echo "Using $TMP_DIR/$rep as working directory" -echo "***********************************************" -cmake $IDEFIX_DIR $options || finish "!!!! Example $rep failed during configuration" -echo "***********************************************" -echo "Making $rep" -echo "***********************************************" -make -j 8 || finish "!!!! Test $rep failed during compilation" -./idefix || finish "!!!! Test $rep failed during runtime" - -echo "Test was successfull" -cd $TEST_DIR -rm -rf $TMP_DIR diff --git a/test/checks_vector_potential.sh b/test/checks_vector_potential.sh deleted file mode 100755 index 7fad07f1..00000000 --- a/test/checks_vector_potential.sh +++ /dev/null @@ -1,82 +0,0 @@ -#!/bin/bash -rep_MHD_list="AmbipolarCshock3D FargoMHDSpherical ShearingBox OrszagTang OrszagTang3D" - -# refer to the parent dir of this file, wherever this is called from -# a python equivalent is e.g. -# -# import pathlib -# TEST_DIR = pathlib.Path(__file__).parent -TEST_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -TMP_DIR="$(mktemp -d)" - -function resolve_path { - # resolve relative paths - # work around the fact that `realpath` is not bundled with every UNIX distro - echo "`cd "$1";pwd`" -} - -target_dir=$(resolve_path $TEST_DIR/..) - -if [ -z ${var+IDEFIX_DIR} ] & [ -d "$IDEFIX_DIR" ] ; then - global_dir=$(resolve_path $IDEFIX_DIR) - if [ $target_dir != $global_dir ] ; then - echo \ - "Warning: IDEFIX_DIR is set globally to $global_dir" \ - ", but is redefined to $target_dir" - fi -fi - -export IDEFIX_DIR=$target_dir -echo $IDEFIX_DIR - -set -e -options=$@ - -function finish () -{ - echo $1 - echo "Cleaning directory $TMP_DIR" - cd $TEST_DIR - rm -rf $TMP_DIR - exit 1 -} - -# MHD+Vector potential tests -for rep in $rep_MHD_list; do - cp -R $TEST_DIR/MHD/$rep/* $TMP_DIR - cd $TMP_DIR - echo "***********************************************" - echo "Configuring $rep" - echo "Using $TMP_DIR/$rep as working directory" - echo "***********************************************" - - def_files=$(ls definitions*.hpp) - for def in $def_files; do - cmake $IDEFIX_DIR $options -DIdefix_EVOLVE_VECTOR_POTENTIAL=ON -DIdefix_DEFS=$def|| finish "!!!! MHD $rep failed during configuration" - echo "***********************************************" - echo "Making $rep with $def" - echo "***********************************************" - make -j 8 || finish "!!!! MHD $rep failed during compilation with $def" - - ini_files=$(ls *.ini) - for ini in $ini_files; do - echo "***********************************************" - echo "Running $rep with $ini" - echo "***********************************************" - ./idefix -i $ini -nolog || finish "!!!! MHD $rep failed running with $def and $ini" - - cd python - echo "***********************************************" - echo "Testing $rep with $ini and $def" - echo "***********************************************" - python3 testidefix.py -noplot || finish "!!!! MHD $rep failed validation with $def and $ini" - cd .. - done - done - rm -rf *.vtk *.dbl *.dmp *.ini python CMakeLists.txt - cd $TEST_DIR -done - -echo "Test was successfull" -cd $TEST_DIR -rm -rf $TMP_DIR diff --git a/test/python_requirements.txt b/test/python_requirements.txt index 3fc1098a..77ce6e9c 100644 --- a/test/python_requirements.txt +++ b/test/python_requirements.txt @@ -5,3 +5,6 @@ numpy>=1.16.6 matplotlib>=2.2.5 scipy>=1.2.3 + +# note that no version of inifix supports Python older than 3.6 +inifix>=0.11.2 diff --git a/test/utils/dumpImage/setup.cpp b/test/utils/dumpImage/setup.cpp index 371e02dd..8f38ce8f 100644 --- a/test/utils/dumpImage/setup.cpp +++ b/test/utils/dumpImage/setup.cpp @@ -26,7 +26,7 @@ void Analysis(DataBlock& data) { // Create a dumpImage from the created dump char filename[20]; std::snprintf(filename, 20, "dump.%04d.dmp", outnum); - DumpImage image(filename,*myOutput); + DumpImage image(filename,&data); // Check that whetever is in the image matches the current state DataBlockHost d(data); @@ -40,23 +40,31 @@ void Analysis(DataBlock& data) { idfx::cout << "Analysis: checking dumpImage consistency with current state..." << std::flush; // Check that the save/load routines have left everything unchanged. char fieldName[20]; + for(auto const& [name, arr] : image.arrays) { + idfx::cout << "Array: " << name; + idfx::cout << " ; size: " << arr.extent(0) << " " << arr.extent(1) << " " << arr.extent(2) << std::endl; + + } for(int n = 0; n < NVAR ; n++) { - std::snprintf(fieldName,20,"Vc-%s",data.hydro.VcName[n].c_str()); - for(int k = d.beg[KDIR]; k < d.end[KDIR] ; k++) { - for(int j = d.beg[JDIR]; j < d.end[JDIR] ; j++) { - for(int i = d.beg[IDIR]; i < d.end[IDIR] ; i++) { - if(image.arrays[fieldName](k-d.beg[KDIR], j-d.beg[JDIR], i-d.beg[IDIR]) != d.Vc(n,k,j,i)) { - errornum++; - idfx::cout << "-----------------------------------------" << std::endl - << " Error in Vc at (i,j,k,n) = ( " << i << ", " << j << ", " << k << ", " << n << ")" << std::endl - << " Coordinates (x1,x2,x3) = ( " << d.x[IDIR](i) << ", " << d.x[JDIR](j) << ", " << d.x[KDIR](k) << ")" << std::endl; + std::snprintf(fieldName,20,"Vc-%s",data.hydro->VcName[n].c_str()); + if(auto it = image.arrays.find(std::string(fieldName)) ; it != image.arrays.end()) { + idfx::cout << "doing " << std::string(fieldName) << std::endl; + IdefixHostArray3D arr = it->second; + for(int k = d.beg[KDIR]; k < d.end[KDIR] ; k++) { + for(int j = d.beg[JDIR]; j < d.end[JDIR] ; j++) { + for(int i = d.beg[IDIR]; i < d.end[IDIR] ; i++) { + if(arr(k-d.beg[KDIR], j-d.beg[JDIR], i-d.beg[IDIR]) != d.Vc(n,k,j,i)) { + errornum++; + idfx::cout << "-----------------------------------------" << std::endl + << " Error in Vc at (i,j,k,n) = ( " << i << ", " << j << ", " << k << ", " << n << ")" << std::endl + << " Coordinates (x1,x2,x3) = ( " << d.x[IDIR](i) << ", " << d.x[JDIR](j) << ", " << d.x[KDIR](k) << ")" << std::endl; + } } - } } } } - std::snprintf(fieldName,20,"Vs-%s",data.hydro.VsName[0].c_str()); + std::snprintf(fieldName,20,"Vs-%s",data.hydro->VsName[0].c_str()); for(int k = d.beg[KDIR]; k < d.end[KDIR] ; k++) { for(int j = d.beg[JDIR]; j < d.end[JDIR] ; j++) { for(int i = d.beg[IDIR]; i < d.end[IDIR]+IOFFSET ; i++) { @@ -70,7 +78,7 @@ void Analysis(DataBlock& data) { } } } - std::snprintf(fieldName,20,"Vs-%s",data.hydro.VsName[1].c_str()); + std::snprintf(fieldName,20,"Vs-%s",data.hydro->VsName[1].c_str()); for(int k = d.beg[KDIR]; k < d.end[KDIR] ; k++) { for(int j = d.beg[JDIR]; j < d.end[JDIR]+JOFFSET ; j++) { for(int i = d.beg[IDIR]; i < d.end[IDIR] ; i++) { @@ -84,7 +92,7 @@ void Analysis(DataBlock& data) { } } } - std::snprintf(fieldName,20,"Vs-%s",data.hydro.VsName[2].c_str()); + std::snprintf(fieldName,20,"Vs-%s",data.hydro->VsName[2].c_str()); for(int k = d.beg[KDIR]; k < d.end[KDIR]+KOFFSET ; k++) { for(int j = d.beg[JDIR]; j < d.end[JDIR] ; j++) { for(int i = d.beg[IDIR]; i < d.end[IDIR] ; i++) { diff --git a/test/utils/dumpImage/testme.py b/test/utils/dumpImage/testme.py new file mode 100755 index 00000000..d2db9e86 --- /dev/null +++ b/test/utils/dumpImage/testme.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python3 + +""" + +@author: glesur +""" +import os +import sys +sys.path.append(os.getenv("IDEFIX_DIR")) +import pytools.idfx_test as tst + +test=tst.idfxTest() + +test.configure() +test.compile() +# this test succeeds if it runs successfully +test.run() diff --git a/test/utils/lookupTable/makeNpy.py b/test/utils/lookupTable/makeNpy.py deleted file mode 100755 index b4e402ee..00000000 --- a/test/utils/lookupTable/makeNpy.py +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -""" -Created on Thu Mar 5 11:29:41 2020 - -@author: glesur -""" - - -import numpy as np -from scipy.interpolate import RegularGridInterpolator - -x=np.arange(1,10,1.0) -y=np.arange(5,10,1.0) -z=np.arange(2,5,1.0) - -xp, yp, zp = np.meshgrid(x,y,z,indexing='ij') - -data=xp+2*yp-zp - -np.save("x.npy",x) -np.save("y.npy",y) -np.save("z.npy",z) -np.save("data.npy",data) -f=RegularGridInterpolator((x, y, z), data) -print(f([2.7,7.4,3.9])) diff --git a/test/utils/lookupTable/testme.py b/test/utils/lookupTable/testme.py new file mode 100755 index 00000000..d4088465 --- /dev/null +++ b/test/utils/lookupTable/testme.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python3 + +""" + +@author: glesur +""" +import os +import sys +sys.path.append(os.getenv("IDEFIX_DIR")) +import numpy as np +#from scipy.interpolate import RegularGridInterpolator +import pytools.idfx_test as tst + +def MakeNumpyFile(): + x=np.arange(1,10,1.0) + y=np.arange(5,10,1.0) + z=np.arange(2,5,1.0) + + xp, yp, zp = np.meshgrid(x,y,z,indexing='ij') + + data=xp+2*yp-zp + + np.save("x.npy",x) + np.save("y.npy",y) + np.save("z.npy",z) + np.save("data.npy",data) + # show the expected result + #f=RegularGridInterpolator((x, y, z), data) + #print(f([2.7,7.4,3.9])) + + + +test=tst.idfxTest() +MakeNumpyFile() + +test.configure() +test.compile() +# this test succeeds if it runs successfully +test.run()