From fafbb9bf42e5e63250c249cb8c69d0a061219849 Mon Sep 17 00:00:00 2001 From: Luke Shingles Date: Sun, 8 Sep 2024 21:09:27 +0100 Subject: [PATCH] Switch uptrans and downtrans from pointers to indices --- atomic.h | 8 +++--- globals.h | 8 +++--- input.cc | 81 ++++++++++++++++++++++++++++++++----------------------- 3 files changed, 55 insertions(+), 42 deletions(-) diff --git a/atomic.h b/atomic.h index 4ccd8c950..25979c3ad 100644 --- a/atomic.h +++ b/atomic.h @@ -360,8 +360,8 @@ inline auto get_includedlevels() -> int { return includedlevels; } return globals::elements[element].ions[ion].levels[level].ndowntrans; } -[[nodiscard]] inline auto get_downtranslist(const int element, const int ion, const int level) { - return globals::elements[element].ions[ion].levels[level].downtrans; +[[nodiscard]] inline auto get_downtranslist(const int element, const int ion, const int level) -> LevelTransition * { + return globals::alltrans + globals::elements[element].ions[ion].levels[level].downtrans; } // the number of upward bound-bound transitions from the specified level @@ -372,8 +372,8 @@ inline auto get_includedlevels() -> int { return includedlevels; } return globals::elements[element].ions[ion].levels[level].nuptrans; } -[[nodiscard]] inline auto get_uptranslist(const int element, const int ion, const int level) { - return globals::elements[element].ions[ion].levels[level].uptrans; +[[nodiscard]] inline auto get_uptranslist(const int element, const int ion, const int level) -> LevelTransition * { + return globals::alltrans + globals::elements[element].ions[ion].levels[level].uptrans; } [[nodiscard]] inline auto get_uptransspan(const int element, const int ion, const int level) { diff --git a/globals.h b/globals.h index db4911180..9a20ff73e 100644 --- a/globals.h +++ b/globals.h @@ -81,11 +81,11 @@ struct LevelTransition { }; struct EnergyLevel { - double epsilon{-1}; // Excitation energy of this level relative to the neutral ground level. - LevelTransition *uptrans{}; // Allowed upward transitions from this level - LevelTransition *downtrans{}; // Allowed downward transitions from this level + double epsilon{-1}; // Excitation energy of this level relative to the neutral ground level. int nuptrans{0}; + int uptrans{}; // Allowed upward transitions from this level int ndowntrans{0}; + int downtrans{}; // Allowed downward transitions from this level PhotoionTarget *phixstargets{}; // pointer to table of target states and probabilities int phixsstart{-1}; // index to start of photoionisation cross-sections table in global::allphixs int nphixstargets{0}; // length of phixstargets array: @@ -109,7 +109,6 @@ struct Ion { int uniquelevelindexstart; int groundcontindex; double ionpot; // Ionisation threshold to the next ionstage - LevelTransition *alltransitions; }; struct Element { @@ -258,6 +257,7 @@ inline int opacity_case{}; // 0 grey, 1 for Fe-grp dependence. inline std::vector ion_alpha_sp; // alpha_sp for each ion and temperature table value inline float *allphixs{}; +inline LevelTransition *alltrans; inline std::vector elements; diff --git a/input.cc b/input.cc index f934bd6bd..1dce260ad 100644 --- a/input.cc +++ b/input.cc @@ -477,7 +477,8 @@ void read_ion_transitions(std::fstream &ftransitiondata, const int tottransition void add_transitions_to_unsorted_linelist(const int element, const int ion, const int nlevelsmax, const std::vector &transitiontable, std::vector &iondowntranstmplineindicies, int &lineindex, - std::vector &temp_linelist) { + std::vector &temp_linelist, + std::vector &temp_alltranslist) { const int lineindex_initial = lineindex; ptrdiff_t totupdowntrans = 0; // pass 0 to get transition counts of each level @@ -485,29 +486,13 @@ void add_transitions_to_unsorted_linelist(const int element, const int ion, cons for (int pass = 0; pass < 2; pass++) { lineindex = lineindex_initial; if (pass == 1) { - int alltransindex = 0; - auto &ionalltrans = globals::elements[element].ions[ion].alltransitions; - -#ifdef MPI_ON - MPI_Barrier(MPI_COMM_WORLD); - MPI_Win win_alltransblock = MPI_WIN_NULL; - - const auto [_, noderank_trans] = get_range_chunk(totupdowntrans, globals::node_nprocs, globals::rank_in_node); - - auto size = static_cast(noderank_trans * sizeof(LevelTransition)); - int disp_unit = sizeof(LevelTransition); - MPI_Win_allocate_shared(size, disp_unit, MPI_INFO_NULL, globals::mpi_comm_node, &ionalltrans, &win_alltransblock); - - MPI_Win_shared_query(win_alltransblock, 0, &size, &disp_unit, &ionalltrans); -#else - ionalltrans = static_cast(malloc(totupdowntrans * sizeof(LevelTransition))); -#endif - + int alltransindex = temp_alltranslist.size(); + temp_alltranslist.resize(temp_alltranslist.size() + totupdowntrans); for (int level = 0; level < nlevelsmax; level++) { - globals::elements[element].ions[ion].levels[level].downtrans = &ionalltrans[alltransindex]; + globals::elements[element].ions[ion].levels[level].downtrans = alltransindex; alltransindex += get_ndowntrans(element, ion, level); - globals::elements[element].ions[ion].levels[level].uptrans = &ionalltrans[alltransindex]; + globals::elements[element].ions[ion].levels[level].uptrans = alltransindex; alltransindex += get_nuptrans(element, ion, level); set_ndowntrans(element, ion, level, 0); @@ -567,18 +552,20 @@ void add_transitions_to_unsorted_linelist(const int element, const int ion, cons // the line list has not been sorted yet, so the store the level index for now and // the index into the sorted line list will be set later - get_downtranslist(element, ion, level)[nupperdowntrans - 1] = {.lineindex = -1, - .targetlevelindex = lowerlevel, - .einstein_A = transition.A, - .coll_str = transition.coll_str, - .osc_strength = f_ul, - .forbidden = transition.forbidden}; - get_uptranslist(element, ion, lowerlevel)[nloweruptrans - 1] = {.lineindex = -1, - .targetlevelindex = level, - .einstein_A = transition.A, - .coll_str = transition.coll_str, - .osc_strength = f_ul, - .forbidden = transition.forbidden}; + temp_alltranslist[globals::elements[element].ions[ion].levels[level].downtrans + nupperdowntrans - 1] = { + .lineindex = -1, + .targetlevelindex = lowerlevel, + .einstein_A = transition.A, + .coll_str = transition.coll_str, + .osc_strength = f_ul, + .forbidden = transition.forbidden}; + temp_alltranslist[globals::elements[element].ions[ion].levels[lowerlevel].uptrans + nloweruptrans - 1] = { + .lineindex = -1, + .targetlevelindex = level, + .einstein_A = transition.A, + .coll_str = transition.coll_str, + .osc_strength = f_ul, + .forbidden = transition.forbidden}; } } else if (pass == 1 && globals::rank_in_node == 0) { @@ -970,6 +957,7 @@ void read_atomicdata_files() { globals::elements.resize(nelements_in); std::vector temp_linelist; + std::vector temp_alltranslist; std::vector iontransitiontable; std::vector iondowntranstmplineindicies; @@ -1143,7 +1131,7 @@ void read_atomicdata_files() { iondowntranstmplineindicies.resize(downtranslevelstart(nlevelsmax)); add_transitions_to_unsorted_linelist(element, ion, nlevelsmax, iontransitiontable, iondowntranstmplineindicies, - lineindex, temp_linelist); + lineindex, temp_linelist, temp_alltranslist); for (int level = 0; level < nlevelsmax; level++) { uniquelevelindex++; @@ -1204,6 +1192,31 @@ void read_atomicdata_files() { } } + { + // create a shared all transitions list and then copy data across, freeing the local copy + const auto totupdowntrans = totaluptrans + totaldowntrans; + assert_always(totupdowntrans == static_cast(temp_alltranslist.size())); +#ifdef MPI_ON + MPI_Barrier(MPI_COMM_WORLD); + MPI_Win win_alltransblock = MPI_WIN_NULL; + + const auto [_, noderank_trans] = get_range_chunk(totupdowntrans, globals::node_nprocs, globals::rank_in_node); + + auto size = static_cast(noderank_trans * sizeof(LevelTransition)); + int disp_unit = sizeof(LevelTransition); + MPI_Win_allocate_shared(size, disp_unit, MPI_INFO_NULL, globals::mpi_comm_node, &globals::alltrans, + &win_alltransblock); + + MPI_Win_shared_query(win_alltransblock, 0, &size, &disp_unit, &globals::alltrans); +#else + globals::alltrans = static_cast(malloc(totupdowntrans * sizeof(LevelTransition))); +#endif + if (globals::rank_in_node == 0) { + std::copy_n(temp_alltranslist.data(), totupdowntrans, globals::alltrans); + temp_alltranslist.clear(); + } + } + // create a linelist shared on node and then copy data across, freeing the local copy TransitionLine *nonconstlinelist{}; {