From 1e0f2c7557ded9e0b98429d51169b9875ac939d0 Mon Sep 17 00:00:00 2001 From: Joan Vinyals Date: Fri, 16 Jul 2021 21:53:05 +0200 Subject: [PATCH 1/4] Stedy! --- Applications/Applications/siesta/README.md | 328 ++++++++++++++++-- .../Applications/siesta/job.offnode.c5n.py | 90 +++++ .../siesta/job.offnode.c6gn.c6n.py | 90 +++++ .../Applications/siesta/job.offnode.py | 90 +++++ .../Applications/siesta/job.onnode.py | 94 +++++ Applications/Applications/siesta/package.py | 115 ++++++ 6 files changed, 786 insertions(+), 21 deletions(-) create mode 100644 Applications/Applications/siesta/job.offnode.c5n.py create mode 100644 Applications/Applications/siesta/job.offnode.c6gn.c6n.py create mode 100644 Applications/Applications/siesta/job.offnode.py create mode 100644 Applications/Applications/siesta/job.onnode.py create mode 100644 Applications/Applications/siesta/package.py diff --git a/Applications/Applications/siesta/README.md b/Applications/Applications/siesta/README.md index 5820ce83..d32ed456 100644 --- a/Applications/Applications/siesta/README.md +++ b/Applications/Applications/siesta/README.md @@ -4,7 +4,7 @@ **URL:** https://departments.icmab.es/leem/siesta/ -**Team:** +**Team:** Garotes de Premià ## Compilation @@ -20,14 +20,227 @@ Pull request for Spack recipe changes: -#### Compiler 1 +#### GNU Fortran Compiler (GNU) ``` -spack install % +spack install siesta@4.0.1%gcc@10.3.0+metis ``` ``` -$ spack spec -Il % +$ spack spec -Il siesta@4.0.1%gcc@10.3.0+metis +Input spec + - siesta@4.0.1%gcc@10.3.0+metis + +Concretized +-------------------------------- +[+] e374e4q siesta@4.0.1%gcc@10.3.0+metis patches=b8f722add750b1524767062c7a86de63a1da7990c27fda5321e43e34179e50fc arch=linux-amzn2-graviton2 +[+] a3tjh3r ^metis@5.1.0%gcc@10.3.0~gdb~int64~real64+shared build_type=Release patches=4991da938c1d3a1d3dea78e49bbebecba00273f98df2a656e38b83d55b281da1,b1225da886605ea558db7ac08dd8054742ea5afe5ed61ad4d0fe7a495b1270d2 arch=linux-amzn2-graviton2 +[+] m7325ee ^cmake@3.20.5%gcc@10.3.0~doc+ncurses+openssl+ownlibs~qt build_type=Release arch=linux-amzn2-graviton2 +[+] iwzirqc ^ncurses@6.2%gcc@10.3.0~symlinks+termlib abi=none arch=linux-amzn2-graviton2 +[+] s4pw7zm ^pkgconf@1.7.4%gcc@10.3.0 arch=linux-amzn2-graviton2 +[+] 5i3lgfb ^openssl@1.1.1k%gcc@10.3.0~docs+systemcerts arch=linux-amzn2-graviton2 +[+] 4m7exgb ^perl@5.32.1%gcc@10.3.0+cpanm+shared+threads arch=linux-amzn2-graviton2 +[+] y42m6yr ^berkeley-db@18.1.40%gcc@10.3.0+cxx~docs+stl patches=b231fcc4d5cff05e5c3a4814f6a5af0e9a966428dc2176540d2c05aff41de522 arch=linux-amzn2-graviton2 +[+] rqrpmap ^bzip2@1.0.8%gcc@10.3.0~debug~pic+shared arch=linux-amzn2-graviton2 +[+] 2w7bert ^diffutils@3.7%gcc@10.3.0 arch=linux-amzn2-graviton2 +[+] y5ei3cm ^libiconv@1.16%gcc@10.3.0 arch=linux-amzn2-graviton2 +[+] wjwqncx ^gdbm@1.19%gcc@10.3.0 arch=linux-amzn2-graviton2 +[+] 3zy7kxk ^readline@8.1%gcc@10.3.0 arch=linux-amzn2-graviton2 +[+] qepjcvj ^zlib@1.2.11%gcc@10.3.0+optimize+pic+shared arch=linux-amzn2-graviton2 +[+] tfzbzgi ^netcdf-c@4.8.0%gcc@10.3.0~dap~fsync~hdf4~jna+mpi~parallel-netcdf+pic+shared arch=linux-amzn2-graviton2 +[+] cg7z7ep ^hdf5@1.10.7%gcc@10.3.0~cxx~fortran+hl~ipo~java+mpi+shared~szip~threadsafe+tools api=default build_type=RelWithDebInfo arch=linux-amzn2-graviton2 +[+] zvamksn ^openmpi@4.1.0%gcc@10.3.0~atomics~cuda~cxx~cxx_exceptions+gpfs~internal-hwloc~java~legacylaunchers~lustre~memchecker+pmi~singularity~sqlite3+static~thread_multiple+vt+wrapper-rpath fabrics=ofi patches=60ce20bc14d98c572ef7883b9fcd254c3f232c2f3a13377480f96466169ac4c8 schedulers=slurm arch=linux-amzn2-graviton2 +[+] cukmqbg ^hwloc@2.5.0%gcc@10.3.0~cairo~cuda~gl~libudev+libxml2~netloc~nvml+pci+shared arch=linux-amzn2-graviton2 +[+] asgtk6a ^libpciaccess@0.16%gcc@10.3.0 arch=linux-amzn2-graviton2 +[+] z2uysov ^libtool@2.4.6%gcc@10.3.0 arch=linux-amzn2-graviton2 +[+] ebhjpix ^m4@1.4.18%gcc@10.3.0+sigsegv patches=3877ab548f88597ab2327a2230ee048d2d07ace1062efe81fc92e91b7f39cd00,fc9b61654a3ba1a8d6cd78ce087e7c96366c290bc8d2c299f09828d793b853c8 arch=linux-amzn2-graviton2 +[+] ltbv6bk ^libsigsegv@2.13%gcc@10.3.0 arch=linux-amzn2-graviton2 +[+] 4xr3hhh ^util-macros@1.19.3%gcc@10.3.0 arch=linux-amzn2-graviton2 +[+] iyhm3wi ^libxml2@2.9.10%gcc@10.3.0~python arch=linux-amzn2-graviton2 +[+] ye3kcvv ^xz@5.2.5%gcc@10.3.0~pic libs=shared,static arch=linux-amzn2-graviton2 +[+] tadxrfp ^libevent@2.1.12%gcc@10.3.0+openssl arch=linux-amzn2-graviton2 +[+] 72f5gvk ^libfabric@1.11.1-aws%gcc@10.3.0~debug~kdreg fabrics=sockets,tcp,udp arch=linux-amzn2-graviton2 +[+] mhav5gn ^numactl@2.0.14%gcc@10.3.0 patches=4e1d78cbbb85de625bad28705e748856033eaafab92a66dffd383a3d7e00cc94,62fc8a8bf7665a60e8f4c93ebbd535647cebf74198f7afafec4c085a8825c006 arch=linux-amzn2-graviton2 +[+] jkuhz64 ^autoconf@2.69%gcc@10.3.0 arch=linux-amzn2-graviton2 +[+] xb2w5nc ^automake@1.16.3%gcc@10.3.0 arch=linux-amzn2-graviton2 +[+] wturp6c ^openssh@8.5p1%gcc@10.3.0 arch=linux-amzn2-graviton2 +[+] ivotdt7 ^libedit@3.1-20210216%gcc@10.3.0 arch=linux-amzn2-graviton2 +[+] wqpuvmh ^slurm@20-02-4-1%gcc@10.3.0~gtk~hdf5~hwloc~mariadb~pmix+readline~restd sysconfdir=PREFIX/etc arch=linux-amzn2-graviton2 +[+] 6ku6khi ^netcdf-fortran@4.5.3%gcc@10.3.0~doc+pic+shared arch=linux-amzn2-graviton2 +[+] 2jffbna ^netlib-scalapack@2.1.0%gcc@10.3.0~ipo~pic+shared build_type=Release patches=1c9ce5fee1451a08c2de3cc87f446aeda0b818ebbce4ad0d980ddf2f2a0b2dc4,f2baedde688ffe4c20943c334f580eb298e04d6f35c86b90a1f4e8cb7ae344a2 arch=linux-amzn2-graviton2 +[+] rv7gj6u ^openblas@0.3.15%gcc@10.3.0~bignuma~consistent_fpcsr~ilp64+locking+pic+shared threads=none arch=linux-amzn2-graviton2 +``` + +#### ARM and Nvidia HPC Compilers + +Unfortunetly we could not compile siesta with neither the ARM compiler nor the Nvidia one. + +On both compilers the compilation ends with the same error, after solving all the other errors +we encountered. The error is due to multiple definitions of `mpi_comm_world`, and we were not +able to find a solution. The error: + +``` +==> siesta: Executing phase: 'build' +==> Error: ProcessError: Command exited with status 2: + 'make' + +2 errors found in build log: + 355 NVFORTRAN-S-0155-mpi_comm_world is use-associated from modules mpi_siesta and mpi__include, and cannot be accessed (/tmp/jvinyals/spack-stage/spack-stage-siesta-4.0.1-emukb7kkp46sefjkkzjt5nafsfai54l7/spack-src/Src/fdf/fdf + .F90: 2829) + 356 NVFORTRAN-S-0155-mpi_comm_world is use-associated from modules mpi_siesta and mpi__include, and cannot be accessed (/tmp/jvinyals/spack-stage/spack-stage-siesta-4.0.1-emukb7kkp46sefjkkzjt5nafsfai54l7/spack-src/Src/fdf/fdf + .F90: 2837) + 357 0 inform, 0 warnings, 2 severes, 0 fatal for fdf_sendinput + 358 NVFORTRAN-S-0155-mpi_comm_world is use-associated from modules mpi_siesta and mpi__include, and cannot be accessed (/tmp/jvinyals/spack-stage/spack-stage-siesta-4.0.1-emukb7kkp46sefjkkzjt5nafsfai54l7/spack-src/Src/fdf/fdf + .F90: 2868) + 359 NVFORTRAN-S-0155-mpi_comm_world is use-associated from modules mpi_siesta and mpi__include, and cannot be accessed (/tmp/jvinyals/spack-stage/spack-stage-siesta-4.0.1-emukb7kkp46sefjkkzjt5nafsfai54l7/spack-src/Src/fdf/fdf + .F90: 2881) + 360 0 inform, 0 warnings, 2 severes, 0 fatal for fdf_recvinput + >> 361 make[1]: *** [fdf.o] Error 2 + 362 make[1]: Leaving directory `/tmp/jvinyals/spack-stage/spack-stage-siesta-4.0.1-emukb7kkp46sefjkkzjt5nafsfai54l7/spack-src/Obj/fdf' + >> 363 make: *** [libfdf.a] Error 2 +``` + +##### ARM + +Our first try was using the `spack install` command as below, only specifying the compiler. This did not work. + +``` +spack install siesta%arm +``` + +The first error we found in whilst compiling with the ARM compiler was a module overload by the siesta code. +The code overloaded the `iso_fortran_env`. To solve these we added in the configure phase two commands to rename +the module to `siesta_fortran_env`. + +``` +def configure(self, spec, prefix): + + ... + + sh("-c", "find -type f -exec sed -i 's/iso_fortran_env/siesta_fortran_env/g' {} \\;") + sh("-c", "find -iname iso_fortran_env.F90 -exec rename iso siesta {} \\; + + ... + +``` + +This solved the problem leading us to the final problem with this compiler (mentioned avove). Leaving +the final spec of our attempt to install it as shown below. + +``` +$ spack spec -Il siesta%arm + +Input spec +-------------------------------- + - siesta%arm + +Concretized +-------------------------------- +==> Warning: arm@21.0.0.879 cannot build optimized binaries for "graviton2". Using best target possible: "aarch64" + - 564l6xo siesta@4.0.1%arm@21.0.0.879~metis patches=b8f722add750b1524767062c7a86de63a1da7990c27fda5321e43e34179e50fc arch=linux-amzn2-aarch64 +[+] 32qnomy ^netcdf-c@4.8.0%arm@21.0.0.879~dap~fsync~hdf4~jna+mpi~parallel-netcdf+pic+shared arch=linux-amzn2-aarch64 +[+] e4dajf6 ^hdf5@1.10.7%arm@21.0.0.879~cxx~fortran+hl~ipo~java+mpi+shared~szip~threadsafe+tools api=default build_type=RelWithDebInfo arch=linux-amzn2-aarch64 +[+] fqvybaf ^cmake@3.20.5%arm@21.0.0.879~doc+ncurses+openssl+ownlibs~qt build_type=Release arch=linux-amzn2-aarch64 +[+] uhtqtlb ^ncurses@6.2%arm@21.0.0.879~symlinks+termlib abi=none arch=linux-amzn2-aarch64 +[+] zpuzm23 ^pkgconf@1.7.4%arm@21.0.0.879 arch=linux-amzn2-aarch64 +[+] vc3waha ^openssl@1.1.1k%arm@21.0.0.879~docs+systemcerts arch=linux-amzn2-aarch64 +[+] vv6txro ^perl@5.32.1%arm@21.0.0.879+cpanm+shared+threads arch=linux-amzn2-aarch64 +[+] 33wiajj ^berkeley-db@18.1.40%arm@21.0.0.879+cxx~docs+stl patches=b231fcc4d5cff05e5c3a4814f6a5af0e9a966428dc2176540d2c05aff41de522 arch=linux-amzn2-aarch64 +[+] z4ybgri ^bzip2@1.0.8%arm@21.0.0.879~debug~pic+shared arch=linux-amzn2-aarch64 +[+] adtc6yc ^diffutils@3.7%arm@21.0.0.879 arch=linux-amzn2-aarch64 +[+] 7vnthzn ^libiconv@1.16%arm@21.0.0.879 arch=linux-amzn2-aarch64 +[+] 645q4qj ^gdbm@1.19%arm@21.0.0.879 arch=linux-amzn2-aarch64 +[+] 3haw5gt ^readline@8.1%arm@21.0.0.879 arch=linux-amzn2-aarch64 +[+] puuxvg2 ^zlib@1.2.11%arm@21.0.0.879+optimize+pic+shared arch=linux-amzn2-aarch64 +[+] lmaoy5t ^openmpi@4.1.0%arm@21.0.0.879~atomics~cuda~cxx~cxx_exceptions+gpfs~internal-hwloc~java~legacylaunchers~lustre~memchecker+pmi~singularity~sqlite3+static~thread_multiple+vt+wrapper-rpath fabrics=ofi patches=60ce20bc14d98c572ef7883b9fcd254c3f232c2f3a13377480f96466169ac4c8 schedulers=slurm arch=linux-amzn2-aarch64 +[+] xl6anaa ^hwloc@2.5.0%arm@21.0.0.879~cairo~cuda~gl~libudev+libxml2~netloc~nvml+pci+shared arch=linux-amzn2-aarch64 +[+] jueqz7p ^libpciaccess@0.16%arm@21.0.0.879 arch=linux-amzn2-aarch64 +[+] e4ssqx6 ^libtool@2.4.6%arm@21.0.0.879 arch=linux-amzn2-aarch64 +[+] i2jmeo4 ^m4@1.4.18%arm@21.0.0.879+sigsegv patches=3877ab548f88597ab2327a2230ee048d2d07ace1062efe81fc92e91b7f39cd00,fc9b61654a3ba1a8d6cd78ce087e7c96366c290bc8d2c299f09828d793b853c8 arch=linux-amzn2-aarch64 +[+] 6jhzlul ^libsigsegv@2.13%arm@21.0.0.879 arch=linux-amzn2-aarch64 +[+] uwcxkin ^util-macros@1.19.3%arm@21.0.0.879 arch=linux-amzn2-aarch64 +[+] dypqz2i ^libxml2@2.9.10%arm@21.0.0.879~python arch=linux-amzn2-aarch64 +[+] zqsab4f ^xz@5.2.5%arm@21.0.0.879~pic libs=shared,static arch=linux-amzn2-aarch64 +[+] gonqskn ^libevent@2.1.12%arm@21.0.0.879+openssl arch=linux-amzn2-aarch64 +[+] qdn27nh ^libfabric@1.11.1-aws%arm@21.0.0.879~debug~kdreg fabrics=sockets,tcp,udp arch=linux-amzn2-aarch64 +[+] mv2g7r5 ^numactl@2.0.14%arm@21.0.0.879 patches=4e1d78cbbb85de625bad28705e748856033eaafab92a66dffd383a3d7e00cc94,62fc8a8bf7665a60e8f4c93ebbd535647cebf74198f7afafec4c085a8825c006 arch=linux-amzn2-aarch64 +[+] dcs645r ^autoconf@2.69%arm@21.0.0.879 arch=linux-amzn2-aarch64 +[+] edezkz3 ^automake@1.16.3%arm@21.0.0.879 arch=linux-amzn2-aarch64 +[+] 6vvthuo ^openssh@8.5p1%arm@21.0.0.879 arch=linux-amzn2-aarch64 +[+] xe4evc4 ^libedit@3.1-20210216%arm@21.0.0.879 arch=linux-amzn2-aarch64 +[+] x5xehti ^slurm@20-02-4-1%arm@21.0.0.879~gtk~hdf5~hwloc~mariadb~pmix+readline~restd sysconfdir=PREFIX/etc arch=linux-amzn2-aarch64 +[+] im5sbgn ^netcdf-fortran@4.5.3%arm@21.0.0.879~doc+pic+shared arch=linux-amzn2-aarch64 +[+] xc2r6zp ^netlib-scalapack@2.1.0%arm@21.0.0.879~ipo~pic+shared build_type=Release patches=1c9ce5fee1451a08c2de3cc87f446aeda0b818ebbce4ad0d980ddf2f2a0b2dc4,f2baedde688ffe4c20943c334f580eb298e04d6f35c86b90a1f4e8cb7ae344a2 arch=linux-amzn2-aarch64 +[+] cwuo4ek ^openblas@0.3.15%arm@21.0.0.879~bignuma~consistent_fpcsr~ilp64+locking+pic+shared threads=none arch=linux-amzn2-aarch64 +``` + +##### Nvidia HPC (NVHPC) + +Our first aproach was again to simpli specify the Nvidia HPC Compiler. The problem with this module was +a conflict with both `cmake` and `openblas` which weren compatibles with the compiler. + +``` +spack install siesta%nvhpc +``` + +Then we tried to compile these modules with the GNU Compiler, trying to get a binary, using the `spack` command +below. This solved the conflicts, and lead us to the final error. +Because we were not able to solve the `mpi_comm_world` problem, we did not try to patch the packages of the dependencies +to compile with this compiler. + +``` +spack install siesta%nvhpc ^cmake%gcc ^openblas%gcc +``` + +The final spec of our attempt to compile it was as below. + +``` +$ spack spec -Il siesta%nvhpc ^cmake%gcc ^openblas%gcc + +Input spec +-------------------------------- + - siesta%nvhpc + - ^cmake%gcc + - ^openblas%gcc + +Concretized +-------------------------------- + - emukb7k siesta@4.0.1%nvhpc@21.2~metis patches=b8f722add750b1524767062c7a86de63a1da7990c27fda5321e43e34179e50fc arch=linux-amzn2-graviton2 +[+] baltkx5 ^netcdf-c@4.8.0%nvhpc@21.2~dap~fsync~hdf4~jna+mpi~parallel-netcdf+pic+shared arch=linux-amzn2-graviton2 +[+] wwncepq ^hdf5@1.10.7%nvhpc@21.2~cxx~fortran+hl~ipo~java+mpi+shared~szip~threadsafe+tools api=default build_type=RelWithDebInfo arch=linux-amzn2-graviton2 +[+] it4etcv ^cmake@3.20.5%gcc@10.3.0~doc+ncurses+openssl+ownlibs~qt build_type=Release arch=linux-amzn2-graviton2 +[+] iwzirqc ^ncurses@6.2%gcc@10.3.0~symlinks+termlib abi=none arch=linux-amzn2-graviton2 +[+] s4pw7zm ^pkgconf@1.7.4%gcc@10.3.0 arch=linux-amzn2-graviton2 +[+] kssecxk ^openssl@1.1.1k%gcc@10.3.0~docs+systemcerts arch=linux-amzn2-graviton2 +[+] zyh3ju5 ^perl@5.32.1%gcc@10.3.0+cpanm+shared+threads arch=linux-amzn2-graviton2 +[+] y42m6yr ^berkeley-db@18.1.40%gcc@10.3.0+cxx~docs+stl patches=b231fcc4d5cff05e5c3a4814f6a5af0e9a966428dc2176540d2c05aff41de522 arch=linux-amzn2-graviton2 +[+] rqrpmap ^bzip2@1.0.8%gcc@10.3.0~debug~pic+shared arch=linux-amzn2-graviton2 +[+] 2w7bert ^diffutils@3.7%gcc@10.3.0 arch=linux-amzn2-graviton2 +[+] y5ei3cm ^libiconv@1.16%gcc@10.3.0 arch=linux-amzn2-graviton2 +[+] wjwqncx ^gdbm@1.19%gcc@10.3.0 arch=linux-amzn2-graviton2 +[+] 3zy7kxk ^readline@8.1%gcc@10.3.0 arch=linux-amzn2-graviton2 +[+] 4js6ect ^zlib@1.2.11%nvhpc@21.2+optimize+pic+shared arch=linux-amzn2-graviton2 +[+] dc5i2vh ^openmpi@4.1.0%nvhpc@21.2~atomics~cuda~cxx~cxx_exceptions+gpfs~internal-hwloc~java~legacylaunchers~lustre~memchecker+pmi~singularity~sqlite3+static~thread_multiple+vt+wrapper-rpath fabrics=ofi patches=60ce20bc14d98c572ef7883b9fcd254c3f232c2f3a13377480f96466169ac4c8,fba0d3a784a9723338722b48024a22bb32f6a951db841a4e9f08930a93f41d7a schedulers=slurm arch=linux-amzn2-graviton2 +[+] euby7td ^hwloc@2.5.0%nvhpc@21.2~cairo~cuda~gl~libudev+libxml2~netloc~nvml+pci+shared arch=linux-amzn2-graviton2 +[+] e4m4ued ^libpciaccess@0.16%nvhpc@21.2 patches=6e08dc445ece06e9e8b1344397f2d3f169005703ddc0f2ae24f366cde78c7377 arch=linux-amzn2-graviton2 +[+] kk4ax3i ^libtool@2.4.6%nvhpc@21.2 arch=linux-amzn2-graviton2 +[+] 6c4kz5g ^m4@1.4.18%nvhpc@21.2+sigsegv patches=3877ab548f88597ab2327a2230ee048d2d07ace1062efe81fc92e91b7f39cd00,5746cf51f45b405661c3edae7a78c33d41e54d83f635d16e2bf1f956dbfbf635,fc9b61654a3ba1a8d6cd78ce087e7c96366c290bc8d2c299f09828d793b853c8 arch=linux-amzn2-graviton2 +[+] pa6wm5j ^libsigsegv@2.13%nvhpc@21.2 arch=linux-amzn2-graviton2 +[+] 4imdwuy ^util-macros@1.19.3%nvhpc@21.2 arch=linux-amzn2-graviton2 +[+] jzxyspx ^libxml2@2.9.10%nvhpc@21.2~python patches=05ff238cf435825ef835c7ae39376b52dc83d8caf19e962f0766c841386a305a,10a88ad47f9797cf7cf2d7d07241f665a3b6d1f31fa026728c8c2ae93e1664e9 arch=linux-amzn2-graviton2 +[+] br733tn ^xz@5.2.5%nvhpc@21.2~pic libs=shared,static arch=linux-amzn2-graviton2 +[+] qs5m2pb ^libevent@2.1.12%nvhpc@21.2+openssl arch=linux-amzn2-graviton2 +[+] xl6zavq ^libfabric@1.11.1-aws%nvhpc@21.2~debug~kdreg fabrics=sockets,tcp,udp arch=linux-amzn2-graviton2 +[+] 5yq4tpw ^numactl@2.0.14%nvhpc@21.2 patches=4e1d78cbbb85de625bad28705e748856033eaafab92a66dffd383a3d7e00cc94,62fc8a8bf7665a60e8f4c93ebbd535647cebf74198f7afafec4c085a8825c006 arch=linux-amzn2-graviton2 +[+] dghtild ^autoconf@2.69%nvhpc@21.2 arch=linux-amzn2-graviton2 +[+] umo35bq ^automake@1.16.3%nvhpc@21.2 arch=linux-amzn2-graviton2 +[+] tydb3k5 ^openssh@8.5p1%gcc@10.3.0 arch=linux-amzn2-graviton2 +[+] ivotdt7 ^libedit@3.1-20210216%gcc@10.3.0 arch=linux-amzn2-graviton2 +[+] zehhooy ^slurm@20-02-4-1%nvhpc@21.2~gtk~hdf5~hwloc~mariadb~pmix+readline~restd sysconfdir=PREFIX/etc arch=linux-amzn2-graviton2 +[+] vekbvrj ^netcdf-fortran@4.5.3%nvhpc@21.2~doc+pic+shared arch=linux-amzn2-graviton2 +[+] 5vrjohv ^netlib-scalapack@2.1.0%nvhpc@21.2~ipo~pic+shared build_type=Release patches=1c9ce5fee1451a08c2de3cc87f446aeda0b818ebbce4ad0d980ddf2f2a0b2dc4,f2baedde688ffe4c20943c334f580eb298e04d6f35c86b90a1f4e8cb7ae344a2 arch=linux-amzn2-graviton2 +[+] rv7gj6u ^openblas@0.3.15%gcc@10.3.0~bignuma~consistent_fpcsr~ilp64+locking+pic+shared threads=none arch=linux-amzn2-graviton2 + ``` @@ -36,31 +249,106 @@ $ spack spec -Il % [ReFrame Benchmark 1](#) ``` -../bin/reframe -c benchmark.py -r --performance-report +$ reframe -c job.onnode.py -r --performance-report ``` ### Validation -Details of the validation for `Test Case 1`. +The only validation that we do is check that the execution finishes without error. We used this +method because we did not know about the science behind, and were not able to find validation data. ### ReFrame Output +#### On-node + ``` ============================================================================== PERFORMANCE REPORT ------------------------------------------------------------------------------ - **** +SIESTA_Scalability h2o_64 input_siesta_4_0_1__gcc_10_3_0__metis_N_1_MPI_2_OMP_1 +- aws:c6gn + - builtin + * num_tasks: 2 + * Run Time: 1356.59 None +------------------------------------------------------------------------------ +SIESTA_Scalability h2o_64 input_siesta_4_0_1__gcc_10_3_0__metis_N_1_MPI_4_OMP_1 + - builtin + * num_tasks: 4 + * Run Time: 753.26 None +------------------------------------------------------------------------------ +SIESTA_Scalability h2o_64 input_siesta_4_0_1__gcc_10_3_0__metis_N_1_MPI_8_OMP_1 + - builtin + * num_tasks: 8 + * Run Time: 435.4 None +------------------------------------------------------------------------------ +SIESTA_Scalability h2o_64 input_siesta_4_0_1__gcc_10_3_0__metis_N_1_MPI_16_OMP_1 + - builtin + * num_tasks: 16 + * Run Time: 280.76 None +------------------------------------------------------------------------------ +SIESTA_Scalability h2o_64 input_siesta_4_0_1__gcc_10_3_0__metis_N_1_MPI_32_OMP_1 + - builtin + * num_tasks: 32 + * Run Time: 187.52 None +------------------------------------------------------------------------------ +SIESTA_Scalability h2o_64 input_siesta_4_0_1__gcc_10_3_0__metis_N_1_MPI_64_OMP_1 + - builtin + * num_tasks: 64 + * Run Time: 151.42 None ------------------------------------------------------------------------------ ``` +#### Off-node + +``` +============================================================================== +PERFORMANCE REPORT +------------------------------------------------------------------------------ +SIESTA_Scalability h2o_64 input_siesta_4_0_1__gcc_10_3_0__metis_N_1_MPI_8_OMP_1 +- aws:c6gn + - builtin + * num_tasks: 8 + * Run Time: 430.85 None +------------------------------------------------------------------------------ +SIESTA_Scalability h2o_64 input_siesta_4_0_1__gcc_10_3_0__metis_N_1_MPI_16_OMP_1 + - builtin + * num_tasks: 16 + * Run Time: 278.13 None +------------------------------------------------------------------------------ +SIESTA_Scalability h2o_64 input_siesta_4_0_1__gcc_10_3_0__metis_N_1_MPI_32_OMP_1 + - builtin + * num_tasks: 32 + * Run Time: 185.62 None +------------------------------------------------------------------------------ +SIESTA_Scalability h2o_64 input_siesta_4_0_1__gcc_10_3_0__metis_N_1_MPI_64_OMP_1 + - builtin + * num_tasks: 64 + * Run Time: 152.54 None +------------------------------------------------------------------------------ +SIESTA_Scalability h2o_64 input_siesta_4_0_1__gcc_10_3_0__metis_N_2_MPI_128_OMP_1 + - builtin + * num_tasks: 128 + * Run Time: 230.6 None +------------------------------------------------------------------------------ +SIESTA_Scalability h2o_64 input_siesta_4_0_1__gcc_10_3_0__metis_N_4_MPI_256_OMP_1 + - builtin + * num_tasks: 256 + * Run Time: 306.58 None +``` + ### On-node Compiler Comparison Performance comparison of two compilers. -| Cores | Compiler 1 | Compiler 2 | -|-------|------------|------------| -| | | | +| Cores | GNU | +|-------|----------| +| 2 | 1356.59 | +| 4 | 753.26 | +| 8 | 435.40 | +| 16 | 280.76 | +| 32 | 187.52 | +| 64 | 151.52 | ### Serial Hot-spot Profile @@ -110,18 +398,16 @@ Profiling command used: ### Strong Scaling Study -On-node scaling study for two compilers. - -| Cores | Compiler 1 | Compiler 2 | -|-------|------------|------------| -| 1 | | | -| 2 | | | -| 4 | | | -| 8 | | | -| 16 | | | -| 32 | | | -| 64 | | | +On-node scaling study for the GNU Compiler. +| Cores | GNU | +|-------|----------| +| 2 | 1356.59 | +| 4 | 753.26 | +| 8 | 435.40 | +| 16 | 280.76 | +| 32 | 187.52 | +| 64 | 151.52 | ### Off-Node Scaling Study diff --git a/Applications/Applications/siesta/job.offnode.c5n.py b/Applications/Applications/siesta/job.offnode.c5n.py new file mode 100644 index 00000000..5b3bb35f --- /dev/null +++ b/Applications/Applications/siesta/job.offnode.c5n.py @@ -0,0 +1,90 @@ +import reframe as rfm +import reframe.utility.sanity as sn +import hackathon as hack +# +#@rfm.simple_test +#class SiestaTest(hack.HackathonBase): +# # Where to run the binaries 'aws:c6gn' on Arm or 'aws:c5n' on Intel +# test_name = parameter( ['si64_coop', 'h2o', 'n_chain', 'carbon_nanoscroll'] ) +# +# valid_systems = ['aws:c6gn'] +# +# # Logging Variables +# log_team_name = 'GarotesdePremia' +# log_app_name = 'SIESTA' +# +# +# executable = 'siesta' +# +# spec = parameter([ +# 'siesta@4.0.1 %gcc@10.3.0' # Siesta GCC +# ]) +# +# parallelism = parameter([ +# { 'nodes' : 1, 'mpi' : 4, 'omp' : 1}, +# ]) +# +# @run_after('init') +# def get_name(self): +# name=self.test_name +# self.log_test_name = 'Test {} input'.format(name) +# self.prerun_cmds = ['cp -r /home/gnuille/siesta_exec/siesta .', 'cd siesta/Tests/{}'.format(name), 'for spec in `cat {}.pseudos`; do ln ../Pseudos/$spec.psf . ; done'.format(name) ] +# self.executable_opts = ['<', '{}.fdf'.format(name)] +# self.logfile = 'test_{}.log'.format(name) +# self.keep_files = [self.logfile] +# +# @run_before('sanity') +# def set_sanity_patterns(self): +# self.sanity_patterns = sn.assert_found('Job completed', sn.print(self.stdout)) + +@rfm.simple_test +class SiestaScalabilityTest(hack.HackathonBase): + + #test_name = parameter( ['h2o_2', 'h2o_4', 'h2o_512'] ) + test_name = parameter( ['h2o_64'] ) + + # Where to run the binaries 'aws:c6gn' on Arm or 'aws:c5n' on Intel + valid_systems = ['aws:c5n'] + + # Logging Variables + log_team_name = 'GarotesdePremia' + log_app_name = 'SIESTA' + + + executable = 'siesta' + + spec = parameter([ + 'siesta@4.0.1 %gcc@10.3.0 +metis' # Siesta GCC metis + ]) + + parallelism = parameter([ + { 'nodes' : 1, 'mpi' : 8, 'omp' : 1}, + { 'nodes' : 1, 'mpi' : 16, 'omp' : 1}, + { 'nodes' : 1, 'mpi' : 32, 'omp' : 1}, + { 'nodes' : 1, 'mpi' : 64, 'omp' : 1}, + { 'nodes' : 2, 'mpi' : 128, 'omp' : 1}, + { 'nodes' : 4, 'mpi' : 256, 'omp' : 1}, + ]) + + @run_after('init') + def get_name(self): + name=self.test_name + self.log_test_name = 'Scalability {} input'.format(name) + self.prerun_cmds = ['cp -r /scratch/home/jvinyals/siesta .', 'cd siesta/Tests/{}'.format(name), 'for spec in `cat {}.pseudos`; do ln ../Pseudos/$spec.psf . ; done'.format(name) ] + self.executable_opts = ['<', '{}.fdf'.format(name)] + self.logfile = 'test_{}.log'.format(name) + self.keep_files = [self.logfile] + + @run_before('sanity') + def set_sanity_patterns(self): + self.sanity_patterns = sn.assert_found('Job completed', sn.print(self.stdout)) + + @run_before('run') + def set_timmer(self): + self.job.launcher.options = ['/usr/bin/time -f "Elapsed for reframe: %e"'] + + @run_before('performance') + def set_perf_pattern(self): + self.perf_patterns = { + 'Run Time': sn.max(sn.extractall(r'Elapsed for reframe:\s+(\S+).*', self.stderr, 1, float)) + } diff --git a/Applications/Applications/siesta/job.offnode.c6gn.c6n.py b/Applications/Applications/siesta/job.offnode.c6gn.c6n.py new file mode 100644 index 00000000..593fb175 --- /dev/null +++ b/Applications/Applications/siesta/job.offnode.c6gn.c6n.py @@ -0,0 +1,90 @@ +import reframe as rfm +import reframe.utility.sanity as sn +import hackathon as hack +# +#@rfm.simple_test +#class SiestaTest(hack.HackathonBase): +# # Where to run the binaries 'aws:c6gn' on Arm or 'aws:c5n' on Intel +# test_name = parameter( ['si64_coop', 'h2o', 'n_chain', 'carbon_nanoscroll'] ) +# +# valid_systems = ['aws:c6gn'] +# +# # Logging Variables +# log_team_name = 'GarotesdePremia' +# log_app_name = 'SIESTA' +# +# +# executable = 'siesta' +# +# spec = parameter([ +# 'siesta@4.0.1 %gcc@10.3.0' # Siesta GCC +# ]) +# +# parallelism = parameter([ +# { 'nodes' : 1, 'mpi' : 4, 'omp' : 1}, +# ]) +# +# @run_after('init') +# def get_name(self): +# name=self.test_name +# self.log_test_name = 'Test {} input'.format(name) +# self.prerun_cmds = ['cp -r /home/gnuille/siesta_exec/siesta .', 'cd siesta/Tests/{}'.format(name), 'for spec in `cat {}.pseudos`; do ln ../Pseudos/$spec.psf . ; done'.format(name) ] +# self.executable_opts = ['<', '{}.fdf'.format(name)] +# self.logfile = 'test_{}.log'.format(name) +# self.keep_files = [self.logfile] +# +# @run_before('sanity') +# def set_sanity_patterns(self): +# self.sanity_patterns = sn.assert_found('Job completed', sn.print(self.stdout)) + +@rfm.simple_test +class SiestaScalabilityTest(hack.HackathonBase): + + #test_name = parameter( ['h2o_2', 'h2o_4', 'h2o_512'] ) + test_name = parameter( ['h2o_64'] ) + + # Where to run the binaries 'aws:c6gn' on Arm or 'aws:c5n' on Intel + valid_systems = ['aws:c6n', 'aws:c6gn'] + + # Logging Variables + log_team_name = 'GarotesdePremia' + log_app_name = 'SIESTA' + + + executable = 'siesta' + + spec = parameter([ + 'siesta@4.0.1 %gcc@10.3.0 +metis' # Siesta GCC metis + ]) + + parallelism = parameter([ + { 'nodes' : 1, 'mpi' : 8, 'omp' : 1}, + { 'nodes' : 1, 'mpi' : 16, 'omp' : 1}, + { 'nodes' : 1, 'mpi' : 32, 'omp' : 1}, + { 'nodes' : 1, 'mpi' : 64, 'omp' : 1}, + { 'nodes' : 2, 'mpi' : 128, 'omp' : 1}, + { 'nodes' : 4, 'mpi' : 256, 'omp' : 1}, + ]) + + @run_after('init') + def get_name(self): + name=self.test_name + self.log_test_name = 'Scalability {} input'.format(name) + self.prerun_cmds = ['cp -r /scratch/home/jvinyals/siesta .', 'cd siesta/Tests/{}'.format(name), 'for spec in `cat {}.pseudos`; do ln ../Pseudos/$spec.psf . ; done'.format(name) ] + self.executable_opts = ['<', '{}.fdf'.format(name)] + self.logfile = 'test_{}.log'.format(name) + self.keep_files = [self.logfile] + + @run_before('sanity') + def set_sanity_patterns(self): + self.sanity_patterns = sn.assert_found('Job completed', sn.print(self.stdout)) + + @run_before('run') + def set_timmer(self): + self.job.launcher.options = ['/usr/bin/time -f "Elapsed for reframe: %e"'] + + @run_before('performance') + def set_perf_pattern(self): + self.perf_patterns = { + 'Run Time': sn.max(sn.extractall(r'Elapsed for reframe:\s+(\S+).*', self.stderr, 1, float)) + } diff --git a/Applications/Applications/siesta/job.offnode.py b/Applications/Applications/siesta/job.offnode.py new file mode 100644 index 00000000..3d0c0e37 --- /dev/null +++ b/Applications/Applications/siesta/job.offnode.py @@ -0,0 +1,90 @@ +import reframe as rfm +import reframe.utility.sanity as sn +import hackathon as hack +# +#@rfm.simple_test +#class SiestaTest(hack.HackathonBase): +# # Where to run the binaries 'aws:c6gn' on Arm or 'aws:c5n' on Intel +# test_name = parameter( ['si64_coop', 'h2o', 'n_chain', 'carbon_nanoscroll'] ) +# +# valid_systems = ['aws:c6gn'] +# +# # Logging Variables +# log_team_name = 'GarotesdePremia' +# log_app_name = 'SIESTA' +# +# +# executable = 'siesta' +# +# spec = parameter([ +# 'siesta@4.0.1 %gcc@10.3.0' # Siesta GCC +# ]) +# +# parallelism = parameter([ +# { 'nodes' : 1, 'mpi' : 4, 'omp' : 1}, +# ]) +# +# @run_after('init') +# def get_name(self): +# name=self.test_name +# self.log_test_name = 'Test {} input'.format(name) +# self.prerun_cmds = ['cp -r /home/gnuille/siesta_exec/siesta .', 'cd siesta/Tests/{}'.format(name), 'for spec in `cat {}.pseudos`; do ln ../Pseudos/$spec.psf . ; done'.format(name) ] +# self.executable_opts = ['<', '{}.fdf'.format(name)] +# self.logfile = 'test_{}.log'.format(name) +# self.keep_files = [self.logfile] +# +# @run_before('sanity') +# def set_sanity_patterns(self): +# self.sanity_patterns = sn.assert_found('Job completed', sn.print(self.stdout)) + +@rfm.simple_test +class SiestaScalabilityTest(hack.HackathonBase): + + #test_name = parameter( ['h2o_2', 'h2o_4', 'h2o_512'] ) + test_name = parameter( ['h2o_64'] ) + + # Where to run the binaries 'aws:c6gn' on Arm or 'aws:c5n' on Intel + valid_systems = ['aws:c6gn'] + + # Logging Variables + log_team_name = 'GarotesdePremia' + log_app_name = 'SIESTA' + + + executable = 'siesta' + + spec = parameter([ + 'siesta@4.0.1 %gcc@10.3.0 +metis' # Siesta GCC metis + ]) + + parallelism = parameter([ + { 'nodes' : 1, 'mpi' : 8, 'omp' : 1}, + { 'nodes' : 1, 'mpi' : 16, 'omp' : 1}, + { 'nodes' : 1, 'mpi' : 32, 'omp' : 1}, + { 'nodes' : 1, 'mpi' : 64, 'omp' : 1}, + { 'nodes' : 2, 'mpi' : 128, 'omp' : 1}, + { 'nodes' : 4, 'mpi' : 256, 'omp' : 1}, + ]) + + @run_after('init') + def get_name(self): + name=self.test_name + self.log_test_name = 'Scalability {} input'.format(name) + self.prerun_cmds = ['cp -r /scratch/home/jvinyals/siesta .', 'cd siesta/Tests/{}'.format(name), 'for spec in `cat {}.pseudos`; do ln ../Pseudos/$spec.psf . ; done'.format(name) ] + self.executable_opts = ['<', '{}.fdf'.format(name)] + self.logfile = 'test_{}.log'.format(name) + self.keep_files = [self.logfile] + + @run_before('sanity') + def set_sanity_patterns(self): + self.sanity_patterns = sn.assert_found('Job completed', sn.print(self.stdout)) + + @run_before('run') + def set_timmer(self): + self.job.launcher.options = ['/usr/bin/time -f "Elapsed for reframe: %e"'] + + @run_before('performance') + def set_perf_pattern(self): + self.perf_patterns = { + 'Run Time': sn.max(sn.extractall(r'Elapsed for reframe:\s+(\S+).*', self.stderr, 1, float)) + } diff --git a/Applications/Applications/siesta/job.onnode.py b/Applications/Applications/siesta/job.onnode.py new file mode 100644 index 00000000..0508a3dc --- /dev/null +++ b/Applications/Applications/siesta/job.onnode.py @@ -0,0 +1,94 @@ +import reframe as rfm +import reframe.utility.sanity as sn +import hackathon as hack +# +#@rfm.simple_test +#class SiestaTest(hack.HackathonBase): +# # Where to run the binaries 'aws:c6gn' on Arm or 'aws:c5n' on Intel +# test_name = parameter( ['si64_coop', 'h2o', 'n_chain', 'carbon_nanoscroll'] ) +# +# valid_systems = ['aws:c6gn'] +# +# # Logging Variables +# log_team_name = 'GarotesdePremia' +# log_app_name = 'SIESTA' +# +# +# executable = 'siesta' +# +# spec = parameter([ +# 'siesta@4.0.1 %gcc@10.3.0' # Siesta GCC +# ]) +# +# parallelism = parameter([ +# { 'nodes' : 1, 'mpi' : 4, 'omp' : 1}, +# ]) +# +# @run_after('init') +# def get_name(self): +# name=self.test_name +# self.log_test_name = 'Test {} input'.format(name) +# self.prerun_cmds = ['cp -r /home/gnuille/siesta_exec/siesta .', 'cd siesta/Tests/{}'.format(name), 'for spec in `cat {}.pseudos`; do ln ../Pseudos/$spec.psf . ; done'.format(name) ] +# self.executable_opts = ['<', '{}.fdf'.format(name)] +# self.logfile = 'test_{}.log'.format(name) +# self.keep_files = [self.logfile] +# +# @run_before('sanity') +# def set_sanity_patterns(self): +# self.sanity_patterns = sn.assert_found('Job completed', sn.print(self.stdout)) + +@rfm.simple_test +class SiestaScalabilityTest(hack.HackathonBase): + + #test_name = parameter( ['h2o_2', 'h2o_4', 'h2o_512'] ) + test_name = parameter( ['h2o_64'] ) + + # Where to run the binaries 'aws:c6gn' on Arm or 'aws:c5n' on Intel + valid_systems = ['aws:c6gn'] + + # Logging Variables + log_team_name = 'GarotesdePremia' + log_app_name = 'SIESTA' + + + executable = 'siesta' + + spec = parameter([ + 'siesta@4.0.1 %gcc@10.3.0 +metis' # Siesta GCC metis + ]) + + parallelism = parameter([ + { 'nodes' : 1, 'mpi' : 1, 'omp' : 1}, + { 'nodes' : 1, 'mpi' : 2, 'omp' : 1}, + { 'nodes' : 1, 'mpi' : 4, 'omp' : 1}, + { 'nodes' : 1, 'mpi' : 8, 'omp' : 1}, + { 'nodes' : 1, 'mpi' : 16, 'omp' : 1}, + { 'nodes' : 1, 'mpi' : 32, 'omp' : 1}, + { 'nodes' : 1, 'mpi' : 64, 'omp' : 1}, + #{ 'nodes' : 2, 'mpi' : 128, 'omp' : 1}, + #{ 'nodes' : 4, 'mpi' : 256, 'omp' : 1}, + #{ 'nodes' : 8, 'mpi' : 512, 'omp' : 1}, + ]) + + @run_after('init') + def get_name(self): + name=self.test_name + self.log_test_name = 'Scalability {} input'.format(name) + self.prerun_cmds = ['cp -r /scratch/home/jvinyals/siesta .', 'cd siesta/Tests/{}'.format(name), 'for spec in `cat {}.pseudos`; do ln ../Pseudos/$spec.psf . ; done'.format(name) ] + self.executable_opts = ['<', '{}.fdf'.format(name)] + self.logfile = 'test_{}.log'.format(name) + self.keep_files = [self.logfile] + + @run_before('sanity') + def set_sanity_patterns(self): + self.sanity_patterns = sn.assert_found('Job completed', sn.print(self.stdout)) + + @run_before('run') + def set_timmer(self): + self.job.launcher.options = ['/usr/bin/time -f "Elapsed for reframe: %e"'] + + @run_before('performance') + def set_perf_pattern(self): + self.perf_patterns = { + 'Run Time': sn.max(sn.extractall(r'Elapsed for reframe:\s+(\S+).*', self.stderr, 1, float)) + } diff --git a/Applications/Applications/siesta/package.py b/Applications/Applications/siesta/package.py new file mode 100644 index 00000000..b298d666 --- /dev/null +++ b/Applications/Applications/siesta/package.py @@ -0,0 +1,115 @@ +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + + +import os + +from spack import * + + +class Siesta(Package): + """SIESTA performs electronic structure calculations and ab initio molecular + dynamics simulations of molecules and solids.""" + + homepage = "https://departments.icmab.es/leem/siesta/" + + version('4.0.1', sha256='bfb9e4335ae1d1639a749ce7e679e739fdead5ee5766b5356ea1d259a6b1e6d1', url='https://launchpad.net/siesta/4.0/4.0.1/+download/siesta-4.0.1.tar.gz') + version('3.2-pl-5', sha256='e438bb007608e54c650e14de7fa0b5c72562abb09cbd92dcfb5275becd929a23', url='http://departments.icmab.es/leem/siesta/CodeAccess/Code/siesta-3.2-pl-5.tgz') + + variant('metis', default=False, description='Enable METIS partitioner') + + with when('+metis'): + depends_on('metis') + + patch('configure.patch', when='@:4.0') + + depends_on('mpi') + depends_on('blas') + depends_on('lapack') + depends_on('scalapack') + depends_on('netcdf-c') + depends_on('netcdf-fortran') + + phases = ['configure', 'build', 'install'] + + def flag_handler(self, name, flags): + if '%gcc@10:' in self.spec and name == 'fflags': + flags.append('-fallow-argument-mismatch') + return (flags, None, None) + + def configure(self, spec, prefix): + sh = which('sh') + + # Patch config.guess and config.sub from gnu savannah + sh("-c", "rm -f Src/config.guess Src/config.sub Src/FoX/config/config.guess Src/FoX/config/config.sub") + sh("-c", "wget -O Src/config.guess \"https://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess;hb=HEAD\"") + sh("-c", "wget -O Src/config.sub \"https://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub;hb=HEAD\"") + + sh("-c", "wget -O Src/FoX/config/config.guess \"https://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess;hb=HEAD\"") + sh("-c", "wget -O Src/FoX/config/config.sub \"https://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub;hb=HEAD\"") + + sh("-c", "find -type f -exec sed -i 's/iso_fortran_env/siesta_fortran_env/g' {} \\;") + sh("-c", "find -iname iso_fortran_env.F90 -exec rename iso siesta {} \\;") + + configure_args = ['--enable-mpi', + '--with-blas=%s' % spec['blas'].libs, + '--with-lapack=%s' % spec['lapack'].libs, + # need to include BLAS below because Intel MKL's + # BLACS depends on BLAS, otherwise the compiler + # test fails + '--with-blacs=%s' % (spec['scalapack'].libs + + spec['blas'].libs), + '--with-scalapack=%s' % spec['scalapack'].libs, + '--with-netcdf=%s' % (spec['netcdf-fortran'].libs + + spec['netcdf-c'].libs), + # need to specify MPIFC explicitly below, otherwise + # Intel's mpiifort is not found + 'MPIFC=%s' % spec['mpi'].mpifc + ] + + if self.spec.satisfies('+metis'): + configure_args.append('LDFLAGS=-L%s -lmetis' % spec['metis'].libs) + + if self.spec.satisfies('%gcc'): + configure_args.append('FCFLAGS=-ffree-line-length-0 -DON_DOMAIN_DECOMP') + else: + if self.spec.satisfies('%gcc'): + configure_args.append('FCFLAGS=-ffree-line-length-0') + + if self.spec.satisfies('%arm'): + configure_args.append('FCFLAGS=-DNO_MPI_INTERFACES') + + + for d in ['Obj', 'Obj_trans']: + with working_dir(d, create=True): + sh('../Src/configure', *configure_args) + if spec.satisfies('@:4.0%intel'): + with open('arch.make', 'a') as f: + f.write('\natom.o: atom.F\n') + f.write('\t$(FC) -c $(FFLAGS) -O1') + f.write('$(INCFLAGS) $(FPPFLAGS) $<') + sh('../Src/obj_setup.sh') + + def build(self, spec, prefix): + + with working_dir('Obj'): + make(parallel=False) + with working_dir('Obj_trans'): + make('transiesta', parallel=False) + with working_dir('Util'): + sh = which('sh') + sh('build_all.sh') + + def install(self, spec, prefix): + mkdir(prefix.bin) + with working_dir('Obj'): + install('siesta', prefix.bin) + with working_dir('Obj_trans'): + install('transiesta', prefix.bin) + for root, _, files in os.walk('Util'): + for fname in files: + fname = join_path(root, fname) + if os.access(fname, os.X_OK): + install(fname, prefix.bin) From 0127c1f62477f332450128f83d01c35305d09ead Mon Sep 17 00:00:00 2001 From: Joan Vinyals Date: Fri, 16 Jul 2021 22:43:42 +0200 Subject: [PATCH 2/4] Go! Pending typo revision. --- Applications/Applications/siesta/README.md | 271 ++++++++++---------- Applications/Applications/siesta/package.py | 3 - 2 files changed, 141 insertions(+), 133 deletions(-) diff --git a/Applications/Applications/siesta/README.md b/Applications/Applications/siesta/README.md index d32ed456..8a97e32a 100644 --- a/Applications/Applications/siesta/README.md +++ b/Applications/Applications/siesta/README.md @@ -337,6 +337,13 @@ SIESTA_Scalability h2o_64 input_siesta_4_0_1__gcc_10_3_0__metis_N_4_MPI_256_OMP_ * Run Time: 306.58 None ``` +430.85 +278.13 +185.62 +152.54 +230.60 +306.58 + ### On-node Compiler Comparison Performance comparison of two compilers. @@ -350,52 +357,6 @@ Performance comparison of two compilers. | 32 | 187.52 | | 64 | 151.52 | - -### Serial Hot-spot Profile - -List of top-10 functions / code locations from a serial profile. - -Profiling command used: -``` -: -``` - -| Position | Routine | Time (s) | Time (%) | -|----------|---------|----------|----------| -| 1 | | | | -| 2 | | | | -| 3 | | | | -| 4 | | | | -| 5 | | | | -| 6 | | | | -| 7 | | | | -| 8 | | | | -| 9 | | | | -| 10 | | | | - - -### Full Node Hot-spot Profile - -List of top-10 functions / code locations from a full node profile. - -Profiling command used: -``` -: -``` - -| Position | Routine | Time (s) | Time (%) | MPI (%) | -|----------|---------|----------|----------|---------| -| 1 | | | | | -| 2 | | | | | -| 3 | | | | | -| 4 | | | | | -| 5 | | | | | -| 6 | | | | | -| 7 | | | | | -| 8 | | | | | -| 9 | | | | | -| 10 | | | | | - ### Strong Scaling Study On-node scaling study for the GNU Compiler. @@ -409,110 +370,160 @@ On-node scaling study for the GNU Compiler. | 32 | 187.52 | | 64 | 151.52 | -### Off-Node Scaling Study - -Off-node scaling study comparing C6g and C6gn instances. - -| Nodes | Cores | C6g | C6gn | -|-------|-------|-----|------| -| 1 | 8 | | | -| 1 | 16 | | | -| 1 | 32 | | | -| 1 | 64 | | | -| 2 | 128 | | | -| 4 | 256 | | | -| 8 | 512 | | | - - ### On-Node Architecture Comparison On-node scaling study for two architectures. -| Cores | C6gn (Aarch64) | C5n (X86) | -|-------|----------------|-----------| -| 1 | | | -| 2 | | | -| 4 | | | -| 8 | | | -| 16 | | | -| 32 | | | -| 64 | | | - +The c6ng on-node reframe report is the same one seent above in the # On-node section. -## Optimisation +The c5n on-node reframe report: +``` +[==========] Finished on Fri Jul 16 20:29:06 2021 +============================================================================== +PERFORMANCE REPORT +------------------------------------------------------------------------------ +SIESTA_Scalability h2o_64 input_siesta_4_0_1__gcc_10_3_0__metis_N_1_MPI_2_OMP_1 +- aws:c5n + - builtin + * num_tasks: 2 + * Run Time: 1385.13 None +------------------------------------------------------------------------------ +SIESTA_Scalability h2o_64 input_siesta_4_0_1__gcc_10_3_0__metis_N_1_MPI_4_OMP_1 + - builtin + * num_tasks: 4 + * Run Time: 782.3 None +------------------------------------------------------------------------------ +SIESTA_Scalability h2o_64 input_siesta_4_0_1__gcc_10_3_0__metis_N_1_MPI_8_OMP_1 + - builtin + * num_tasks: 8 + * Run Time: 457.13 None +------------------------------------------------------------------------------ +SIESTA_Scalability h2o_64 input_siesta_4_0_1__gcc_10_3_0__metis_N_1_MPI_16_OMP_1 + - builtin + * num_tasks: 16 + * Run Time: 298.04 None +------------------------------------------------------------------------------ +SIESTA_Scalability h2o_64 input_siesta_4_0_1__gcc_10_3_0__metis_N_1_MPI_32_OMP_1 + - builtin + * num_tasks: 32 + * Run Time: 205.45 None +------------------------------------------------------------------------------ +SIESTA_Scalability h2o_64 input_siesta_4_0_1__gcc_10_3_0__metis_N_1_MPI_64_OMP_1 + - builtin + * num_tasks: 64 + * Run Time: 160.11 None +------------------------------------------------------------------------------ +``` -Details of steps taken to optimise performance of the application. -Please document work with compiler flags, maths libraries, system libraries, code optimisations, etc. +The scalability comparison betwen architectures on-node. -### Compiler Flag Tuning +| Cores | C6gn (Aarch64) | C5n (X86) | +|-------|----------------|-----------| +| 2 | 430.85 | 1385.13 | +| 4 | 278.13 | 782.30 | +| 8 | 185.62 | 457.13 | +| 16 | 152.54 | 298.04 | +| 32 | 230.60 | 205.45 | +| 64 | 306.58 | 160.11 | -Compiler flags before: +The c6ng off-node reframe report: ``` -CFLAGS= -FFLAGS= +============================================================================== +PERFORMANCE REPORT +------------------------------------------------------------------------------ +SIESTA_Scalability h2o_64 input_siesta_4_0_1__gcc_10_3_0__metis_N_1_MPI_8_OMP_1 +- aws:c6gn + - builtin + * num_tasks: 8 + * Run Time: 434.38 None +------------------------------------------------------------------------------ +SIESTA_Scalability h2o_64 input_siesta_4_0_1__gcc_10_3_0__metis_N_1_MPI_16_OMP_1 + - builtin + * num_tasks: 16 + * Run Time: 279.96 None +------------------------------------------------------------------------------ +SIESTA_Scalability h2o_64 input_siesta_4_0_1__gcc_10_3_0__metis_N_1_MPI_32_OMP_1 + - builtin + * num_tasks: 32 + * Run Time: 187.25 None +------------------------------------------------------------------------------ +SIESTA_Scalability h2o_64 input_siesta_4_0_1__gcc_10_3_0__metis_N_1_MPI_64_OMP_1 + - builtin + * num_tasks: 64 + * Run Time: 150.2 None +------------------------------------------------------------------------------ +SIESTA_Scalability h2o_64 input_siesta_4_0_1__gcc_10_3_0__metis_N_2_MPI_128_OMP_1 + - builtin + * num_tasks: 128 + * Run Time: 237.7 None +------------------------------------------------------------------------------ +SIESTA_Scalability h2o_64 input_siesta_4_0_1__gcc_10_3_0__metis_N_4_MPI_256_OMP_1 + - builtin + * num_tasks: 256 + * Run Time: 327.59 None +------------------------------------------------------------------------------ ``` - -Compiler flags after: +The c5n off-node reframe report: ``` -CFLAGS= -FFLAGS= +============================================================================== +PERFORMANCE REPORT +------------------------------------------------------------------------------ +SIESTA_Scalability h2o_64 input_siesta_4_0_1__gcc_10_3_0__metis_N_1_MPI_8_OMP_1 +- aws:c5n + - builtin + * num_tasks: 8 + * Run Time: 459.34 None +------------------------------------------------------------------------------ +SIESTA_Scalability h2o_64 input_siesta_4_0_1__gcc_10_3_0__metis_N_1_MPI_16_OMP_1 + - builtin + * num_tasks: 16 + * Run Time: 301.42 None +------------------------------------------------------------------------------ +SIESTA_Scalability h2o_64 input_siesta_4_0_1__gcc_10_3_0__metis_N_1_MPI_32_OMP_1 + - builtin + * num_tasks: 32 + * Run Time: 208.06 None +------------------------------------------------------------------------------ +SIESTA_Scalability h2o_64 input_siesta_4_0_1__gcc_10_3_0__metis_N_1_MPI_64_OMP_1 + - builtin + * num_tasks: 64 + * Run Time: 161.46 None +------------------------------------------------------------------------------ +SIESTA_Scalability h2o_64 input_siesta_4_0_1__gcc_10_3_0__metis_N_2_MPI_128_OMP_1 + - builtin + * num_tasks: 128 + * Run Time: 343.96 None +------------------------------------------------------------------------------ +SIESTA_Scalability h2o_64 input_siesta_4_0_1__gcc_10_3_0__metis_N_4_MPI_256_OMP_1 + - builtin + * num_tasks: 256 + * Run Time: 417.51 None +------------------------------------------------------------------------------ ``` -#### Compiler Flag Performance - -| Cores | Original Flags | New Flags | -|-------|----------------|-----------| -| 1 | | | -| 2 | | | -| 4 | | | -| 8 | | | -| 16 | | | -| 32 | | | -| 64 | | | - - -### Maths Library Report - -Report on use of maths library calls generated by (Perf Lib Tools)[https://github.com/ARM-software/perf-libs-tools]. -Please attach the corresponding apl files. - - -### Maths Library Optimisation - -Performance analysis of the use of different maths libraries. - - -| Cores | OpenBLAS | ArmPL | BLIS | -|-------|----------|-------| ---- | -| 1 | | | | -| 2 | | | | -| 4 | | | | -| 8 | | | | -| 16 | | | | -| 32 | | | | -| 64 | | | | - - -### Performance Regression - -How fast can you make the code? - -Use all of the above aproaches and any others to make the code as fast as possible. -Demonstrate your gains by providing a scaling study for your test case, demonstrating the performance before and after. - +The scalability comparison betwen architectures off-node. +| Cores | Cores | C6gn (Aarch64) | C5n (X86) | +|-------|-------|----------------|-----------| +| 1 | 8 | 434.38 | 459,34 | +| 1 | 16 | 239.68 | 301.42 | +| 1 | 32 | 187.25 | 208.06 | +| 1 | 64 | 150.20 | 161.46 | +| 2 | 128 | 237.70 | 343.96 | +| 4 | 256 | 327.59 | 417.51 | ## Report ### Compilation Summary -Details of lessons from compiling the application. +Siesta was an easy application to compile using the GNU Compiler, on the other hand +using the ARM Compiler and the Nvidia HPC Compiler was not possible due to the code +that implements siesta. It has some incompatible/non-standard module implementation +that made it impossible for us to compile the aplication using said compailers. ### Performance Summary -Details of lessons from analysing the performance of the application. - - -### Optimisation Summary +Siesta is an aplication that scales fairly well on-node, and when scaling off-node it +starts to lose performance. Whereas we think that its scalaility is hardly related to +the input set, we think there is space for improvement. -Details of lessons from performance optimising the application. diff --git a/Applications/Applications/siesta/package.py b/Applications/Applications/siesta/package.py index b298d666..92f43393 100644 --- a/Applications/Applications/siesta/package.py +++ b/Applications/Applications/siesta/package.py @@ -50,9 +50,6 @@ def configure(self, spec, prefix): sh("-c", "wget -O Src/FoX/config/config.guess \"https://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess;hb=HEAD\"") sh("-c", "wget -O Src/FoX/config/config.sub \"https://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub;hb=HEAD\"") - sh("-c", "find -type f -exec sed -i 's/iso_fortran_env/siesta_fortran_env/g' {} \\;") - sh("-c", "find -iname iso_fortran_env.F90 -exec rename iso siesta {} \\;") - configure_args = ['--enable-mpi', '--with-blas=%s' % spec['blas'].libs, '--with-lapack=%s' % spec['lapack'].libs, From b4cd91321d24f6e5f3adb22bf380e4e79857dfcb Mon Sep 17 00:00:00 2001 From: Joan Vinyals Date: Fri, 16 Jul 2021 22:50:07 +0200 Subject: [PATCH 3/4] Pending of typo revision --- Applications/Applications/siesta/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Applications/Applications/siesta/README.md b/Applications/Applications/siesta/README.md index 8a97e32a..ce67feec 100644 --- a/Applications/Applications/siesta/README.md +++ b/Applications/Applications/siesta/README.md @@ -14,7 +14,7 @@ Details of any changes to the Spack recipe used. Git commit hash of checkout for pacakage: -Pull request for Spack recipe changes: +Pull request for Spack recipe changes: https://github.com/spack/spack/pull/24937 ### Building SIESTA From 279541526204dc5b2d33d91e3d9d9a8d60dbed27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joan=20Vinyals=20Ylla-Catal=C3=A0?= Date: Fri, 16 Jul 2021 22:54:55 +0200 Subject: [PATCH 4/4] Update README.md Added commit hash. --- Applications/Applications/siesta/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Applications/Applications/siesta/README.md b/Applications/Applications/siesta/README.md index ce67feec..307699a0 100644 --- a/Applications/Applications/siesta/README.md +++ b/Applications/Applications/siesta/README.md @@ -12,7 +12,7 @@ Details of any changes to the Spack recipe used. -Git commit hash of checkout for pacakage: +Git commit hash of checkout for pacakage: b4cd913 Pull request for Spack recipe changes: https://github.com/spack/spack/pull/24937