From 9b2794c6ada62fe866f1814460e7d38eca13ba71 Mon Sep 17 00:00:00 2001 From: Jacob Nelson Date: Tue, 26 Jan 2016 22:21:32 -0600 Subject: [PATCH 01/11] Add configure flags for setting compiler for third-party dependences separately from main project. --- configure | 31 +++++++++++++++++++++++-------- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/configure b/configure index 96a46c7c1..f0d64487d 100755 --- a/configure +++ b/configure @@ -9,6 +9,8 @@ opt.mode = 'Release' opt.cc = ENV['CC'] opt.cxx = ENV['CXX'] opt.third_party = ENV['GRAPPA_THIRD_PARTY'] +opt.third_party_cc = '' +opt.third_party_cxx = '' # verify we're running a supported ruby version required_version = '1.9.3' @@ -39,28 +41,30 @@ OptionParser.new{|p| p.on('--profiling',"Enable gperftools-based sampled profiling."){|b| opt.profiling = true } p.on('--vampir=path/to/vampirtrace/root',"Specify path to VampirTrace build (enables tracing)."){|p| opt.vampir = p; opt.tracing = true } p.on('--third-party=path/to/built/deps/root',"Can optionally pre-build third-party dependencies #{line_break}instead of re-building for each configuration."){|p| opt.third_party = p } + p.on('--third-party-cc=path/to/c/compiler','Use alternative C compiler just for third-party builds.'){|p| opt.third_party_cc = p; } + p.on('--third-party-cxx=path/to/c++/compiler','Use alternative C++ compiler just for third-party builds.'){|p| opt.third_party_cxx = p; } p.on('--build-here', "Just configure into current directory rather than creating new build directories."){|b| opt.build_here = true } p.on('--third-party-tarfile=/path/to/file.tar', "Instead of downloading third-party dependences from the web, extract them from the specified tar file (available from Grappa website)."){|p| opt.third_party_tarfile = p } p.on('--verbose', "Output verbose configure information."){|b| opt.verbose = true } p.on('--prefix=path/to/installed/grappa',"Specify destination for Grappa installation (default=/opt/grappa)."){|p| opt.install_prefix = p } }.parse!(ARGV) -if opt.used_cc_flag +if opt.used_cc_flag && not(opt.used_cxx_flag) # automatically determine cxx from --cc case opt.cc - when /gcc/ + when /gcc$/ opt.cxx = opt.cc.gsub(/gcc$/,'g++') - when /clang/ + when /clang$/ opt.cxx = opt.cc.gsub(/clang$/,'clang++') end end -if opt.used_cxx_flag +if opt.used_cxx_flag && not(opt.used_cc_flag) # automatically determine cc from --cxx case opt.cxx - when /g\+\+/ + when /g\+\+$/ opt.cc = opt.cxx.gsub(/g\+\+$/,'gcc') - when /clang\+\+/ + when /clang\+\+$/ opt.cc = opt.cxx.gsub(/clang\+\+$/,'clang') end end @@ -137,9 +141,20 @@ def configure(generator, mode, opt) args << "-DBASE_C_COMPILER=#{opt.cc}" args << "-DBASE_CXX_COMPILER=#{opt.cxx}" end - + args << "-DTHIRD_PARTY_ROOT:PATH=#{opt.third_party}" if opt.third_party - + + if opt.third_party_cc == '' + opt.third_party_cc = opt.cc + end + + if opt.third_party_cxx == '' + opt.third_party_cxx = opt.cxx + end + + args << "-DTHIRD_PARTY_C_COMPILER=#{opt.third_party_cc}" + args << "-DTHIRD_PARTY_CXX_COMPILER=#{opt.third_party_cxx}" + if mode # note: use 'RelWithDebInfo' because it adds '-g'... args << "-DCMAKE_BUILD_TYPE=" + {'Debug'=>'Debug', 'Release'=>'RelWithDebInfo'}[mode] From 8bdc8450a056643e9de84774288cd5be4528bdf2 Mon Sep 17 00:00:00 2001 From: Jacob Nelson Date: Tue, 26 Jan 2016 22:22:14 -0600 Subject: [PATCH 02/11] Upgrade third-party gflags to work with Cray compiler --- third-party/CMakeLists.txt | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/third-party/CMakeLists.txt b/third-party/CMakeLists.txt index 0fcd0cfda..f2feac1cf 100644 --- a/third-party/CMakeLists.txt +++ b/third-party/CMakeLists.txt @@ -70,23 +70,21 @@ endif() if( NOT GFLAGS_FOUND ) if(NOT "${CMAKE_THIRD_PARTY_TARFILE}" STREQUAL "") - set(URL "${THIRD_PARTY_DOWNLOADS}/gflags-2.0-no-svn-files.tar.gz") + set(URL "${THIRD_PARTY_DOWNLOADS}/v2.1.2.tar.gz") else() - set(URL "http://gflags.googlecode.com/files/gflags-2.0-no-svn-files.tar.gz") + set(URL "http://github.com/gflags/gflags/archive/v2.1.2.tar.gz") endif() message(" gflags: ") + ExternalProject_Add( third-party-gflags URL ${URL} ##URL_HASH SHA1=bfa0b399e03147b20cafcae62b59ee9163309821 # ignored to support older cmake - SOURCE_DIR ${EXTERN_BUILD_DIR}/src/gflags-2.0 + SOURCE_DIR ${EXTERN_BUILD_DIR}/src/gflags-2.1.2 PREFIX ${EXTERN_BUILD_DIR}/gflags STAMP_DIR ${EXTERN_STAMP_DIR}/gflags BINARY_DIR ${EXTERN_BUILD_DIR}/gflags INSTALL_DIR ${THIRD_PARTY_ROOT} - CONFIGURE_COMMAND - CC=${BASE_C_COMPILER} CXX=${BASE_CXX_COMPILER} CFLAGS=-g ${APPLE_CXXFLAGS} /configure --prefix= - BUILD_COMMAND make -j 4 - INSTALL_COMMAND make install + CMAKE_ARGS "${CMAKE_ARGS};-DBUILD_SHARED_LIBS=OFF;-DBUILD_STATIC_LIBS=ON;-DBUILD_gflags_LIB=ON;-DBUILD_gflags_nothreads_LIB=ON;-DBUILD_TESTING=OFF;-DBUILD_PACKAGING=OFF;-DBUILD_NC_TESTS=OFF;-DINSTALL_HEADERS=ON;-DCMAKE_INSTALL_PREFIX=${THIRD_PARTY_ROOT}" ) list(APPEND tool_list third-party-gflags) else() From 777f6435369d8fd9dbf63d4bb3da855b236b971d Mon Sep 17 00:00:00 2001 From: Jacob Nelson Date: Tue, 26 Jan 2016 22:25:10 -0600 Subject: [PATCH 03/11] Update private copy of glog to use separate third-party compiler, and to point at proper installation for gflags. --- third-party/CMakeLists.txt | 3 ++- third-party/google-glog/configure | 5 +++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/third-party/CMakeLists.txt b/third-party/CMakeLists.txt index f2feac1cf..ebd893a06 100644 --- a/third-party/CMakeLists.txt +++ b/third-party/CMakeLists.txt @@ -103,7 +103,8 @@ if( NOT GLOG_FOUND ) BINARY_DIR ${EXTERN_BUILD_DIR}/glog INSTALL_DIR ${THIRD_PARTY_ROOT} CONFIGURE_COMMAND - CC=${BASE_C_COMPILER} CXX=${BASE_CXX_COMPILER} ${APPLE_CXXFLAGS} CFLAGS=-g CXXFLAGS=-g CFLAGS=-I${THIRD_PARTY_ROOT}/include CXXFLAGS=-I${THIRD_PARTY_ROOT}/include LDFLAGS=-L${THIRD_PARTY_ROOT}/lib /configure --prefix= + CC=${THIRD_PARTY_C_COMPILER} CXX=${THIRD_PARTY_CXX_COMPILER} /configure --enable-shared=no --prefix= GFLAGS_CFLAGS="-I${THIRD_PARTY_ROOT}/include" GFLAGS_LIBS="${THIRD_PARTY_ROOT}/lib/libgflags.a" + BUILD_COMMAND make -j 4 INSTALL_COMMAND make install DEPENDS third-party-gflags diff --git a/third-party/google-glog/configure b/third-party/google-glog/configure index 53dc32dc7..1c05cae28 100755 --- a/third-party/google-glog/configure +++ b/third-party/google-glog/configure @@ -15907,8 +15907,9 @@ fi #CFLAGS="$SAVE_CFLAGS" #LIBS="$SAVE_LIBS" # Modified for Ymer 2011-01-30: use local copy of gflags -GFLAGS_CFLAGS="-I../gflags/src" -GFLAGS_LIBS="../gflags/.libs/libgflags.la" +### Modified again to allow Grappa to point at gflags in third-party +###GFLAGS_CFLAGS="-I../gflags/src" +###GFLAGS_LIBS="../gflags/.libs/libgflags.la" $as_echo "#define HAVE_LIB_GFLAGS 1" >>confdefs.h From 5a55a1022ce8e5e32672feb9c6eae2b08dbccb37 Mon Sep 17 00:00:00 2001 From: Jacob Nelson Date: Tue, 26 Jan 2016 23:31:01 -0600 Subject: [PATCH 04/11] Update documentation for Cray XC builds; update documentation to match current Slurm usage --- BUILD.md | 25 +++++++++++++++++++++---- README.md | 6 +++--- doc/running.md | 31 +++++++++++++++++++++++++++---- 3 files changed, 51 insertions(+), 11 deletions(-) diff --git a/BUILD.md b/BUILD.md index e55ae8930..3ed2682e0 100644 --- a/BUILD.md +++ b/BUILD.md @@ -18,8 +18,8 @@ This will build the Grappa static library (in `build/Make+Release/system/libGrap # in `build/Make+Release` make graph_new.exe # generates /build/Make+Release/applications/graph500/grappa/graph_new.exe - # to run, use the 'srun' script which has been copied to build/Make+Release/bin: - bin/grappa_srun --nnode=4 --ppn=4 -- applications/graph500/grappa/graph_new.exe --scale=20 --bench=bfs + # now run as you would any other MPI job. If your cluster uses Slurm, do: + srun --nodes=4 --ntasks-per-node=4 -- applications/graph500/grappa/graph_new.exe --scale=20 --bench=bfs This is the simplest build configuration. You are likely to want to specify more things, such as a specific compiler, a directory for already-installed dependencies so you don't have to rebuild them for each new configuration, and more. So read on. @@ -40,6 +40,7 @@ You must have a Linux system with the following installed to be able to build Gr * MVAPICH2 >= 1.9 * MPICH >= 3.1 * Intel MPI >= 5.0.2.044 + * Cray MPT >= 6.0, probably (tested with 7.1.3) The following dependencies are dealt with automatically. You may want to override the default behavior for your specific system as described in the next section, especially in the case of Boost (if you already have a copy in a non-standard place). @@ -107,7 +108,9 @@ CMake will download and build `gflags`, `boost`, and `gperfools`. It will build The external dependencies can be shared between Grappa configurations. If you specify a directory to `--third-party`, CMake will build and install the dependencies there, and then any other configurations will reuse them. Sometimes this won't work; for instance, if using two different compilers, you may have difficulty sharing a third-party directory. If this happens, just make a new third-party directory and rebuild them using the new configuration, or don't specify it and have this configuration build them just for itself. -Because Boost takes the longest to compile and is often included in systems, Boost can be specified separately from the other third-party installs. Existing system installs of the other dependencies should typically *not* be relied on. +Because Boost takes the longest to compile and is often included in systems, Boost can be specified separately from the other third-party installs. You can use the ```--boost``` argument to ```configure``` to point at an existing Boost installation. + +Existing system installs of the other dependencies should typically *not* be relied on. ### No web access for third-party dependencies @@ -115,6 +118,20 @@ If you want to build Grappa on a machine without access to the web, and that mac To do so, download this file: [http://grappa.cs.washington.edu/files/grappa-third-party-downloads.tar](http://grappa.cs.washington.edu/files/grappa-third-party-downloads.tar). Then run ```configure``` with the option ```--third-party-tarfile=``` pointing at the tarfile. +### Building on a Cray XC + +Cray provides helpful compiler wrapper scripts (```cc``` and ```CC```) that cause problems with some of the third-party dependences, so the ```configure``` script allows you to specify separate compilers for the dependences with the ```--third-party-cc``` and ```-third-party-cxx``` flags. Since the Cray C++ compiler is not supported, you'll need to use GCC or another supported compiler instead. Here's an example of how to build on a Cray XC: + +``` +module load cmake ruby +module swap PrgEnv-cray PrgEnv-gnu +./configure --cc=`which cc` --cxx=`which CC` --third-party-cc=`which gcc` --third-party-cxx=`which g++` +cd build/Make+Release +make -j16 && make install +``` + +Note that due to problems with dynamic linking, Grappa's unit tests don't build properly on a Cray XC yet. + ## Installing Grappa Grappa supports a ```install``` target which will install compiled libraries and header files to a directory on your system. The default install path is an ```install``` directory under the build directory, like ```build/Make+Release/install```. If you'd prefer it to go somewhere else, specify the ```--prefix=``` option when you run ```configure```. @@ -212,7 +229,7 @@ To make it easy to prototype ideas, there's a directory in root: `scratch`. Any make rebuild_cache # then... make scratch-test.exe - bin/grappa_srun --nnode=2 --ppn=1 -- scratch/scratch-test.exe + srun --nodes=2 --ntasks-per-node=1 -- scratch/scratch-test.exe ### Demos Similar to the scratch directory, all sub-directories in the `applications/demos` directory will be searched for `.cpp` files, and added as targets (`demo-#{base_name}.exe`). (note: search is not recursive, just one level of subdirectory). diff --git a/README.md b/README.md index 83afd7389..b9b73191c 100644 --- a/README.md +++ b/README.md @@ -48,13 +48,13 @@ cd build/Make+Release make demo-hello_world ``` -Now you should have a binary which you can launch as an MPI job. If you have Slurm installed on your system, you can use our convenient job-launch script: +Now you should have a binary which you can launch as an MPI job. If you have Slurm installed on your system, you should be able to do: ```bash -bin/grappa_srun --nnode=2 --ppn=2 -- applications/demos/hello_world/hello_world.exe +srun --nodes=2 --ntasks-per-node=2 -- applications/demos/hello_world/hello_world.exe ``` -Otherwise, just source ```bin/settings.sh``` and run as a MPI job on your system. +If you don't have Slurm, just run like you would any other MPI job. For more detailed instructions on building Grappa, see [BUILD.md](BUILD.md). diff --git a/doc/running.md b/doc/running.md index ed2b6482e..2a0502a64 100644 --- a/doc/running.md +++ b/doc/running.md @@ -1,10 +1,33 @@ Running Grappa programs =============================================================================== -Grappa jobs are run like any other MPI job. This process varies depending on how your cluster is configured and what version of MPI you use. The Grappa team primarily works on machines that use the Slurm scheduler, and thus we have built a tools to make it easy to run Grappa programs under Slurm. However, it is not necessary to use these tools to run Grappa jobs---as long as you make sure the right environment variables are set, any MPI job launcher will work. +Grappa jobs are run like any other MPI job. This process varies depending on how your cluster is configured and what version of MPI you use. The Grappa team primarily works on machines that use the Slurm scheduler, and thus we have built tools to make it easy to run Grappa programs under Slurm. However, it is not necessary to use these tools to run Grappa jobs---as long as you make sure the right environment variables are set, any MPI job launcher will work. -Using Slurm and `grappa_srun` +Using Slurm directly ------------------------------------------------------------------------------- -The Ruby script `bin/grappa_srun` can be used to automatically manage the pesky `srun` flags and GASNET settings that are needed to run Grappa programs correctly on a number of machines that we've tested on. Try running `bin/grappa_srun --help` for more detailed usage information. +Most of the machine we run on use Slurm as their job scheduler. You + +### Running an application: ### +```bash +# in build/Make+Release: # +# build the desired executable # +make -j graph_new.exe +# launch a Slurm job to run it: # +srun --nodes=2 --ntasks-per-node=2 -- graph_new.exe --scale=26 --bench=bfs --nbfs=8 --num_starting_workers=512 +``` + +### Running a Grappa test: ### +```bash +# in your configured directory, (e.g. build/Make+Release) # +# (builds and runs the test) # +make -j check-New_delegate_tests +# or in two steps: # +make -j New_delegate_tests.test +srun --nodes=2 --ntasks-per-node=2 -- system/New_delegate_tests.test +``` + +Using Slurm and `grappa_srun` (deprecated) +------------------------------------------------------------------------------- +The Ruby script `bin/grappa_srun` can be used to automatically manage the pesky `srun` flags and environment variables that are specific to a few machines used by teh Grappa team. We don't maintain the script anymore, so this probably won't be useful to you; we've documented it here for posterity. Try running `bin/grappa_srun --help` for more detailed usage information. ### Running an application: ### ```bash @@ -27,4 +50,4 @@ bin/grappa_srun --nnode=2 --ppn=2 --test=New_delegate_tests Using another MPI job launcher ------------------------------------------------------------------------------- -You may use whatever mechanism you normally do to launch MPI jobs, but make sure you set the appropriate environment variables. Take a look at `bin/srun_prolog` and `bin/srun_epilog` to see what we do for Slurm. +You may use whatever other mechanism you normally do to launch MPI jobs as well. You may want to set environment variables as set in ```util/env.sh``` or ```bin/env.sh``` (which will be automatically set if you ```source /bin/settings.sh```), but they shouldn't make a big difference. From 0825aeac9e1fcdc84fadf10d8876d2af9d935db9 Mon Sep 17 00:00:00 2001 From: Jacob Nelson Date: Tue, 2 Feb 2016 13:52:29 -0600 Subject: [PATCH 05/11] update standalone demo with clean target --- applications/demos/standalone/Makefile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/applications/demos/standalone/Makefile b/applications/demos/standalone/Makefile index 4406968d0..cd26b19e7 100644 --- a/applications/demos/standalone/Makefile +++ b/applications/demos/standalone/Makefile @@ -9,3 +9,6 @@ GRAPPA_IMPLICIT_RULES:=on include $(GRAPPA_PREFIX)/share/Grappa/grappa.mk standalone: standalone.o + +clean: + rm -f standalone standalone.o From 2a3e9ced06033befc3665725170b405a3dad3f0b Mon Sep 17 00:00:00 2001 From: Jacob Nelson Date: Tue, 2 Feb 2016 13:58:33 -0600 Subject: [PATCH 06/11] link static libraries by specifying full path to .a file --- CMakeLists.txt | 8 +++----- util/grappa.mk | 4 ++-- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 6dcc6c95b..9d7ab8d03 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -503,10 +503,8 @@ foreach(lib ${CMAKE_BINARY_DIR}/system/libGrappa.a ${GRAPPA_STATIC_LIBS}) list(APPEND GRAPPA_LIB_PATHS_INSTALLED ${dir}) endif() - # add library to link list - get_filename_component(name ${lib} NAME_WE) # remove any extension - string(SUBSTRING ${name} 3 -1 name) # chop off "lib" - list(APPEND GRAPPA_STATIC_LIB_NAMES ${name}) + # add library to link list (use full path for static linking) + list(APPEND GRAPPA_STATIC_LIB_NAMES ${lib}) # check for reasonable extension get_filename_component(ext ${lib} EXT) @@ -553,7 +551,7 @@ list(REMOVE_DUPLICATES GRAPPA_LIB_PATHS_INSTALLED) string(REPLACE ";" " \\\n\t${CMAKE_LIBRARY_PATH_FLAG}" GMAKE_LIB_PATHS_INSTALLED ";${GRAPPA_LIB_PATHS_INSTALLED}") list(REMOVE_DUPLICATES GRAPPA_STATIC_LIB_NAMES) -string(REPLACE ";" " \\\n\t${CMAKE_LINK_LIBRARY_FLAG}" GMAKE_STATIC_LIBS ";${GRAPPA_STATIC_LIB_NAMES}") +string(REPLACE ";" " \\\n\t" GMAKE_STATIC_LIBS ";${GRAPPA_STATIC_LIB_NAMES}") list(REMOVE_DUPLICATES GRAPPA_DYNAMIC_LIB_NAMES) string(REPLACE ";" " \\\n\t${CMAKE_LINK_LIBRARY_FLAG}" GMAKE_DYNAMIC_LIBS ";${GRAPPA_DYNAMIC_LIB_NAMES}") diff --git a/util/grappa.mk b/util/grappa.mk index 1b566c77f..2677c3289 100644 --- a/util/grappa.mk +++ b/util/grappa.mk @@ -77,8 +77,8 @@ GRAPPA_LINK_PATHS=-L$(GRAPPA_PREFIX)/lib ${GMAKE_LIB_PATHS_INSTALLED} GRAPPA_LDFLAGS=$(GRAPPA_LINK_FLAGS) $(GRAPPA_LINK_PATHS) # assign to LDLIBS for automatic builds using gmake default implicit rules -GRAPPA_LDLIBS=${CMAKE_EXE_LINK_STATIC_CXX_FLAGS} ${GMAKE_STATIC_LIBS} \ - ${CMAKE_EXE_LINK_DYNAMIC_CXX_FLAGS} ${GMAKE_DYNAMIC_LIBS} +GRAPPA_LDLIBS=${GMAKE_STATIC_LIBS} \ + ${GMAKE_DYNAMIC_LIBS} ############################################################################# # support for implicit make rules From 88016d2e71ea3c329eaf95f8e35f4f9879ad2166 Mon Sep 17 00:00:00 2001 From: Jacob Nelson Date: Tue, 26 Jan 2016 14:50:08 -0800 Subject: [PATCH 07/11] remove no-longer-used Boost path hint --- configure | 2 -- 1 file changed, 2 deletions(-) diff --git a/configure b/configure index f0d64487d..b0f694724 100755 --- a/configure +++ b/configure @@ -21,8 +21,6 @@ if (av_arr <=> rv_arr) < 0 end case `hostname` -when /n[\d+]/ - opt.boost = "/sampa/share/gcc-4.7.2/src/boost_1_51_0" when /pal/ opt.boost = "~nels707/boost153-install" end From adc47222a4d7940393d5476863ba349417dd868e Mon Sep 17 00:00:00 2001 From: Jacob Nelson Date: Tue, 2 Feb 2016 12:22:38 -0800 Subject: [PATCH 08/11] fix up static/dynamic library naming --- CMakeLists.txt | 80 +++++++++++++++++++++++++++++++------------------- 1 file changed, 50 insertions(+), 30 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 9d7ab8d03..75f2cd41e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -491,52 +491,71 @@ endif() #################################### # Build lists of library names and paths -# static libs -foreach(lib ${CMAKE_BINARY_DIR}/system/libGrappa.a ${GRAPPA_STATIC_LIBS}) - - # add directory to search paths - get_filename_component(dir ${lib} PATH) # get directory for libs - list(APPEND GRAPPA_LIB_PATHS ${dir}) - string(FIND "${dir}" "${CMAKE_BINARY_DIR}" STATIC_IN_BUILD_DIR) # -1 if not found, 0 if found - string(FIND "${dir}" "${THIRD_PARTY_ROOT}" STATIC_IN_TPR_DIR) # -1 if not found, 0 if found - if(NOT STATIC_IN_BUILD_DIR EQUAL 0 AND NOT STATIC_IN_TPR_DIR EQUAL 0) - list(APPEND GRAPPA_LIB_PATHS_INSTALLED ${dir}) +# fixup library lists in case things got added to the wrong one +foreach(lib ${GRAPPA_DYNAMIC_LIBS}) + get_filename_component(ext "${lib}" EXT) + if("${ext}" STREQUAL ".a") + list(APPEND GRAPPA_STATIC_LIBS "${lib}") + else() + list(APPEND GRAPPA_CLEANED_DYNAMIC_LIBS "${lib}") endif() - - # add library to link list (use full path for static linking) - list(APPEND GRAPPA_STATIC_LIB_NAMES ${lib}) - - # check for reasonable extension - get_filename_component(ext ${lib} EXT) - if(NOT ${ext} STREQUAL ".a") - message(WARNING "Static library found didn't have .a extension: ${lib}") +endforeach() +foreach(lib ${GRAPPA_STATIC_LIBS}) + get_filename_component(ext "${lib}" EXT) + if("${ext}" STREQUAL ".so") + message(WARNING "Adding to dynamic list: ${lib}") + list(APPEND GRAPPA_DYNAMIC_LIBS "${lib}") + else() + list(APPEND GRAPPA_CLEANED_STATIC_LIBS "${lib}") endif() endforeach() -# dynamic libs -foreach(lib ${GRAPPA_DYNAMIC_LIBS}) +# generate list of dynamic libs +foreach(lib ${GRAPPA_CLEANED_DYNAMIC_LIBS}) # add directory to search paths - get_filename_component(dir ${lib} PATH) - list(APPEND GRAPPA_LIB_PATHS ${dir}) + get_filename_component(dir "${lib}" PATH) + list(APPEND GRAPPA_LIB_PATHS "${dir}") string(FIND "${dir}" "${CMAKE_BINARY_DIR}" DYNAMIC_IN_BUILD_DIR) # -1 if not found, 0 if found string(FIND "${dir}" "${THIRD_PARTY_ROOT}" DYNAMIC_IN_TPR_DIR) # -1 if not found, 0 if found if(NOT DYNAMIC_IN_BUILD_DIR EQUAL 0 AND NOT DYNAMIC_IN_TPR_DIR EQUAL 0) - list(APPEND GRAPPA_LIB_PATHS_INSTALLED ${dir}) + list(APPEND GRAPPA_LIB_PATHS_INSTALLED "${dir}") endif() # add library to link list - get_filename_component(name ${lib} NAME_WE) - string(SUBSTRING ${name} 3 -1 name) - list(APPEND GRAPPA_DYNAMIC_LIB_NAMES ${name}) + get_filename_component(name "${lib}" NAME_WE) + string(SUBSTRING "${name}" 3 -1 name) + list(APPEND GRAPPA_DYNAMIC_LIB_NAMES "${name}") # check for reasonable extension - get_filename_component(ext ${lib} EXT) - if(NOT ${ext} STREQUAL ".so") + get_filename_component(ext "${lib}" EXT) + if(NOT "${ext}" STREQUAL ".so") message(WARNING "Dynamic library found didn't have .so extension: ${lib}") endif() endforeach() +# static libs +foreach(lib "${CMAKE_BINARY_DIR}/system/libGrappa.a" ${GRAPPA_CLEANED_STATIC_LIBS}) + + # add directory to search paths + get_filename_component(dir "${lib}" PATH) # get directory for libs + list(APPEND GRAPPA_LIB_PATHS "${dir}") + string(FIND "${dir}" "${CMAKE_BINARY_DIR}" STATIC_IN_BUILD_DIR) # -1 if not found, 0 if found + string(FIND "${dir}" "${THIRD_PARTY_ROOT}" STATIC_IN_TPR_DIR) # -1 if not found, 0 if found + if(NOT STATIC_IN_BUILD_DIR EQUAL 0 AND NOT STATIC_IN_TPR_DIR EQUAL 0) + list(APPEND GRAPPA_LIB_PATHS_INSTALLED "${dir}") + endif() + + # add library to link list (use full path for static linking) + list(APPEND GRAPPA_STATIC_LIB_NAMES "${lib}") + + # check for reasonable extension + get_filename_component(ext "${lib}" EXT) + if(NOT "${ext}" STREQUAL ".a") + message(WARNING "Static library found didn't have .a extension: ${lib}") + endif() +endforeach() + if(VERBOSE) message("GRAPPA_LIB_PATHS: ${GRAPPA_LIB_PATHS}") message("GRAPPA_STATIC_LIB_NAMES: ${GRAPPA_STATIC_LIB_NAMES}") @@ -557,10 +576,11 @@ list(REMOVE_DUPLICATES GRAPPA_DYNAMIC_LIB_NAMES) string(REPLACE ";" " \\\n\t${CMAKE_LINK_LIBRARY_FLAG}" GMAKE_DYNAMIC_LIBS ";${GRAPPA_DYNAMIC_LIB_NAMES}") # fill in prototype file and place in build directory -configure_file(util/grappa.mk util/grappa.mk) +configure_file("util/grappa.mk" "util/grappa.mk") # # Installation # -install(FILES ${CMAKE_BINARY_DIR}/util/grappa.mk DESTINATION "share/Grappa/") +install(FILES "${CMAKE_BINARY_DIR}/util/grappa.mk" + DESTINATION "share/Grappa/") From be8f7341b4747d8e3a6c594983671af3f4eaf0e8 Mon Sep 17 00:00:00 2001 From: Jacob Nelson Date: Tue, 2 Feb 2016 13:39:19 -0800 Subject: [PATCH 09/11] switch off RPATH by default --- BUILD.md | 34 ++++++++++++++++----- CMakeLists.txt | 81 ++++++++++++++++++++++++++++++-------------------- configure | 5 ++++ 3 files changed, 79 insertions(+), 41 deletions(-) diff --git a/BUILD.md b/BUILD.md index 3ed2682e0..f3c26d705 100644 --- a/BUILD.md +++ b/BUILD.md @@ -70,23 +70,33 @@ The `configure` script creates a new "build/*" subdirectory and runs CMake to ge Default: Make. Can specify multiple with commas. --mode=[Release]|Debug[,*] Build mode. Default: Release (with debug symbols) - --cc=path/to/c/compiler Use alternative C compiler. + --cc=path/to/c/compiler Use alternative C compiler (infer C++ compiler from this path). + --cxx=path/to/c++/compiler Use alternative C++ compiler (infer C compiler from this path). --boost=path/to/boost/root Specify location of compatible boost (>= 1.53) (otherwise, cmake will download and build it) - --name=NAME Add an additional name to this configuration to distinguish it - (i.e. compiler version) - --tracing Enable VampirTrace/gperftools-based sampled tracing. Looks for - VampirTrace build in 'third-party' dir. + --name=NAME Add an additional name to this configuration to + distinguish it (i.e. compiler version) + --tracing Enable VampirTrace/gperftools-based sampled tracing. + Looks for VampirTrace build in 'third-party' dir. + --profiling Enable gperftools-based sampled profiling. --vampir=path/to/vampirtrace/root Specify path to VampirTrace build (enables tracing). --third-party=path/to/built/deps/root - Can optionally pre-build third-party dependencies instead of - re-building for each configuration. + Can optionally pre-build third-party dependencies + instead of re-building for each configuration. + --third-party-cc=path/to/c/compiler + Use alternative C compiler just for third-party builds. + --third-party-cxx=path/to/c++/compiler + Use alternative C++ compiler just for third-party builds. + --build-here Just configure into current directory rather than creating new build directories. --third-party-tarfile=/path/to/file.tar Instead of downloading third-party dependences from the web, extract them from the specified tar file (available from Grappa website). - --prefix=path/to/grappa/installation + --enable_rpath Set RPATH on binaries to help them find necessary libraries. + --verbose Output verbose configure information. + --prefix=path/to/installed/grappa Specify destination for Grappa installation (defaults to inside build directory). + To build, after calling `configure`, cd into the generated directory, and use the build tool selected (e.g. `make` or `ninja`), specifying the desired target (e.g. `graph_new.exe` to build the new Graph500 implementation, or `check-New_delegate_tests` to run the delegate tests, or `demo-gups.exe` to build the GUPS demo). ### Generators @@ -186,6 +196,14 @@ We've made it easier to use Grappa as a library by providing a GNU Make include An example of the first usage is included in the directory ```applications/demos/standalone```. +## Finding libraries; RPATH; RUNPATH + +A standard problem when running MPI jobs is making sure the binary is able to find all the shared libraries it needs. Some systems are good about propagating the user's environment to all the job's processes (we've had good experiences with Slurm); but some require setting environment variables like ```LD_LIBRARY_PATH``` manually, which quickly gets complicated. Ideally we'd just build all binaries as static, but some MPI distributions make this difficult. + +To make this easier, you can pass the argument ```--enable_rpath``` to the configure script, and Grappa will be configured to set the ```RPATH``` search path in binaries it builds. This is not a foolproof solution. In particular, it overrides any ```LD_LIBRARY_PATH``` variable set in the environment. For this reason, this feature is disabled by default. + +If your binaries have trouble finding libraries when you run them, try reconfiguring/rebuilding with ```--enable_rpath```. + ## CMake Notes A couple notes about adding new targets for CMake to build. First: each directory where something is built should typically have a `CMakeLists.txt` file. Somewhere up the directory hierarchy, this directory must be 'added'. For instance, applications directories are added from `applications/CMakeLists.txt`: diff --git a/CMakeLists.txt b/CMakeLists.txt index 75f2cd41e..0a0d5540c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -7,6 +7,8 @@ include(util/grappa.cmake) option(TRACING "Sample statistics with VTrace at regular intervals." OFF) option(PROFILING "Sample profile with GPerftools at regular intervals." OFF) +option(ENABLE_RPATH "Set RPATH when building binaries." OFF) + SET(GRAPPA_INSTALL_PREFIX "${CMAKE_BINARY_DIR}/install" CACHE PATH "Prefix prepended to install directories") SET(CMAKE_INSTALL_PREFIX "${GRAPPA_INSTALL_PREFIX}" CACHE INTERNAL "Prefix prepended to install directories" FORCE) @@ -44,32 +46,6 @@ else() message(WARNING "You are using an unsupported compiler! Compilation has only been tested with Clang and GCC.") endif() -########################### -# Use RUNPATH if available -# use, i.e. don't skip the full RPATH for the build tree -set(CMAKE_SKIP_BUILD_RPATH FALSE) -# use final INSTALL_RPATH even in build tree (this lets us manually add things to CMAKE_INSTALL_RPATH) -set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE) -set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/lib:${CMAKE_INSTALL_RPATH}") -# add the automatically determined parts of the RPATH -# which point to directories outside the build tree to the install RPATH -# set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE) -set(CMAKE_INSTALL_RPATH_USE_LINK_PATH FALSE) - -# the RPATH to be used when installing, but only if it's not a system directory -list(FIND CMAKE_PLATFORM_IMPLICIT_LINK_DIRECTORIES "${CMAKE_INSTALL_PREFIX}/lib" isSystemDir) -if("${isSystemDir}" STREQUAL "-1") # not a system directory - set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_RPATH}:${CMAKE_INSTALL_PREFIX}/lib") -endif("${isSystemDir}" STREQUAL "-1") - -## Not setting runpath because having MPI libs in LD_LIBRARY_PATH was messing up VampirTrace's ability to find its own libs. Maybe there's another way to fix this, but it just seems more robust (for now) to not allow LD_LIBRARY_PATH to affect our libs (after configure uses it to find them). -# set runpath, too -# if(NOT APPLE) -# set(CMAKE_EXE_LINKER_FLAGS "-Wl,--enable-new-dtags") -# endif() - -##### ######## - ############################################################################# # configure profiling and tracing @@ -100,7 +76,9 @@ if(TRACING) # set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} --as-needed --whole-archive -lvt --no-whole-archive --no-as-needed -lopen-trace-format -lz -lpapi -ldl -lm") if(NOT THIRD_PARTY_ROOT STREQUAL ${VAMPIR_ROOT}) - set(CMAKE_INSTALL_RPATH "${VAMPIR_ROOT}/lib:${CMAKE_INSTALL_RPATH}") + if(ENABLE_RPATH) + set(CMAKE_INSTALL_RPATH "${VAMPIR_ROOT}/lib:${CMAKE_INSTALL_RPATH}") + endif() endif() endif() @@ -132,7 +110,10 @@ else() endif() include_directories("${THIRD_PARTY_ROOT}/include") link_directories("${THIRD_PARTY_ROOT}/lib") -set(CMAKE_INSTALL_RPATH "${THIRD_PARTY_ROOT}/lib:${CMAKE_INSTALL_RPATH}") + +if(ENABLE_RPATH) + set(CMAKE_INSTALL_RPATH "${THIRD_PARTY_ROOT}/lib:${CMAKE_INSTALL_RPATH}") +endif() # @@ -356,12 +337,46 @@ endif() add_subdirectory(third-party) +# RPATH causes problems on some systems, but is helpful on others. Make it an option +if(ENABLE_RPATH) + ########################### + # Use RUNPATH if available + # use, i.e. don't skip the full RPATH for the build tree + set(CMAKE_SKIP_BUILD_RPATH FALSE) + # use final INSTALL_RPATH even in build tree (this lets us manually add things to CMAKE_INSTALL_RPATH) + set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE) + set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_RPATH}:${CMAKE_INSTALL_PREFIX}/lib") + # add the automatically determined parts of the RPATH + # which point to directories outside the build tree to the install RPATH + # set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE) + set(CMAKE_INSTALL_RPATH_USE_LINK_PATH FALSE) + + # the RPATH to be used when installing, but only if it's not a system directory + list(FIND CMAKE_PLATFORM_IMPLICIT_LINK_DIRECTORIES "${CMAKE_INSTALL_PREFIX}/lib" isSystemDir) + if("${isSystemDir}" STREQUAL "-1") # not a system directory + set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_RPATH}:${CMAKE_INSTALL_PREFIX}/lib") + endif("${isSystemDir}" STREQUAL "-1") + + # not sure if this is useful any more, but what the heck. + foreach(lib ${Boost_LIBRARY_DIRS}) + set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_RPATH}:${lib}") + endforeach() + + ## Not setting runpath because having MPI libs in LD_LIBRARY_PATH was messing up VampirTrace's ability to find its own libs. Maybe there's another way to fix this, but it just seems more robust (for now) to not allow LD_LIBRARY_PATH to affect our libs (after configure uses it to find them). + # set runpath, too + # if(NOT APPLE) + # set(CMAKE_EXE_LINKER_FLAGS "-Wl,--enable-new-dtags") + # endif() + + + ##### ######## + +else() + set(CMAKE_INSTALL_RPATH "") +endif() + +message("Setting RPATH to ${CMAKE_INSTALL_RPATH}") -# RPATH -# not sure if this is useful any more, but what the heck. -foreach(lib ${Boost_LIBRARY_DIRS}) - set(CMAKE_INSTALL_RPATH "${lib}:${CMAKE_INSTALL_RPATH}") -endforeach() diff --git a/configure b/configure index b0f694724..4b5d0014c 100755 --- a/configure +++ b/configure @@ -43,6 +43,7 @@ OptionParser.new{|p| p.on('--third-party-cxx=path/to/c++/compiler','Use alternative C++ compiler just for third-party builds.'){|p| opt.third_party_cxx = p; } p.on('--build-here', "Just configure into current directory rather than creating new build directories."){|b| opt.build_here = true } p.on('--third-party-tarfile=/path/to/file.tar', "Instead of downloading third-party dependences from the web, extract them from the specified tar file (available from Grappa website)."){|p| opt.third_party_tarfile = p } + p.on('--enable_rpath', "Set RPATH on binaries to help them find necessary libraries."){|b| opt.enable_rpath = true } p.on('--verbose', "Output verbose configure information."){|b| opt.verbose = true } p.on('--prefix=path/to/installed/grappa',"Specify destination for Grappa installation (default=/opt/grappa)."){|p| opt.install_prefix = p } }.parse!(ARGV) @@ -164,6 +165,10 @@ def configure(generator, mode, opt) args << "-DBOOST_ROOT=#{opt.boost}" end + if opt.enable_rpath + args << "-DENABLE_RPATH=ON" + end + if opt.verbose args << "-DVERBOSE=ON" end From a39a469da5bc4e879f55f34762728fe5a5842e8c Mon Sep 17 00:00:00 2001 From: Jacob Nelson Date: Tue, 2 Feb 2016 17:08:20 -0600 Subject: [PATCH 10/11] whoops, make glog configure change in non-auto-generated place --- third-party/google-glog/configure.ac | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/third-party/google-glog/configure.ac b/third-party/google-glog/configure.ac index 7f79a538b..41a59a8dd 100644 --- a/third-party/google-glog/configure.ac +++ b/third-party/google-glog/configure.ac @@ -131,8 +131,9 @@ fi #CFLAGS="$SAVE_CFLAGS" #LIBS="$SAVE_LIBS" # Modified for Ymer 2011-01-30: use local copy of gflags -GFLAGS_CFLAGS="-I../gflags/src" -GFLAGS_LIBS="../gflags/.libs/libgflags.la" +### Modified again to allow Grappa to point at gflags in third-party +###GFLAGS_CFLAGS="-I../gflags/src" +###GFLAGS_LIBS="../gflags/.libs/libgflags.la" AC_DEFINE([HAVE_LIB_GFLAGS], [1], [Define to 1 if you have google glog library]) ac_cv_have_libgflags=1 From 9b9064b29d7b978379280197e321ad0535df2111 Mon Sep 17 00:00:00 2001 From: Jacob Nelson Date: Tue, 2 Feb 2016 18:54:33 -0600 Subject: [PATCH 11/11] clean up third party compiler flags --- CMakeLists.txt | 2 -- configure | 20 ++++++++++++++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 0a0d5540c..9594a65ec 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -375,8 +375,6 @@ else() set(CMAKE_INSTALL_RPATH "") endif() -message("Setting RPATH to ${CMAKE_INSTALL_RPATH}") - diff --git a/configure b/configure index 4b5d0014c..9613dc07a 100755 --- a/configure +++ b/configure @@ -68,6 +68,26 @@ if opt.used_cxx_flag && not(opt.used_cc_flag) end end +if opt.third_party_cc != '' && opt.third_party_cxx == '' + # automatically determine cxx from --cc + case opt.third_party_cc + when /gcc$/ + opt.third_party_cxx = opt.third_party_cc.gsub(/gcc$/,'g++') + when /clang$/ + opt.third_party_cxx = opt.third_party_cc.gsub(/clang$/,'clang++') + end +end + +if opt.third_party_cxx != '' && opt.third_party_cc == '' + # automatically determine cc from --cxx + case opt.third_party_cxx + when /g\+\+$/ + opt.third_party_cc = opt.third_party_cxx.gsub(/g\+\+$/,'gcc') + when /clang\+\+$/ + opt.third_party_cc = opt.third_party_cxx.gsub(/clang\+\+$/,'clang') + end +end + # Xcode generator must have CC set to a version of Clang, and must have a corresponding plugin # see github.com/bholt/Clang-Head.xcplugin for an example