started unifying tracing

.github/workflows/cmake.yml

@@ -73,7 +73,8 @@ jobs:
           6.2)
             wget https://repo.radeon.com/amdgpu-install/6.2/ubuntu/jammy/amdgpu-install_6.2.60200-1_all.deb
             sudo apt install ./amdgpu-install_6.2.60200-1_all.deb
-            sudo amdgpu-install -y --usecase=hiplibsdk,rocm --no-dkms
+            sudo amdgpu-install -y --usecase=hiplibsdk --no-dkms
+            rm ./amdgpu-install_6.2.60200-1_all.deb
             ;;
         esac
 

CMakeLists.txt

@@ -57,6 +57,14 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Linux")
 	endif()
 endif()
 
+set(FIRESTARTER_TRACING "None" CACHE STRING "FIRESTARTER_TRACING can be any of None, Logger, or ScoreP. This feature is currently under development and static linking might not work.")
+set_property(CACHE FIRESTARTER_TRACING PROPERTY STRINGS None Logger ScoreP External)
+
+set(FIRESTARTER_TRACING_CXX_FLAGS "" CACHE STRING "FIRESTARTER_TRACING_CXX_FLAGS should add include paths for the selected tracing infrastructure.")
+set(FIRESTARTER_TRACING_LD_FLAGS "" CACHE STRING "FIRESTARTER_TRACING_LD_FLAGS should set linker flags for the selected tracing infrastructure.")
+set(FIRESTARTER_TRACING_ADDITIONAL_FILES "" CACHE STRING "FIRESTARTER_TRACING_ADDITIONAL_FILES can be used to add files to the compilation process that are needed for tracing.")
+
+
 set(FIRESTARTER_BUILD_TYPE "FIRESTARTER" CACHE STRING "FIRESTARTER_BUILD_TYPE can be any of FIRESTARTER, FIRESTARTER_CUDA, FIRESTARTER_ONEAPI, or FIRESTARTER_HIP.")
 set_property(CACHE FIRESTARTER_BUILD_TYPE PROPERTY STRINGS FIRESTARTER FIRESTARTER_CUDA FIRESTARTER_ONEAPI)
 if (${FIRESTARTER_BUILD_TYPE} STREQUAL "FIRESTARTER")

examples/tracing/README.md

@@ -0,0 +1,22 @@
+# In this folder, there are examples on how one could include tracing with an external library
+
+Since these will be mostly shared libraries, we probably will only be able to use them with a dynamic build of FIRESTARTER
+
+You should be able to add the external libraries also to a static build by using the flags `FIRESTARTER_TRACING_*` for `cmake`. There you probably would include an archive. But this was not tested yet.
+
+## Example 1: Score-P
+
+- Needs Score-P: https://www.vi-hps.org/projects/score-p
+- File: `scorep.c`
+- Compilation:
+  - 1. create the adapter configuration: `scorep-config --adapter-init --dynamic --user --nokokkos --nocompiler --thread=pthread > .scorep_init.c`
+  - 2. compile the adapter and the tracing library: `scorep --user --nocompiler --dynamic --nokokkos --noopenmp --thread=pthread gcc -fPIC -c -DSCOREP_USER_ENABLE scorep.c .scorep_init.c`
+  - 3. link the tracing library: `scorep --no-as-needed --dynamic --user --nokokkos --nocompiler --noopenmp --thread=pthread gcc -shared -o libfirestarter_scorep.so scorep.o .scorep_init.o`
+  - 4. cmake FIRESTARTER: `cmake -DFIRESTARTER_TRACING=External -DFIRESTARTER_TRACING_LD_FLAGS="-L/home/rschoene/git/FIRESTARTER/examples/tracing -lfirestarter_scorep" -DFIRESTARTER_LINK_STATIC=OFF ..`
+  - 5. make FIRESTARTER: `make -j`
+
+- Running:
+  - Make sure that the library `libfirestarter_scorep.so` can be found in the `LD_LIBRARY_PATH`
+  - Run `FIRESTARTER` as usual
+  - Running `FIRESTARTER` should create a profile in your current directory starting with `scorep...`
+  - You can change environment variables to tune Score-P for your purposes. Have a look at: https://perftools.pages.jsc.fz-juelich.de/cicd/scorep/tags/latest/pdf/scorep.pdf

examples/tracing/scorep.c

@@ -0,0 +1,35 @@
+/******************************************************************************
+ * FIRESTARTER - A Processor Stress Test Utility
+ * Copyright (C) 2024 TU Dresden, Center for Information Services and High
+ * Performance Computing
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/\>.
+ *
+ * Contact: daniel.hackenberg@tu-dresden.de
+ *****************************************************************************/
+
+#include "../../include/firestarter/Tracing/FIRESTARTER_Tracing.h"
+#include <scorep/SCOREP_User.h>
+
+void firestarter_tracing_initialize(int argc, const char **argv) {
+}
+
+void firestarter_tracing_region_begin(char const* region_name) {
+    SCOREP_USER_REGION_BY_NAME_BEGIN(region_name,
+                                     SCOREP_USER_REGION_TYPE_COMMON);
+}
+
+void firestarter_tracing_region_end(char const* region_name) {
+    SCOREP_USER_REGION_BY_NAME_END(region_name);
+}

include/firestarter/Tracing/FIRESTARTER_Tracing.h

@@ -0,0 +1,32 @@
+/******************************************************************************
+ * FIRESTARTER - A Processor Stress Test Utility
+ * Copyright (C) 2024 TU Dresden, Center for Information Services and High
+ * Performance Computing
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/\>.
+ *
+ * Contact: daniel.hackenberg@tu-dresden.de
+ *****************************************************************************/
+
+#pragma once
+
+#ifdef FIRESTARTER_TRACING
+void firestarter_tracing_initialize(int argc, const char **argv);
+void firestarter_tracing_region_begin(char const* region_name);
+void firestarter_tracing_region_end(char const* region_name);
+#else
+#define firestarter_tracing_initialize(argc, argv) {}
+#define firestarter_tracing_region_begin(region_name) {}
+#define firestarter_tracing_region_end(region_name) {}
+#endif

include/firestarter/Tracing/Tracing.hpp

@@ -0,0 +1,28 @@
+/******************************************************************************
+ * FIRESTARTER - A Processor Stress Test Utility
+ * Copyright (C) 2024 TU Dresden, Center for Information Services and High
+ * Performance Computing
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/\>.
+ *
+ * Contact: daniel.hackenberg@tu-dresden.de
+ *****************************************************************************/
+
+#pragma once
+
+namespace firestarter::tracing {
+  void initialize(int argc, const char **argv);
+  void regionBegin(char const* region_name);
+  void regionEnd(char const* region_name);
+} // firestarter::tracing

src/CMakeLists.txt

@@ -41,8 +41,92 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Linux")
 		)
 endif()
 
-SET(FIRESTARTER_LINK_LIBRARIES
+if(NOT "${FIRESTARTER_TRACING}" STREQUAL "None")
+	SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DFIRESTARTER_TRACING")
+	SET(FIRESTARTER_FILES
+		${FIRESTARTER_FILES}
+		${FIRESTARTER_TRACING_ADDITIONAL_FILES}
+	)
+	SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${FIRESTARTER_TRACING_CXXFLAGS}")
+endif()
+
+if("${FIRESTARTER_TRACING}" STREQUAL "Logger")
+	SET(FIRESTARTER_FILES 
+		${FIRESTARTER_FILES}
+		firestarter/Tracing/LogTracing.cpp
+	)
+elseif("${FIRESTARTER_TRACING}" STREQUAL "ScoreP")
+	find_program(SCOREP_CONFIG scorep-config)
+	if(SCOREP_CONFIG)
+		message(STATUS "Found 'scorep-config' at: ${SCOREP_CONFIG}")
+		SET(FIRESTARTER_FILES
+			${FIRESTARTER_FILES}
+			firestarter/Tracing/ScoreP.cpp
+			${FIRESTARTER_TRACING_ADDITIONAL_FILES}
+		)
+		SET(SCOREP_CONFIG_FLAGS "--nocompiler" "--user" "--nopomp" "--thread=pthread")
+		if ("${FIRESTARTER_LINK_STATIC}")
+			SET(SCOREP_CONFIG_FLAGS "--nocompiler" "--user" "--nopomp" "--thread=pthread" "--static")
+		endif()
+		execute_process(
+			COMMAND ${SCOREP_CONFIG} --cppflags
+			OUTPUT_VARIABLE SCOREP_CPP_FLAGS
+			ERROR_VARIABLE PROGRAM_ERROR
+			OUTPUT_STRIP_TRAILING_WHITESPACE
+		)
+		if (PROGRAM_ERROR)
+			message(FATAL_ERROR "Error when calling scorep-config: ${PROGRAM_ERROR}")
+		endif()
+
+		execute_process(
+			COMMAND ${SCOREP_CONFIG} --adapter-init ${SCOREP_CONFIG_FLAGS}
+			OUTPUT_VARIABLE SCOREP_ADAPTER_INIT_CONTENT
+			ERROR_VARIABLE PROGRAM_ERROR
+			OUTPUT_STRIP_TRAILING_WHITESPACE
+		)
+		if (PROGRAM_ERROR)
+			message(FATAL_ERROR "Error when calling scorep-config: ${PROGRAM_ERROR}")
+		endif()
+
+		file(WRITE ${CMAKE_BINARY_DIR}/.scorep_init.c "${SCOREP_ADAPTER_INIT_CONTENT}\n")
+                SET(FIRESTARTER_FILES   
+                        ${FIRESTARTER_FILES}
+                        ${CMAKE_BINARY_DIR}/.scorep_init.c
+                )
+
+		execute_process(
+			COMMAND ${SCOREP_CONFIG} --ldflags "--nocompiler" "--user" "--nopomp"
+			OUTPUT_VARIABLE SCOREP_LD_FLAGS
+			ERROR_VARIABLE PROGRAM_ERROR
+			OUTPUT_STRIP_TRAILING_WHITESPACE
+		)
+		if (PROGRAM_ERROR)
+			message(FATAL_ERROR "Error when calling scorep-config: ${PROGRAM_ERROR}")
+		endif()
+
+		execute_process(
+            COMMAND ${SCOREP_CONFIG} --libs ${SCOREP_CONFIG_FLAGS}
+			OUTPUT_VARIABLE SCOREP_LIBS
+            ERROR_VARIABLE PROGRAM_ERROR
+            OUTPUT_STRIP_TRAILING_WHITESPACE
+        )
+		if (PROGRAM_ERROR)
+			message(FATAL_ERROR "Error when calling scorep-config: ${PROGRAM_ERROR}")
+		endif()
+
+		SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DSCOREP_USER_ENABLE ${SCOREP_CPP_FLAGS}")
+		SET(FIRESTARTER_TRACING_LDFLAGS "${FIRESTARTER_TRACING_LDFLAGS} ${SCOREP_LD_FLAGS} ${SCOREP_LIBS}")
+	else()
+		message(FATAL_ERROR "You want to compile with Score-P support, but scorep-config could not be found in PATH")
+	endif()
+
+elseif("${FIRESTARTER_TRACING}" STREQUAL "Caliper")
+	SET(FIRESTARTER_FILES 
+		${FIRESTARTER_FILES}
+		firestarter/Tracing/Caliper.cpp
 	)
+	#todo add extra code
+endif()
 
 if (FIRESTARTER_THREAD_AFFINITY)
 	SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DFIRESTARTER_THREAD_AFFINITY")
@@ -231,3 +315,12 @@ elseif(${FIRESTARTER_BUILD_TYPE} STREQUAL "FIRESTARTER")
 		nlohmann_json::nlohmann_json
 		)
 endif()
+
+if(NOT "${FIRESTARTER_TRACING}" STREQUAL "None")
+	string(STRIP "${FIRESTARTER_TRACING_LD_FLAGS}" FIRESTARTER_TRACING_LD_FLAGS)
+	target_link_libraries("${FIRESTARTER_BUILD_TYPE}"
+		"${FIRESTARTER_TRACING_LD_FLAGS}"
+		)
+endif()
+
+message(STATUS "FIRESTARTER_TRACING_LD_FLAGS: ${FIRESTARTER_TRACING_LD_FLAGS}")

src/firestarter/Firestarter.cpp

@@ -21,6 +21,11 @@
 
 #include <firestarter/Firestarter.hpp>
 #include <firestarter/Logging/Log.hpp>
+
+extern "C" {
+#include <firestarter/Tracing/FIRESTARTER_Tracing.h>
+}
+
 #if defined(linux) || defined(__linux__)
 #include <firestarter/Optimizer/Algorithm/NSGA2.hpp>
 #include <firestarter/Optimizer/History.hpp>
@@ -325,6 +330,9 @@ Firestarter::~Firestarter() {
 }
 
 void Firestarter::mainThread() {
+
+  firestarter_tracing_region_begin("Main-Thread");
+
   this->environment().printThreadSummary();
 
 #if defined(FIRESTARTER_BUILD_CUDA) || defined(FIRESTARTER_BUILD_HIP)
@@ -352,6 +360,7 @@ void Firestarter::mainThread() {
     int returnCode;
     if (EXIT_SUCCESS != (returnCode = this->initDumpRegisterWorker(
                              _dumpRegistersTimeDelta, _dumpRegistersOutpath))) {
+      firestarter_tracing_region_end("Main-Thread");
       std::exit(returnCode);
     }
   }
@@ -363,6 +372,7 @@ void Firestarter::mainThread() {
 #if defined(linux) || defined(__linux__)
   // check if optimization is selected
   if (_optimize) {
+    firestarter_tracing_region_begin("Main-Thread-Optimize");
     auto startTime = optimizer::History::getTime();
 
     Firestarter::_optimizer = std::make_unique<optimizer::OptimizerWorker>(
@@ -381,6 +391,10 @@ void Firestarter::mainThread() {
     firestarter::optimizer::History::printBest(_optimizationMetrics,
                                                payloadItems);
 
+
+    firestarter_tracing_region_end("Main-Thread-Optimize");
+    firestarter_tracing_region_end("Main-Thread");
+
     // stop all the load threads
     std::raise(SIGTERM);
   }
@@ -415,6 +429,8 @@ void Firestarter::mainThread() {
   if (_errorDetection) {
     this->printThreadErrorReport();
   }
+
+  firestarter_tracing_region_end("Main-Thread");
 }
 
 void Firestarter::setLoad(unsigned long long value) {
@@ -437,6 +453,11 @@ void Firestarter::sigalrmHandler(int signum) { (void)signum; }
 void Firestarter::sigtermHandler(int signum) {
   (void)signum;
 
+    if (Firestarter::loadVar == LOAD_LOW)
+      firestarter_tracing_region_end("WD_LOW");
+    if (Firestarter::loadVar == LOAD_HIGH)
+      firestarter_tracing_region_end("WD_HIGH");
+
   Firestarter::setLoad(LOAD_STOP);
   // exit loop
   // used in case of 0 < load < 100

src/firestarter/LoadWorker.cpp

@@ -23,18 +23,16 @@
 #include <firestarter/Firestarter.hpp>
 #include <firestarter/Logging/Log.hpp>
 
+extern "C" {
+#include <firestarter/Tracing/FIRESTARTER_Tracing.h>
+}
+
 #if defined(linux) || defined(__linux__)
 extern "C" {
 #include <firestarter/Measurement/Metric/IPCEstimate.h>
 }
 #endif
 
-#ifdef ENABLE_VTRACING
-#include <vt_user.h>
-#endif
-#ifdef ENABLE_SCOREP
-#include <SCOREP_User.h>
-#endif
 
 #include <cmath>
 #include <cstdlib>
@@ -350,33 +348,20 @@ void Firestarter::loadThreadWorker(std::shared_ptr<LoadWorkerData> td) {
       // will be terminated by watchdog
       for (;;) {
         // call high load function
-#ifdef ENABLE_VTRACING
-        VT_USER_START("HIGH_LOAD_FUNC");
-#endif
-#ifdef ENABLE_SCOREP
-        SCOREP_USER_REGION_BY_NAME_BEGIN("HIGH",
-                                         SCOREP_USER_REGION_TYPE_COMMON);
-#endif
+
+        firestarter_tracing_region_begin("High");
+
         td->iterations = td->config().payload().highLoadFunction(
             td->addrMem, td->addrHigh, td->iterations);
 
+        firestarter_tracing_region_end("High");
+        firestarter_tracing_region_begin("Low");
+
         // call low load function
-#ifdef ENABLE_VTRACING
-        VT_USER_END("HIGH_LOAD_FUNC");
-        VT_USER_START("LOW_LOAD_FUNC");
-#endif
-#ifdef ENABLE_SCOREP
-        SCOREP_USER_REGION_BY_NAME_END("HIGH");
-        SCOREP_USER_REGION_BY_NAME_BEGIN("LOW", SCOREP_USER_REGION_TYPE_COMMON);
-#endif
         td->config().payload().lowLoadFunction(td->addrHigh, td->period);
-#ifdef ENABLE_VTRACING
-        VT_USER_END("LOW_LOAD_FUNC");
-#endif
-#ifdef ENABLE_SCOREP
-        SCOREP_USER_REGION_BY_NAME_END("LOW");
-#endif
 
+        firestarter_tracing_region_end("Low");
+
         // terminate if master signals end of run and record stop timestamp
         if (*td->addrHigh == LOAD_STOP) {
           td->stopTsc = td->environment().topology().timestamp();