discopop-project · lukasrothenberger · Nov 15, 2023 · Oct 17, 2023 · Oct 17, 2023 · Oct 17, 2023
diff --git a/.github/workflows/tests/profiler.sh b/.github/workflows/tests/profiler.sh
@@ -23,7 +23,7 @@ function test_discopopPass {
   cp ${DISCOPOP_SRC}/scripts/dp-fmap .
   ./dp-fmap
   clang++ -g -c -O0 -S -emit-llvm -fno-discard-value-names "$1" -o out.ll || return 1
-  opt-11 -S -load=${DISCOPOP_INSTALL}/libi/LLVMDiscoPoP.so --DiscoPoP out.ll -o out_dp.ll --fm-path FileMapping.txt || return 1
+  opt-11 -S -load=${DISCOPOP_INSTALL}/libi/LLVMDiscoPoP.so --DiscoPoP out.ll -o out_dp.ll || return 1
   clang++ out_dp.ll -o out_prof -L${DISCOPOP_INSTALL}/rtlib -lDiscoPoP_RT -lpthread || return 1
   ./out_prof || return 1
 }

diff --git a/DiscoPoP/DiscoPoP.cpp b/DiscoPoP/DiscoPoP.cpp
@@ -112,6 +112,18 @@ bool DiscoPoP::doInitialization(Module &M) {
         errs() << "DiscoPoP | 190: init pass DiscoPoP \n";
     }
 
+    // prepare .discopop directory if not present
+      struct stat st1 = {0};
+      if (stat(".discopop", &st1) == -1){
+          mkdir(".discopop", 0777);
+      }
+      // prepare profiler directory if not present
+      struct stat st2 = {0};
+      if (stat(".discopop/profiler", &st2) == -1){
+          mkdir(".discopop/profiler", 0777);
+      }
+
+
 // CUGeneration
     {
         CUIDCounter = 0;
@@ -189,6 +201,8 @@ bool DiscoPoP::doFinalization(Module &M) {
     //CUGeneration
 
     // write the current count of CUs to a file to avoid duplicate CUs.
+    outCUIDCounter = new std::ofstream();
+    outCUIDCounter->open(".discopop/profiler/DP_CUIDCounter.txt", std::ios_base::out);
     if (outCUIDCounter && outCUIDCounter->is_open()) {
         *outCUIDCounter << CUIDCounter;
         outCUIDCounter->flush();
@@ -871,7 +885,7 @@ void DiscoPoP::fillStartEndLineNumbers(Node *root, LoopInfo &LI) {
 }
 
 void DiscoPoP::initializeCUIDCounter() {
-    std::string CUCounterFile = "DP_CUIDCounter.txt";
+    std::string CUCounterFile = ".discopop/profiler/DP_CUIDCounter.txt";
     if (dputil::fexists(CUCounterFile)) {
         std::fstream inCUIDCounter(CUCounterFile, std::ios_base::in);;
         inCUIDCounter >> CUIDCounter;
@@ -925,16 +939,17 @@ bool DiscoPoP::inlinedFunction(Function *F) {
 void DiscoPoP::instrument_function(llvm::Function *function, map <string, string> *trueVarNamesFromMetadataMap) {
 
     // get the corresponding file id
-    unsigned file_id = dp_reduction_get_file_id(function);
-    if (file_id == 0) {
+    int32_t tmp_file_id;
+    determineFileID(*function, tmp_file_id);
+    if (tmp_file_id == 0) {
         return;
     }
 
     llvm::LoopInfo &loop_info = getAnalysis<llvm::LoopInfoWrapperPass>(*function).getLoopInfo();
 
     for (auto loop_it = loop_info.begin(); loop_it != loop_info.end();
          ++loop_it) {
-        instrument_loop(*function, file_id, *loop_it, loop_info, trueVarNamesFromMetadataMap);
+        instrument_loop(*function, tmp_file_id, *loop_it, loop_info, trueVarNamesFromMetadataMap);
     }
 }
 
@@ -1811,7 +1826,7 @@ void DiscoPoP::dp_reduction_insert_functions() {
 
     // insert function calls to monitor loop iterations
     std::ofstream loop_metadata_file;
-    loop_metadata_file.open("loop_meta.txt");
+    loop_metadata_file.open(".discopop/profiler/loop_meta.txt");
     int loop_id = 1;
     llvm::Type* loop_incr_fn_arg_type = llvm::Type::getInt32Ty(*ctx_);
     llvm::ArrayRef<llvm::Type*> loop_incr_fn_args(loop_incr_fn_arg_type);
@@ -1999,16 +2014,18 @@ bool DiscoPoP::runOnModule(Module &M) {
     ctx_ = &module_->getContext();
 
     reduction_file = new std::ofstream();
-    reduction_file->open("reduction.txt", std::ios_base::app);
+    reduction_file->open(".discopop/profiler/reduction.txt", std::ios_base::app);
 
     loop_counter_file = new std::ofstream();
-    loop_counter_file->open("loop_counter_output.txt", std::ios_base::app);
+    loop_counter_file->open(".discopop/profiler/loop_counter_output.txt", std::ios_base::app);
 
+    /*
     bool success = dp_reduction_init_util(FileMappingPath);
     if (!success) {
         llvm::errs() << "could not find the FileMapping file: " << FileMappingPath << "\n";
         return false;
     }
+    */
     instrument_module(&M, &trueVarNamesFromMetadataMap);
 
     dp_reduction_insert_functions();
@@ -2435,7 +2452,7 @@ bool DiscoPoP::runOnFunction(Function &F) {
         // Report statically identified dependencies
 
         staticDependencyFile = new std::ofstream();
-        staticDependencyFile->open("static_dependencies.txt", std::ios_base::app);
+        staticDependencyFile->open(".discopop/profiler/static_dependencies.txt", std::ios_base::app);
 
         for (auto pair: conditionalBBDepMap) {
                 for (auto s: pair.second) {
@@ -2845,13 +2862,10 @@ string DiscoPoP::xmlEscape(string data) {
 
 void DiscoPoP::secureStream() {
     outOriginalVariables = new std::ofstream();
-    outOriginalVariables->open("OriginalVariables.txt", std::ios_base::app);
+    outOriginalVariables->open(".discopop/profiler/OriginalVariables.txt", std::ios_base::app);
 
     outCUs = new std::ofstream();
-    outCUs->open("Data.xml", std::ios_base::app);
-
-    outCUIDCounter = new std::ofstream();
-    outCUIDCounter->open("DP_CUIDCounter.txt", std::ios_base::out);
+    outCUs->open(".discopop/profiler/Data.xml", std::ios_base::app);
 }
 
 string DiscoPoP::getLineNumbersString(set<int> LineNumbers) {
@@ -3030,10 +3044,6 @@ void DiscoPoP::closeOutputFiles() {
         outOriginalVariables->close();
     }
 
-    if(outCUIDCounter != NULL && outCUIDCounter->is_open()){
-        outCUIDCounter->flush();
-        outCUIDCounter->close();
-    }
     // delete outCUs;
 }
 /************** End of output functions *******************/

diff --git a/DiscoPoP/DiscoPoP.hpp b/DiscoPoP/DiscoPoP.hpp
@@ -57,6 +57,9 @@
 #include <string.h>
 #include <utility>
 
+#include <sys/stat.h>
+#include <sys/types.h>
+
 #define DP_DEBUG false
 
 using namespace llvm;

diff --git a/README.md b/README.md
@@ -30,6 +30,7 @@ Detailed information about each execution step, the setup as well as the functio
 ### Quick Links
 - [Quickstart guide - GUI](https://discopop-project.github.io/discopop/Quickstart)
 - [Quickstart guide - CMake projects](https://discopop-project.github.io/discopop/Quickstart_CMake)
+- [Visual Studio Code Extension](https://marketplace.visualstudio.com/items?itemName=TUDarmstadt-LaboratoryforParallelProgramming.discopop)
 
 ## Example
 DiscoPoP creates parallelization suggestions for sequential source code.
@@ -98,7 +99,8 @@ Please refer to the [parallel patterns](https://discopop-project.github.io/disco
         return 0;
     }
 
-
+### Convenience
+For a convenient set, configuration, management, application, and browsing of identified parallelization suggestions, please consider using our [Visual Studio Code Extension](https://marketplace.visualstudio.com/items?itemName=TUDarmstadt-LaboratoryforParallelProgramming.discopop).
 
 
 

diff --git a/discopop_explorer/PETGraphX.py b/discopop_explorer/PETGraphX.py
@@ -701,8 +701,6 @@ def map_static_and_dynamic_dependencies(self):
                             mem_reg_mappings[d2.memory_region].add(d1.memory_region)
         print("Done.")
 
-        print("\t\t\tMappings: ", mem_reg_mappings)
-
         print("\t\tInstantiating static dependencies...", end=" ")
         # create copies of static dependency edges for all dynamic mappings
         for node_id in [n.id for n in self.all_nodes(CUNode)]:
@@ -719,7 +717,6 @@ def map_static_and_dynamic_dependencies(self):
                             edge_data = copy.deepcopy(d)
                             edge_data.memory_region = dynamic_mapping
                             self.g.add_edge(s, t, data=edge_data)
-                            print("Added Instance: ", s, t, edge_data)
 
         print("Done.")
 

diff --git a/discopop_explorer/__main__.py b/discopop_explorer/__main__.py
@@ -25,20 +25,20 @@ def parse_args() -> ExplorerArguments:
     # fmt: off
     parser.add_argument(
         "--path", type=str, default="./",
-        help="Directory with input data")
+        help="Path to the .discopop directory to be analyzed")
     parser.add_argument(
-        "--cu-xml", type=str, default="Data.xml",
+        "--cu-xml", type=str, default="profiler/Data.xml",
         help="CU node xml file")
     parser.add_argument(
-        "--dep-file", type=str, default="dp_run_dep.txt",
+        "--dep-file", type=str, default="profiler/dynamic_dependencies.txt",
         help="Dependencies text file"
     )
     parser.add_argument(
-        "--loop-counter", type=str, default="loop_counter_output.txt",
+        "--loop-counter", type=str, default="profiler/loop_counter_output.txt",
         help="Loop counter data"
     )
     parser.add_argument(
-        "--reduction", type=str, default="reduction.txt",
+        "--reduction", type=str, default="profiler/reduction.txt",
         help="Reduction variables file"
     )
     parser.add_argument(
@@ -53,18 +53,18 @@ def parse_args() -> ExplorerArguments:
     )
     # flags related to output and formatting:
     parser.add_argument(
-        "--json", type=str, nargs="?", default=None, const="patterns.json",
+        "--json", type=str, nargs="?", default="explorer/patterns.json",
         help="Json output")
     parser.add_argument(
-        "--profiling", type=str, nargs="?", default=None, const="profiling_stats.txt",
+        "--profiling", type=str, nargs="?", default=None, const="explorer/profiling_stats.txt",
         help="Enable profiling. If a path is given, the profiling stats are written to the given file, otherwise to profiling_stats.txt",
     )
     parser.add_argument(
-        "--dump-pet", type=str, nargs="?", default=None, const="pet_dump.json",
+        "--dump-pet", type=str, nargs="?", default=None, const="explorer/pet_dump.json",
         help="Dump PET Graph to JSON file. If a path is given, the PET Graph is written to the given file, otherwise to pet_dump.json",
     )
     parser.add_argument(
-        "--dump-detection-result", type=str, nargs="?", default=None, const="detection_result_dump.json",
+        "--dump-detection-result", type=str, nargs="?", default="explorer/detection_result_dump.json",
         help="Dump DetectionResult object to JSON file. If a path is given, the DetectionResult object is written to the given file, otherwise to detection_result_dump.json. Contents are equivalent to the json output. NOTE: This dump contains a dump of the PET Graph!",
     )
 

diff --git a/discopop_explorer/discopop_explorer.py b/discopop_explorer/discopop_explorer.py
@@ -18,7 +18,8 @@
 import pstats2  # type:ignore
 from pluginbase import PluginBase  # type:ignore
 
-from discopop_library.PathManagement.PathManagement import get_path
+from discopop_library.LineMapping.initialize import initialize_line_mapping
+from discopop_library.PathManagement.PathManagement import get_path, load_file_mapping
 from discopop_library.discopop_optimizer.Microbench.ExtrapInterpolatedMicrobench import (
     ExtrapInterpolatedMicrobench,
 )
@@ -141,6 +142,14 @@ def __run(
 
 def run(arguments: ExplorerArguments):
     """Run the discopop_explorer with the given arguments"""
+    # create explorer directory if not already present
+    if not os.path.exists(os.path.join(arguments.project_path, "explorer")):
+        os.mkdir(os.path.join(arguments.project_path, "explorer"))
+    # create file to store next free pattern ids if not already present
+    if not os.path.exists("next_free_pattern_id.txt"):
+        with open("next_free_pattern_id.txt", "w") as f:
+            f.write(str(0))
+
     if arguments.enable_profiling_dump_file is not None:
         profile = cProfile.Profile()
         profile.enable()
@@ -207,6 +216,9 @@ def run(arguments: ExplorerArguments):
             stats = pstats2.Stats(profile, stream=f).sort_stats("time").reverse_order()
             stats.print_stats()
 
+    # initialize the line_mapping.json
+    initialize_line_mapping(load_file_mapping(arguments.file_mapping_file), arguments.project_path)
+
     print("Time taken for pattern detection: {0}".format(end - start))
 
     # demonstration of Microbenchmark possibilities

diff --git a/discopop_explorer/pattern_detectors/PatternInfo.py b/discopop_explorer/pattern_detectors/PatternInfo.py
@@ -6,6 +6,7 @@
 # the 3-Clause BSD License.  See the LICENSE file in the package base
 # directory for details.
 import json
+import os
 from typing import Optional
 
 from ..PETGraphX import LoopNode, Node, NodeID, LineID, PETGraphX
@@ -15,6 +16,7 @@
 class PatternInfo(object):
     """Base class for pattern detection info"""
 
+    pattern_id: int
     _node: Node
     node_id: NodeID
     start_line: LineID
@@ -31,6 +33,19 @@ def __init__(self, node: Node):
         """
         :param node: node, where pipeline was detected
         """
+        # use blocking file i/o to synchronize threads
+        with open(os.path.join(os.getcwd(), "next_free_pattern_id.txt"), "r+") as f:
+            lines = f.readlines()
+            f.truncate(0)
+            f.seek(0)
+            if len(lines) == 0:
+                self.pattern_id = 0
+                f.write(str(0))
+            else:
+                for line in lines:
+                    line = line.replace("\n", "").replace("\x00", "")
+                    self.pattern_id = int(line)
+                    f.write(str(self.pattern_id + 1))
         self._node = node
         self.node_id = node.id
         self.start_line = node.start_position()

diff --git a/discopop_explorer/pattern_detectors/combined_gpu_patterns/classes/EntryPoint.py b/discopop_explorer/pattern_detectors/combined_gpu_patterns/classes/EntryPoint.py
@@ -133,7 +133,7 @@ def get_as_metadata(self, pet: PETGraphX, project_folder_path: str):
 
         # get size of memory region
         memory_region_sizes = get_sizes_of_memory_regions(
-            self.memory_regions, os.path.join(project_folder_path, "memory_regions.txt")
+            self.memory_regions, os.path.join(project_folder_path, "profiler/memory_regions.txt")
         )
         if len(memory_region_sizes) > 0:
             max_mem_reg_size = max(memory_region_sizes.values())

diff --git a/discopop_explorer/pattern_detectors/combined_gpu_patterns/classes/ExitPoint.py b/discopop_explorer/pattern_detectors/combined_gpu_patterns/classes/ExitPoint.py
@@ -131,7 +131,7 @@ def get_as_metadata(self, pet: PETGraphX, project_folder_path: str):
 
         # get size of memory region
         memory_region_sizes = get_sizes_of_memory_regions(
-            self.memory_regions, os.path.join(project_folder_path, "memory_regions.txt")
+            self.memory_regions, os.path.join(project_folder_path, "profiler/memory_regions.txt")
         )
         if len(memory_region_sizes) > 0:
             max_mem_reg_size = max(memory_region_sizes.values())

diff --git a/discopop_explorer/pattern_detectors/combined_gpu_patterns/classes/Update.py b/discopop_explorer/pattern_detectors/combined_gpu_patterns/classes/Update.py
@@ -180,7 +180,7 @@ def get_as_metadata_using_variable_names(self, pet: PETGraphX, project_folder_pa
 
         # get size of memory region
         memory_region_sizes = get_sizes_of_memory_regions(
-            self.memory_regions, os.path.join(project_folder_path, "memory_regions.txt")
+            self.memory_regions, os.path.join(project_folder_path, "profiler/memory_regions.txt")
         )
         if len(memory_region_sizes) > 0:
             max_mem_reg_size = max(memory_region_sizes.values())
@@ -238,7 +238,7 @@ def get_as_metadata_using_variable_names_and_memory_regions(self, pet: PETGraphX
 
         # get size of memory region
         memory_region_sizes = get_sizes_of_memory_regions(
-            self.memory_regions, os.path.join(project_folder_path, "memory_regions.txt")
+            self.memory_regions, os.path.join(project_folder_path, "profiler/memory_regions.txt")
         )
         if len(memory_region_sizes) > 0:
             max_mem_reg_size = max(memory_region_sizes.values())

diff --git a/discopop_explorer/pattern_detectors/simple_gpu_patterns/GPULoop.py b/discopop_explorer/pattern_detectors/simple_gpu_patterns/GPULoop.py
@@ -291,7 +291,7 @@ def __get_constructs(
 
                 # get size of memory region
                 memory_region_sizes = get_sizes_of_memory_regions(
-                    memory_regions, os.path.join(project_folder_path, "memory_regions.txt")
+                    memory_regions, os.path.join(project_folder_path, "profiler/memory_regions.txt")
                 )
                 if len(memory_region_sizes) > 0:
                     max_mem_reg_size = max(memory_region_sizes.values())
@@ -318,7 +318,7 @@ def __get_constructs(
 
                 # get size of memory region
                 memory_region_sizes = get_sizes_of_memory_regions(
-                    memory_regions, os.path.join(project_folder_path, "memory_regions.txt")
+                    memory_regions, os.path.join(project_folder_path, "profiler/memory_regions.txt")
                 )
                 if len(memory_region_sizes) > 0:
                     max_mem_reg_size = max(memory_region_sizes.values())
@@ -343,7 +343,7 @@ def __get_constructs(
 
                 # get size of memory region
                 memory_region_sizes = get_sizes_of_memory_regions(
-                    memory_regions, os.path.join(project_folder_path, "memory_regions.txt")
+                    memory_regions, os.path.join(project_folder_path, "profiler/memory_regions.txt")
                 )
                 if len(memory_region_sizes) > 0:
                     max_mem_reg_size = max(memory_region_sizes.values())
@@ -368,7 +368,7 @@ def __get_constructs(
 
                 # get size of memory region
                 memory_region_sizes = get_sizes_of_memory_regions(
-                    memory_regions, os.path.join(project_folder_path, "memory_regions.txt")
+                    memory_regions, os.path.join(project_folder_path, "profiler/memory_regions.txt")
                 )
                 if len(memory_region_sizes) > 0:
                     max_mem_reg_size = max(memory_region_sizes.values())
@@ -429,11 +429,11 @@ def __get_constructs(
         self.declared_global_variables.update(used_global_vars)
         for global_var in used_global_vars:
             constructs.append(
-                omp_construct_dict("#pragma omp declare target  // " + global_var.name, global_var.defLine, [])
+                omp_construct_dict("#pragma omp declare target  // " + str(global_var.name), global_var.defLine, [])
             )
             constructs.append(
                 omp_construct_dict(
-                    "#pragma omp end declare target  // " + global_var.name,
+                    "#pragma omp end declare target  // " + str(global_var.name),
                     global_var.defLine,
                     [],
                     positioning=OmpConstructPositioning.AFTER_LINE,