From 879bfdf8d8516aee0fed26a78dacb5032d1a9fce Mon Sep 17 00:00:00 2001 From: sbaldu Date: Thu, 18 Jan 2024 10:11:08 +0100 Subject: [PATCH 01/16] Write CMake file for building --- CMakeLists.txt | 119 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 119 insertions(+) create mode 100644 CMakeLists.txt diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 00000000..e4dafd25 --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,119 @@ +cmake_minimum_required(VERSION 3.16.0) +project(CLUEstering LANGUAGES CXX) + +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_CXX_EXTENSIONS OFF) + +set(CMAKE_CXX_FLAGS "-Wall -Wextra -g -O2") + +# include alpaka extern subfolder +include_directories(extern/alpaka/include) +# include pybind11 extern subfolder +add_subdirectory(extern/pybind11) + +include(FetchContent) +FetchContent_Declare( + boost + URL https://boostorg.jfrog.io/artifactory/main/release/1.76.0/source/boost_1_76_0.tar.gz +) + +FetchContent_GetProperties(boost) +if (NOT boost_POPULATED) + FetchContent_Populate(boost) +endif() + +# compile convolutional kernel module +pybind11_add_module(CLUE_Convolutional_Kernels SHARED + ./CLUEstering/alpaka/BindingModules/binding_kernels.cc) +target_link_libraries(CLUE_Convolutional_Kernels PRIVATE ${Boost_LIBRARIES}) +target_include_directories(CLUE_Convolutional_Kernels PRIVATE ./build/_deps/boost-src) +target_compile_options( + CLUE_Convolutional_Kernels + PRIVATE -DALPAKA_HOST_ONLY -DALPAKA_ACC_CPU_B_SEQ_T_SEQ_PRESENT + -DALPAKA_ACC_CPU_B_SEQ_T_SEQ_ENABLED + -DALPAKA_ACC_CPU_B_SEQ_T_SEQ_SYNC_BACKEND) +set_target_properties(CLUE_Convolutional_Kernels + PROPERTIES LIBRARY_OUTPUT_DIRECTORY ../CLUEstering/) + +# compile cpu serial module +pybind11_add_module(CLUE_CPU_Serial SHARED + ./CLUEstering/alpaka/BindingModules/binding_cpu.cc) +target_link_libraries(CLUE_CPU_Serial PRIVATE ${Boost_LIBRARIES}) +target_include_directories(CLUE_CPU_Serial PRIVATE ./build/_deps/boost-src) +target_compile_options( + CLUE_CPU_Serial + PRIVATE -DALPAKA_HOST_ONLY -DALPAKA_ACC_CPU_B_SEQ_T_SEQ_PRESENT + -DALPAKA_ACC_CPU_B_SEQ_T_SEQ_ENABLED + -DALPAKA_ACC_CPU_B_SEQ_T_SEQ_SYNC_BACKEND) +set_target_properties(CLUE_CPU_Serial PROPERTIES LIBRARY_OUTPUT_DIRECTORY + ../CLUEstering/) + +find_package(TBB) + +if (TBB_FOUND) + pybind11_add_module(CLUE_CPU_TBB SHARED + ./CLUEstering/alpaka/BindingModules/binding_cpu_tbb.cc) +target_link_libraries(CLUE_CPU_TBB PRIVATE ${Boost_LIBRARIES}) +target_include_directories(CLUE_CPU_TBB PRIVATE ./build/_deps/boost-src) + target_compile_options( + CLUE_CPU_TBB + PRIVATE -ltbb -DALPAKA_ACC_CPU_B_TBB_T_SEQ_PRESENT + -DALPAKA_ACC_CPU_B_TBB_T_SEQ_ENABLED + -DALPAKA_ACC_CPU_B_TBB_T_SEQ_ASYNC_BACKEND) + target_link_libraries(CLUE_CPU_TBB PRIVATE TBB::tbb) + set_target_properties(CLUE_CPU_TBB PROPERTIES LIBRARY_OUTPUT_DIRECTORY + ../CLUEstering/) +endif() + +# compile cpu tbb module + +# check if CUDA is available +include(CheckLanguage) +check_language(CUDA) +if (CMAKE_CUDA_COMPILER) +# enable CUDA +enable_language(CUDA) +# require cuda to compile the test file +find_package(CUDA) + +# set the CUDA standard +if(NOT DEFINED CMAKE_CUDA_STANDARD) + set(CMAKE_CUDA_STANDARD 17) + set(CMAKE_CUDA_STANDARD_REQUIRED ON) +endif() + +set(CMAKE_CUDA_FLAGS "-Wall -Wextra -g -O2") + +# compile nvidia gpu async module +pybind11_add_module(CLUE_GPU_CUDA SHARED + ./CLUEstering/alpaka/BindingModules/binding_gpu_cuda.cu) +target_link_libraries(CLUE_Convolutional_Kernels PRIVATE ${Boost_LIBRARIES}) +target_include_directories(CLUE_Convolutional_Kernels PRIVATE ./build/_deps/boost-src) +# set the cuda architectures +set_target_properties(CLUE_GPU_CUDA PROPERTIES CUDA_ARCHITECTURES + "50;60;61;62;70") +# alpaka compilation flags +target_compile_options( + CLUE_GPU_CUDA + PRIVATE -DALPAKA_ACC_GPU_CUDA_PRESENT -DALPAKA_ACC_GPU_CUDA_ENABLED + -DALPAKA_ACC_GPU_CUDA_ASYNC_BACKEND) +# nvcc compilation flags +target_compile_options( + CLUE_GPU_CUDA PRIVATE --expt-relaxed-constexpr -gencode + arch=compute_61,code=[sm_61,compute_61]) +set_target_properties(CLUE_GPU_CUDA PROPERTIES LIBRARY_OUTPUT_DIRECTORY + ../CLUEstering/) +endif() + +## need to work on the hip module +# find_package(hip REQUIRED) + +# # compile amd gpu module +# pybind11_add_module(CLUE_GPU_HIP SHARED +# ./CLUEstering/alpaka/BindingModules/binding_gpu_hip.cc) +# target_compile_options(CLUE_GPU_HIP PRIVATE -DALPAKA_ACC_GPU_HIP_PRESENT +# -DALPAKA_ACC_GPU_HIP_ENABLED +# -DALPAKA_ACC_GPU_HIP_ASYNC_BACKEND) +# set_target_properties(CLUE_GPU_HIP PROPERTIES LIBRARY_OUTPUT_DIRECTORY +# ../CLUEstering/) From cbaab19b3a1cc2d001bf400bbed2f89e94740b0c Mon Sep 17 00:00:00 2001 From: sbaldu Date: Thu, 18 Jan 2024 10:11:19 +0100 Subject: [PATCH 02/16] Update `setup.py` to use CMake when building the wheel --- setup.py | 46 +++++++++++++++++++++------------------------- 1 file changed, 21 insertions(+), 25 deletions(-) diff --git a/setup.py b/setup.py index 1f38963b..949e8423 100644 --- a/setup.py +++ b/setup.py @@ -1,18 +1,14 @@ + +import os from pathlib import Path -from setuptools import setup, find_packages -from pybind11.setup_helpers import Pybind11Extension +from setuptools import Extension, setup, find_packages -__version__ = "1.4.0" +__version__ = "2.0.0" this_directory = Path(__file__).parent -long_description = (this_directory / 'README.md').read_text() +long_description = (this_directory/'README.md').read_text() -ext_modules = [ - Pybind11Extension( - "CLUEsteringCPP", - ['CLUEstering/binding.cc'], - include_dirs=['CLUEstering/include/'] - ), -] +print("Now we build with cmake") +os.system("cmake -B build && make -C build") setup( name="CLUEstering", @@ -21,18 +17,18 @@ author_email="simone.balducci00@gmail.com", description='''A library that generalizes the original 2-dimensional CLUE algorithm made at CERN.''', - long_description=long_description, - long_description_content_type='text/markdown', - packages=find_packages(), - install_requires=['scikit-learn', 'numpy', 'matplotlib', 'pandas'], - ext_modules=ext_modules, - keywords=['Python', 'Clustering', 'Binding'], - python_requires='>=3.7', - classifiers=[ - 'Intended Audience :: Developers', - 'Programming Language :: Python :: 3', - 'Operating System :: Unix', - 'Operating System :: MacOS :: MacOS X', - 'Operating System :: Microsoft :: Windows', - ] + long_description=long_description, + long_description_content_type='text/markdown', + packages=find_packages(), + install_requires=['scikit-learn','numpy','matplotlib','pandas'], + package_data={'': ['*.so']}, + keywords=['Python','Clustering','Binding'], + python_requires='>=3.7', + classifiers=[ + 'Intended Audience :: Developers', + 'Programming Language :: Python :: 3', + 'Operating System :: Unix', + 'Operating System :: MacOS :: MacOS X', + 'Operating System :: Microsoft :: Windows', + ] ) From a0f84cb4793994564e95599a778fc46e48b6eba6 Mon Sep 17 00:00:00 2001 From: sbaldu Date: Fri, 19 Jan 2024 15:24:46 +0100 Subject: [PATCH 03/16] Only fetch boost if not found --- CMakeLists.txt | 34 +++++++++++++++++++++------------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index e4dafd25..e4773caa 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -12,22 +12,29 @@ include_directories(extern/alpaka/include) # include pybind11 extern subfolder add_subdirectory(extern/pybind11) -include(FetchContent) -FetchContent_Declare( - boost - URL https://boostorg.jfrog.io/artifactory/main/release/1.76.0/source/boost_1_76_0.tar.gz -) - -FetchContent_GetProperties(boost) -if (NOT boost_POPULATED) +find_package(Boost 1.75.0) + +if (NOT Boost_FOUND) + include(FetchContent) + FetchContent_Declare( + boost + URL https://boostorg.jfrog.io/artifactory/main/release/1.76.0/source/boost_1_76_0.tar.gz + ) + + FetchContent_GetProperties(boost) + if (NOT boost_POPULATED) FetchContent_Populate(boost) + endif() + set(Boost_PATH ./build/_deps/boost-src) +else() + set(Boost_PATH ${Boost_INCLUDE_DIRS}) endif() # compile convolutional kernel module pybind11_add_module(CLUE_Convolutional_Kernels SHARED ./CLUEstering/alpaka/BindingModules/binding_kernels.cc) target_link_libraries(CLUE_Convolutional_Kernels PRIVATE ${Boost_LIBRARIES}) -target_include_directories(CLUE_Convolutional_Kernels PRIVATE ./build/_deps/boost-src) +target_include_directories(CLUE_Convolutional_Kernels PRIVATE ${Boost_PATH}) target_compile_options( CLUE_Convolutional_Kernels PRIVATE -DALPAKA_HOST_ONLY -DALPAKA_ACC_CPU_B_SEQ_T_SEQ_PRESENT @@ -39,8 +46,8 @@ set_target_properties(CLUE_Convolutional_Kernels # compile cpu serial module pybind11_add_module(CLUE_CPU_Serial SHARED ./CLUEstering/alpaka/BindingModules/binding_cpu.cc) -target_link_libraries(CLUE_CPU_Serial PRIVATE ${Boost_LIBRARIES}) -target_include_directories(CLUE_CPU_Serial PRIVATE ./build/_deps/boost-src) + target_link_libraries(CLUE_CPU_Serial PRIVATE ${Boost_LIBRARIES}) +target_include_directories(CLUE_CPU_Serial PRIVATE ${Boost_PATH}) target_compile_options( CLUE_CPU_Serial PRIVATE -DALPAKA_HOST_ONLY -DALPAKA_ACC_CPU_B_SEQ_T_SEQ_PRESENT @@ -52,10 +59,11 @@ set_target_properties(CLUE_CPU_Serial PROPERTIES LIBRARY_OUTPUT_DIRECTORY find_package(TBB) if (TBB_FOUND) + # compile cpu tbb module pybind11_add_module(CLUE_CPU_TBB SHARED ./CLUEstering/alpaka/BindingModules/binding_cpu_tbb.cc) -target_link_libraries(CLUE_CPU_TBB PRIVATE ${Boost_LIBRARIES}) -target_include_directories(CLUE_CPU_TBB PRIVATE ./build/_deps/boost-src) + target_link_libraries(CLUE_CPU_TBB PRIVATE ${Boost_LIBRARIES}) + target_include_directories(CLUE_CPU_TBB PRIVATE ${Boost_PATH}) target_compile_options( CLUE_CPU_TBB PRIVATE -ltbb -DALPAKA_ACC_CPU_B_TBB_T_SEQ_PRESENT From bd3f6cffc3150806ba0955a5e518254c9f9651b5 Mon Sep 17 00:00:00 2001 From: sbaldu Date: Fri, 19 Jan 2024 15:29:49 +0100 Subject: [PATCH 04/16] Fix conditional compilation of CUDA --- CMakeLists.txt | 75 ++++++++++++++++++++------------------------------ 1 file changed, 30 insertions(+), 45 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index e4773caa..f154de75 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -74,54 +74,39 @@ if (TBB_FOUND) ../CLUEstering/) endif() -# compile cpu tbb module - -# check if CUDA is available include(CheckLanguage) +# check if CUDA is available check_language(CUDA) if (CMAKE_CUDA_COMPILER) -# enable CUDA -enable_language(CUDA) -# require cuda to compile the test file -find_package(CUDA) - -# set the CUDA standard -if(NOT DEFINED CMAKE_CUDA_STANDARD) - set(CMAKE_CUDA_STANDARD 17) - set(CMAKE_CUDA_STANDARD_REQUIRED ON) -endif() - -set(CMAKE_CUDA_FLAGS "-Wall -Wextra -g -O2") + # enable CUDA + enable_language(CUDA) + set(CMAKE_CUDA_HOST_COMPILER ${CMAKE_CUDA_COMPILER}) + + # set the CUDA standard + if(NOT DEFINED CMAKE_CUDA_STANDARD) + set(CMAKE_CUDA_STANDARD 17) + set(CMAKE_CUDA_STANDARD_REQUIRED ON) + endif() -# compile nvidia gpu async module -pybind11_add_module(CLUE_GPU_CUDA SHARED - ./CLUEstering/alpaka/BindingModules/binding_gpu_cuda.cu) -target_link_libraries(CLUE_Convolutional_Kernels PRIVATE ${Boost_LIBRARIES}) -target_include_directories(CLUE_Convolutional_Kernels PRIVATE ./build/_deps/boost-src) -# set the cuda architectures -set_target_properties(CLUE_GPU_CUDA PROPERTIES CUDA_ARCHITECTURES - "50;60;61;62;70") -# alpaka compilation flags -target_compile_options( - CLUE_GPU_CUDA - PRIVATE -DALPAKA_ACC_GPU_CUDA_PRESENT -DALPAKA_ACC_GPU_CUDA_ENABLED - -DALPAKA_ACC_GPU_CUDA_ASYNC_BACKEND) -# nvcc compilation flags -target_compile_options( - CLUE_GPU_CUDA PRIVATE --expt-relaxed-constexpr -gencode + set(CMAKE_CUDA_FLAGS "-Wall -Wextra -g -O2") + + # compile nvidia gpu async module + pybind11_add_module(CLUE_GPU_CUDA SHARED + ./CLUEstering/alpaka/BindingModules/binding_gpu_cuda.cu) + target_link_libraries(CLUE_Convolutional_Kernels PRIVATE ${Boost_LIBRARIES}) + target_include_directories(CLUE_Convolutional_Kernels PRIVATE ${Boost_PATH}) + # set the cuda architectures + set_target_properties(CLUE_GPU_CUDA PROPERTIES CUDA_ARCHITECTURES + "50;60;61;62;70") + # alpaka compilation flags + target_compile_options( + CLUE_GPU_CUDA + PRIVATE -DALPAKA_ACC_GPU_CUDA_PRESENT -DALPAKA_ACC_GPU_CUDA_ENABLED + -DALPAKA_ACC_GPU_CUDA_ASYNC_BACKEND) + # nvcc compilation flags + target_compile_options( + CLUE_GPU_CUDA PRIVATE --expt-relaxed-constexpr -gencode arch=compute_61,code=[sm_61,compute_61]) -set_target_properties(CLUE_GPU_CUDA PROPERTIES LIBRARY_OUTPUT_DIRECTORY - ../CLUEstering/) + set_target_properties(CLUE_GPU_CUDA PROPERTIES LIBRARY_OUTPUT_DIRECTORY + ../CLUEstering/) endif() - -## need to work on the hip module -# find_package(hip REQUIRED) - -# # compile amd gpu module -# pybind11_add_module(CLUE_GPU_HIP SHARED -# ./CLUEstering/alpaka/BindingModules/binding_gpu_hip.cc) -# target_compile_options(CLUE_GPU_HIP PRIVATE -DALPAKA_ACC_GPU_HIP_PRESENT -# -DALPAKA_ACC_GPU_HIP_ENABLED -# -DALPAKA_ACC_GPU_HIP_ASYNC_BACKEND) -# set_target_properties(CLUE_GPU_HIP PROPERTIES LIBRARY_OUTPUT_DIRECTORY -# ../CLUEstering/) From a9fdab316de18e5a2bf0f0d6881c7bc21583db89 Mon Sep 17 00:00:00 2001 From: sbaldu Date: Fri, 19 Jan 2024 17:02:45 +0100 Subject: [PATCH 05/16] Add compilation for hip in cmake --- CMakeLists.txt | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index f154de75..125dc95a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -46,7 +46,7 @@ set_target_properties(CLUE_Convolutional_Kernels # compile cpu serial module pybind11_add_module(CLUE_CPU_Serial SHARED ./CLUEstering/alpaka/BindingModules/binding_cpu.cc) - target_link_libraries(CLUE_CPU_Serial PRIVATE ${Boost_LIBRARIES}) +target_link_libraries(CLUE_CPU_Serial PRIVATE ${Boost_LIBRARIES}) target_include_directories(CLUE_CPU_Serial PRIVATE ${Boost_PATH}) target_compile_options( CLUE_CPU_Serial @@ -110,3 +110,30 @@ if (CMAKE_CUDA_COMPILER) set_target_properties(CLUE_GPU_CUDA PROPERTIES LIBRARY_OUTPUT_DIRECTORY ../CLUEstering/) endif() + +# check if HIP is available +check_language(HIP) +if (CMAKE_HIP_COMPILER) + # enable HIP + enable_language(HIP) + set(CMAKE_HIP_HOST_COMPILER ${CMAKE_HIP_COMPILER}) + + # look for the hip package folder + find_package(hip) + message("${hip_INCLUDE_DIRS}") + message("${CMAKE_HIP_COMPILER}") + + set(CMAKE_CXX_COMPILER "hipcc") + pybind11_add_module(CLUE_GPU_HIP SHARED + ./CLUEstering/alpaka/BindingModules/binding_gpu_hip.cc) + target_link_libraries(CLUE_Convolutional_Kernels PRIVATE ${Boost_LIBRARIES}) + target_include_directories(CLUE_Convolutional_Kernels PRIVATE ${Boost_PATH}) + target_compile_options(CLUE_GPU_HIP PRIVATE -DALPAKA_ACC_GPU_HIP_PRESENT + -DALPAKA_ACC_GPU_HIP_ENABLED + -DALPAKA_ACC_GPU_HIP_ASYNC_BACKEND) + target_include_directories(CLUE_GPU_HIP PRIVATE ${hip_INCLUDE_DIRS}) + target_include_directories(CLUE_GPU_HIP PRIVATE ${hip_INCLUDE_DIRS}/../hiprand/include) + target_include_directories(CLUE_GPU_HIP PRIVATE ${hip_INCLUDE_DIRS}/../rocrand/include) + set_target_properties(CLUE_GPU_HIP PROPERTIES LIBRARY_OUTPUT_DIRECTORY + ../CLUEstering/) +endif() From bd6d36660817cd31285e1b94fa376c529a978a7b Mon Sep 17 00:00:00 2001 From: sbaldu Date: Tue, 23 Jan 2024 12:03:26 +0100 Subject: [PATCH 06/16] Substitute old `CLUEstering.py` file --- CLUEstering/CLUEstering.py | 293 ++++++++++++++++++++++--------------- 1 file changed, 176 insertions(+), 117 deletions(-) diff --git a/CLUEstering/CLUEstering.py b/CLUEstering/CLUEstering.py index 5cfc018d..19cdfbbe 100644 --- a/CLUEstering/CLUEstering.py +++ b/CLUEstering/CLUEstering.py @@ -3,6 +3,7 @@ """ from dataclasses import dataclass +from glob import glob import random as rnd from math import sqrt import time @@ -13,10 +14,22 @@ import pandas as pd from sklearn.datasets import make_blobs from sklearn.preprocessing import StandardScaler -import CLUEsteringCPP as Algo - - -def test_blobs(n_samples: int, n_dim: int, n_blobs: int = 4, mean: float = 0, +import CLUE_Convolutional_Kernels as clue_kernels +import CLUE_CPU_Serial as cpu_serial +tbb_found = False +if len(glob('./CLUE_CPU_TBB*.so')) != 0: + import CLUE_CPU_TBB as cpu_tbb + tbb_found = True +cuda_found = False +if len(glob('./CLUE_GPU_CUDA*.so')) != 0: + import CLUE_GPU_CUDA as gpu_cuda + cuda_found = True +hip_found = False +if len(glob('./CLUE_GPU_HIP*.so')) != 0: + import CLUE_GPU_HIP as gpu_hip + hip_found = True + +def test_blobs(n_samples: int, n_dim: int , n_blobs: int = 4, mean: float = 0, sigma: float = 0.5, x_max: float = 30, y_max: float = 30) -> pd.DataFrame: """ Returns a dataframe containing randomly generated 2-dimensional or 3-dimensional blobs. @@ -46,8 +59,6 @@ def test_blobs(n_samples: int, n_dim: int, n_blobs: int = 4, mean: float = 0, DataFrame containing n_blobs gaussian blobs. """ - if x_max < 0. or y_max < 0.: - raise ValueError('Wrong parameter value. x_max and y_max must be positive.') if n_blobs < 0: raise ValueError('Wrong parameter value. The number of blobs must be positive.') if sigma < 0.: @@ -66,14 +77,15 @@ def test_blobs(n_samples: int, n_dim: int, n_blobs: int = 4, mean: float = 0, data['x1'] = blob_data.T[1] data['weight'] = np.full(shape=len(blob_data.T[0]), fill_value=1) + return pd.DataFrame(data) if n_dim == 3: data = {'x0': [], 'x1': [], 'x2': [], 'weight': []} sqrt_samples = int(sqrt(n_samples)) - z_values = np.random.normal(mean, sigma, sqrt_samples) + z_values = np.random.normal(mean, sigma,sqrt_samples) centers = [[x_max * rnd.random(), y_max * rnd.random()] for _ in range(n_blobs)] - for value in z_values: # for every z value, a layer is generated. + for value in z_values: # for every z value, a layer is generated. blob_data = make_blobs(n_samples=sqrt_samples, centers=np.array(centers))[0] data['x0'] = np.concatenate([data['x0'], blob_data.T[0]]) data['x1'] = np.concatenate([data['x1'], blob_data.T[1]]) @@ -84,7 +96,6 @@ def test_blobs(n_samples: int, n_dim: int, n_blobs: int = 4, mean: float = 0, return pd.DataFrame(data) - @dataclass() class clustering_data: """ @@ -106,13 +117,11 @@ class clustering_data: Number of points in the clustering data. """ - coords: np.ndarray - original_coords: np.ndarray - weight: np.ndarray - domain_ranges: list - n_dim: int - n_points: int - + coords : np.ndarray + original_coords : np.ndarray + weight : np.ndarray + n_dim : int + n_points : int @dataclass(eq=False) class cluster_properties: @@ -136,12 +145,12 @@ class cluster_properties: Dataframe containing is_seed and cluster_ids as columns. """ - n_clusters: int - cluster_ids: np.ndarray - is_seed: np.ndarray - cluster_points: np.ndarray - points_per_cluster: np.ndarray - output_df: pd.DataFrame + n_clusters : int + cluster_ids : np.ndarray + is_seed : np.ndarray + cluster_points : np.ndarray + points_per_cluster : np.ndarray + output_df : pd.DataFrame def __eq__(self, other): if self.n_clusters != other.n_clusters: @@ -165,10 +174,10 @@ class clusterer: Spatial parameter indicating how large is the region over which the local density of each point is calculated. rhoc : float - Energetic parameter representing the energy threshold value which divides seeds and + Parameter representing the density threshold value which divides seeds and outliers. - Points with an energy density lower than rhoc can't be seeds, can only be followers + Points with a density lower than rhoc can't be seeds, can only be followers or outliers. outlier : float Multiplicative increment of dc_ for getting the region over which the followers of a @@ -196,25 +205,25 @@ def __init__(self, dc_: float, rhoc_: float, outlier_: float, ppbin: int = 10): self.ppbin = ppbin # Initialize attributes - # Data containers + ## Data containers self.clust_data = None self.scaler = StandardScaler() - # Kernel for calculation of local density - self.kernel = Algo.flatKernel(0.5) + ## Kernel for calculation of local density + self.kernel = clue_kernels.FlatKernel(0.5) - # Output attributes + ## Output attributes self.clust_prop = None self.elapsed_time = 0. - def _read_array(self, input_data: Union[list, np.ndarray]) -> None: + def _read_array(self, input_data: Union[list,np.ndarray]) -> None: """ Reads data provided with lists or np.ndarrays Attributes ---------- input_data : list, np.ndarray - The coordinates and energy values of the data points + The coordinates and weight values of the data points Modified attributes ------------------- @@ -228,15 +237,14 @@ def _read_array(self, input_data: Union[list, np.ndarray]) -> None: if len(input_data) < 2 or len(input_data) > 10: raise ValueError("Inadequate data. The data must contain at least one coordinate" + - " and the energy.") + " and the weight.") self.clust_data = clustering_data(np.asarray(input_data[:-1]), np.copy(np.asarray(input_data[:-1])), np.asarray(input_data[-1]), - Algo.domain_t(), len(input_data[:-1]), len(input_data[-1])) - def _read_string(self, input_data: str) -> Union[pd.DataFrame, None]: + def _read_string(self, input_data: str) -> Union[pd.DataFrame,None]: """ Reads data provided by passing a string containing the path to a csv file @@ -255,19 +263,19 @@ def _read_string(self, input_data: str) -> Union[pd.DataFrame, None]: Dataframe containing the input data """ - if input_data[-3:] != 'csv': + if not input_data.endswith('.csv'): raise ValueError('Wrong type of file. The file is not a csv file.') df_ = pd.read_csv(input_data) return df_ - def _read_dict_df(self, input_data: Union[dict, pd.DataFrame]) -> pd.DataFrame: + def _read_dict_df(self, input_data: Union[dict,pd.DataFrame]) -> pd.DataFrame: """ Reads data provided using dictionaries or pandas dataframes Attributes ---------- input_data : dict, pd.DataFrame - The coordinates and energy values of the data points + The coordinates and weight values of the data points Modified attributes ------------------- @@ -307,20 +315,17 @@ def _handle_dataframe(self, df_: pd.DataFrame) -> None: # Check that the dimensionality of the dataset is adequate if len(df_.columns) < 2: raise ValueError("Inadequate data. The data must contain" - + " at least one coordinate and the energy.") + + " at least one coordinate and the weight.") if len(coordinate_columns) > 10: raise ValueError("Inadequate data. The maximum number of" + " dimensions supported is 10.") n_dim = len(coordinate_columns) n_points = len(df_.index) - coords = np.zeros(shape=(n_dim, n_points)) - for dim in range(n_dim): - coords[dim] = np.array(df_.iloc[:, dim]) + coords = df_.iloc[:, 0:-1].to_numpy() self.clust_data = clustering_data(coords, np.copy(coords), np.asarray(df_['weight']), - Algo.domain_t(), n_dim, n_points) @@ -331,7 +336,7 @@ def _rescale(self) -> None: Modified attributes ------------------- clust_data.coords : np.ndarray - Array containing the coordinates and energy values of the data points + Array containing the coordinates and weight values of the data points Returns ------- @@ -339,16 +344,14 @@ def _rescale(self) -> None: """ for dim in range(self.clust_data.n_dim): - self.clust_data.coords[dim] = \ - self.scaler.fit_transform(self.clust_data.coords[dim].reshape(-1, 1)).reshape(1, -1)[0] + self.clust_data.coords.T[dim] = \ + self.scaler.fit_transform(self.clust_data.coords.T[dim].reshape(-1, 1)).reshape(1, -1)[0] def read_data(self, - input_data: Union[pd.DataFrame, str, dict, list, np.ndarray], - rescale: bool = True, - **kwargs: tuple) -> None: + input_data: Union[pd.DataFrame,str,dict,list,np.ndarray]) -> None: """ Reads the data in input and fills the class members containing the coordinates - of the points, the energy weight, the number of dimensions and the number of points. + of the points, the weight, the number of dimensions and the number of points. Parameters ---------- @@ -362,8 +365,6 @@ def read_data(self, input_data : array_like The list or numpy array should contain a list of lists for the coordinates and a list for the weight. - restale : bool - Whether or not ot rescale the input data using a StandardScaler kwargs : tuples Tuples corresponding to the domain of any periodic variables. The keyword should be the keyword of the corrispoding variable. @@ -402,11 +403,7 @@ def read_data(self, self._handle_dataframe(df) # Rescale the coordinates with a standard scaler - if rescale: - self._rescale() - - # Construct the domains of all the coordinates - self.change_domains(**kwargs) + self._rescale() def change_coordinates(self, **kwargs: types.FunctionType) -> None: """ @@ -436,43 +433,11 @@ def change_coordinates(self, **kwargs: types.FunctionType) -> None: self.clust_data.coords[int(coord[1])] = \ self.scaler.fit_transform( self.clust_data.coords[int(coord[1])].reshape(-1, 1) - ).reshape(1, -1)[0] - - def change_domains(self, **kwargs: tuple) -> None: - """ - Change the domain range of the coordinates - - This method allows to change the domain of periodic coordinates by passing the domain of - each of these coordinates as a tuple, with the argument keyword in the form 'x*'. - - Parameters - ---------- - kwargs : tuples - Tuples corresponding to the domain of any periodic variables. The - keyword should be the keyword of the corrispoding variable. - - Modified attributes - ------------------- - domain_ranges : list of Algo.domain_t - List of the domains for each coordinate. - - Returns - ------- - None - """ - - # Construct the domains of all the coordinates - empty_domain = Algo.domain_t() - self.clust_data.domain_ranges = [empty_domain for _ in range(self.clust_data.n_dim)] - - for coord, domain in kwargs.items(): - self.clust_data.domain_ranges[int(coord[1])] = \ - Algo.domain_t(self.scaler.transform([[domain[0]]])[0][0], - self.scaler.transform([[domain[1]]])[0][0]) + ).reshape(1, -1)[0] def choose_kernel(self, choice: str, - parameters: Union[list, None] = None, + parameters: Union[list,None] = None, function: types.FunctionType = lambda: 0) -> None: """ Changes the kernel used in the calculation of local density. The default kernel @@ -503,27 +468,75 @@ def choose_kernel(self, if len(parameters) != 1: raise ValueError("Wrong number of parameters. The flat kernel" + " requires 1 parameter.") - self.kernel = Algo.flatKernel(parameters[0]) + self.kernel = CLUE_Convolutional_Kernels.FlatKernel(parameters[0]) elif choice == "exp": if len(parameters) != 2: raise ValueError("Wrong number of parameters. The exponential" + " kernel requires 2 parameters.") - self.kernel = Algo.exponentialKernel(parameters[0], parameters[1]) + self.kernel = CLUE_Convolutional_Kernels.ExponentialKernel(parameters[0], parameters[1]) elif choice == "gaus": if len(parameters) != 3: raise ValueError("Wrong number of parameters. The gaussian" + " kernel requires 3 parameters.") - self.kernel = Algo.gaussianKernel(parameters[0], parameters[1], parameters[2]) + self.kernel = CLUE_Convolutional_Kernels.GaussinKernel(parameters[0], parameters[1], parameters[2]) elif choice == "custom": if len(parameters) != 0: raise ValueError("Wrong number of parameters. Custom kernels" + " requires 0 parameters.") - self.kernel = Algo.customKernel(function) else: raise ValueError("Invalid kernel. The allowed choices for the" + " kernels are: flat, exp, gaus and custom.") - def run_clue(self, verbose: bool = False) -> None: + def list_devices(self, backend: str = "all") -> None: + """ + Lists the devices available for the chosen backend. + + Parameters + ---------- + backend : string, optional + The backend for which the devices are listed. The allowed values are + 'all', 'cpu serial', 'cpu tbb' and 'gpu cuda'. + The default value is 'all'. + + Raises + ------ + ValueError : If the backend is not valid. + """ + + if backend == "all": + cpu_serial.listDevices('cpu serial') + if tbb_found: + cpu_tbb.listDevices('cpu tbb') + if cuda_found: + gpu_cuda.listDevices('gpu cuda') + if hip_found: + gpu_hip.listDevices('gpu hip') + elif backend == "cpu serial": + cpu_serial.listDevices(backend) + elif backend == "cpu tbb": + if tbb_found: + cpu_tbb.listDevices(backend) + else: + print("TBB module not found. Please re-compile the library and try again.") + elif backend == "gpu cuda": + if cuda_found: + gpu_cuda.listDevices(backend) + else: + print("CUDA module not found. Please re-compile the library and try again.") + elif backend == "gpu hip": + if hip_found: + gpu_hip.listDevices(backend) + else: + print("HIP module not found. Please re-compile the library and try again.") + else: + raise ValueError("Invalid backend. The allowed choices for the" + + " backend are: all, cpu serial, cpu tbb and gpu cuda.") + + def run_clue(self, + backend: str = "cpu serial", + block_size: int = 1024, + device_id: int = 0, + verbose: bool = False) -> None: """ Executes the CLUE clustering algorithm. @@ -553,10 +566,41 @@ def run_clue(self, verbose: bool = False) -> None: """ start = time.time_ns() - cluster_id_is_seed = Algo.mainRun(self.dc_, self.rhoc, self.outlier, self.ppbin, - self.clust_data.domain_ranges, self.kernel, - self.clust_data.coords, self.clust_data.weight, - self.clust_data.n_dim) + if backend == "cpu serial": + cluster_id_is_seed = cpu_serial.mainRun(self.dc_, self.rhoc, self.outlier, self.ppbin, + self.clust_data.coords, self.clust_data.weight, + self.kernel, self.clust_data.n_dim, block_size, + device_id) + elif backend == "cpu tbb": + if tbb_found: + cluster_id_is_seed = cpu_tbb.mainRun(self.dc_, self.rhoc, self.outlier, + self.ppbin, self.clust_data.coords, + self.clust_data.weight, self.kernel, + self.clust_data.n_dim, block_size, + device_id) + else: + print("TBB module not found. Please re-compile the library and try again.") + + elif backend == "gpu cuda": + if cuda_found: + cluster_id_is_seed = gpu_cuda.mainRun(self.dc_, self.rhoc, self.outlier, + self.ppbin, self.clust_data.coords, + self.clust_data.weight, self.kernel, + self.clust_data.n_dim, block_size, + device_id) + else: + print("CUDA module not found. Please re-compile the library and try again.") + + elif backend == "gpu hip": + if hip_found: + cluster_id_is_seed = gpu_hip.mainRun(self.dc_, self.rhoc, self.outlier, + self.ppbin, self.clust_data.coords, + self.clust_data.weight, self.kernel, + self.clust_data.n_dim, block_size, + device_id) + else: + print("HIP module not found. Please re-compile the library and try again.") + finish = time.time_ns() cluster_ids = np.array(cluster_id_is_seed[0]) is_seed = np.array(cluster_id_is_seed[1]) @@ -578,12 +622,12 @@ def run_clue(self, verbose: bool = False) -> None: points_per_cluster, output_df) - self.elapsed_time = (finish - start) / (10**6) + self.elapsed_time = (finish - start)/(10**6) if verbose: - print(f'CLUE run in {self.elapsed_time} ms') + print(f'CLUE executed in {self.elapsed_time} ms') print(f'Number of clusters found: {self.clust_prop.n_clusters}') - def input_plotter(self, plot_title: str = '', title_size: float = 16, + def input_plotter(self, plot_title: str='', title_size: float = 16, x_label: str = 'x', y_label: str = 'y', z_label: str = 'z', label_size: float = 16, pt_size: float = 1, pt_colour: str = 'b', grid: bool = True, grid_style: str = '--', grid_size: float = 0.2, @@ -668,10 +712,10 @@ def input_plotter(self, plot_title: str = '', title_size: float = 16, fig = plt.figure() ax_ = fig.add_subplot(projection='3d') ax_.scatter(cartesian_coords[0], - cartesian_coords[1], - cartesian_coords[2], - s=pt_size, - color=pt_colour) + cartesian_coords[1], + cartesian_coords[2], + s=pt_size, + color=pt_colour) # Customization of the plot title ax_.set_title(plot_title, fontsize=title_size) @@ -753,7 +797,7 @@ def cluster_plotter(self, plot_title: str = '', title_size: float = 16, """ # Convert the used coordinates to cartesian coordiantes - cartesian_coords = np.copy(self.clust_data.original_coords) + cartesian_coords = np.copy(self.clust_data.original_coords.T) for coord, func in kwargs.items(): cartesian_coords[int(coord[1])] = func(self.clust_data.original_coords) @@ -766,12 +810,12 @@ def cluster_plotter(self, plot_title: str = '', title_size: float = 16, max_clusterid = max(df_["cluster_ids"]) - df_out = df_[df_.cluster_ids == -1] # Outliers + df_out = df_[df_.cluster_ids == -1] # Outliers plt.scatter(df_out.x0, df_out.x1, s=outl_size, marker='x', color='0.4') - for i in range(0, max_clusterid + 1): - dfi = df_[df_.cluster_ids == i] # ith cluster + for i in range(0, max_clusterid+1): + dfi = df_[df_.cluster_ids == i] # ith cluster plt.scatter(dfi.x0, dfi.x1, s=pt_size, marker='.') - df_seed = df_[df_.isSeed == 1] # Only Seeds + df_seed = df_[df_.isSeed == 1] # Only Seeds plt.scatter(df_seed.x0, df_seed.x1, s=seed_size, color='r', marker='*') # Customization of the plot title @@ -805,13 +849,13 @@ def cluster_plotter(self, plot_title: str = '', title_size: float = 16, ax_ = fig.add_subplot(projection='3d') df_out = df_[df_.cluster_ids == -1] - ax_.scatter(df_out.x0, df_out.x1, df_out.x2, s=outl_size, color='grey', marker='x') - for i in range(0, max_clusterid + 1): + ax_.scatter(df_out.x0, df_out.x1, df_out.x2, s=outl_size, color = 'grey', marker = 'x') + for i in range(0, max_clusterid+1): dfi = df_[df_.cluster_ids == i] - ax_.scatter(dfi.x0, dfi.x1, dfi.x2, s=pt_size, marker='.') + ax_.scatter(dfi.x0, dfi.x1, dfi.x2, s=pt_size, marker = '.') - df_seed = df_[df_.isSeed == 1] # Only Seeds - ax_.scatter(df_seed.x0, df_seed.x1, df_seed.x2, s=seed_size, color='r', marker='*') + df_seed = df_[df_.isSeed == 1] # Only Seeds + ax_.scatter(df_seed.x0, df_seed.x1, df_seed.x2, s=seed_size, color = 'r', marker = '*') # Customization of the plot title ax_.set_title(plot_title, fontsize=title_size) @@ -858,9 +902,24 @@ def to_csv(self, output_folder: str, file_name: str) -> None: out_path = output_folder + file_name data = {} for i in range(self.clust_data.n_dim): - data['x' + str(i)] = self.clust_data.coords[i] + data['x' + str(i)] = self.clust_data.coords.T[i] data['cluster_ids'] = self.clust_prop.cluster_ids data['is_seed'] = self.clust_prop.is_seed df_ = pd.DataFrame(data) - df_.to_csv(out_path, index=False) + df_.to_csv(out_path,index=False) + +if __name__ == "__main__": + c = clusterer(0.4,5,1.) + c.read_data('./sissa.csv') + c.run_clue(backend="cpu serial", verbose=True) + # c.run_clue(backend="cpu tbb", verbose=True) + # c.run_clue(backend="gpu cuda", verbose=True) + # c.run_clue(backend="gpu hip", verbose=True) + c.cluster_plotter() + # c.to_csv('./','sissa_output_tbb.csv') + c.list_devices('cpu serial') + c.list_devices('cpu tbb') + c.list_devices('gpu cuda') + c.list_devices('gpu hip') + c.list_devices() From 1176beca5835e436b466fac688243c843f7e9638 Mon Sep 17 00:00:00 2001 From: sbaldu Date: Tue, 23 Jan 2024 16:06:29 +0100 Subject: [PATCH 07/16] Fix paths of modules when installing the library --- CLUEstering/CLUEstering.py | 10 ++++-- CMakeLists.txt | 69 +++++++++++++++++++------------------- setup.py | 2 +- 3 files changed, 43 insertions(+), 38 deletions(-) diff --git a/CLUEstering/CLUEstering.py b/CLUEstering/CLUEstering.py index 19cdfbbe..3bdcaa81 100644 --- a/CLUEstering/CLUEstering.py +++ b/CLUEstering/CLUEstering.py @@ -14,18 +14,22 @@ import pandas as pd from sklearn.datasets import make_blobs from sklearn.preprocessing import StandardScaler +from os.path import dirname, join +path = dirname(__file__) +import sys +sys.path.insert(1, join(path, 'lib')) import CLUE_Convolutional_Kernels as clue_kernels import CLUE_CPU_Serial as cpu_serial tbb_found = False -if len(glob('./CLUE_CPU_TBB*.so')) != 0: +if len(glob(join(path, 'lib/CLUE_CPU_TBB*.so'))) != 0: import CLUE_CPU_TBB as cpu_tbb tbb_found = True cuda_found = False -if len(glob('./CLUE_GPU_CUDA*.so')) != 0: +if len(glob(join(path, 'lib/CLUE_GPU_CUDA*.so'))) != 0: import CLUE_GPU_CUDA as gpu_cuda cuda_found = True hip_found = False -if len(glob('./CLUE_GPU_HIP*.so')) != 0: +if len(glob(join(path, 'lib/CLUE_GPU_HIP*.so'))) != 0: import CLUE_GPU_HIP as gpu_hip hip_found = True diff --git a/CMakeLists.txt b/CMakeLists.txt index 125dc95a..6d6cbf0d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -14,15 +14,15 @@ add_subdirectory(extern/pybind11) find_package(Boost 1.75.0) -if (NOT Boost_FOUND) +if(NOT Boost_FOUND) include(FetchContent) FetchContent_Declare( - boost - URL https://boostorg.jfrog.io/artifactory/main/release/1.76.0/source/boost_1_76_0.tar.gz + boost + URL https://boostorg.jfrog.io/artifactory/main/release/1.76.0/source/boost_1_76_0.tar.gz ) FetchContent_GetProperties(boost) - if (NOT boost_POPULATED) + if(NOT boost_POPULATED) FetchContent_Populate(boost) endif() set(Boost_PATH ./build/_deps/boost-src) @@ -41,7 +41,7 @@ target_compile_options( -DALPAKA_ACC_CPU_B_SEQ_T_SEQ_ENABLED -DALPAKA_ACC_CPU_B_SEQ_T_SEQ_SYNC_BACKEND) set_target_properties(CLUE_Convolutional_Kernels - PROPERTIES LIBRARY_OUTPUT_DIRECTORY ../CLUEstering/) + PROPERTIES LIBRARY_OUTPUT_DIRECTORY ../CLUEstering/lib/) # compile cpu serial module pybind11_add_module(CLUE_CPU_Serial SHARED @@ -54,86 +54,87 @@ target_compile_options( -DALPAKA_ACC_CPU_B_SEQ_T_SEQ_ENABLED -DALPAKA_ACC_CPU_B_SEQ_T_SEQ_SYNC_BACKEND) set_target_properties(CLUE_CPU_Serial PROPERTIES LIBRARY_OUTPUT_DIRECTORY - ../CLUEstering/) + ../CLUEstering/lib/) find_package(TBB) -if (TBB_FOUND) +if(TBB_FOUND) # compile cpu tbb module pybind11_add_module(CLUE_CPU_TBB SHARED - ./CLUEstering/alpaka/BindingModules/binding_cpu_tbb.cc) + ./CLUEstering/alpaka/BindingModules/binding_cpu_tbb.cc) target_link_libraries(CLUE_CPU_TBB PRIVATE ${Boost_LIBRARIES}) target_include_directories(CLUE_CPU_TBB PRIVATE ${Boost_PATH}) target_compile_options( - CLUE_CPU_TBB - PRIVATE -ltbb -DALPAKA_ACC_CPU_B_TBB_T_SEQ_PRESENT - -DALPAKA_ACC_CPU_B_TBB_T_SEQ_ENABLED - -DALPAKA_ACC_CPU_B_TBB_T_SEQ_ASYNC_BACKEND) + CLUE_CPU_TBB + PRIVATE -ltbb -DALPAKA_ACC_CPU_B_TBB_T_SEQ_PRESENT + -DALPAKA_ACC_CPU_B_TBB_T_SEQ_ENABLED + -DALPAKA_ACC_CPU_B_TBB_T_SEQ_ASYNC_BACKEND) target_link_libraries(CLUE_CPU_TBB PRIVATE TBB::tbb) set_target_properties(CLUE_CPU_TBB PROPERTIES LIBRARY_OUTPUT_DIRECTORY - ../CLUEstering/) + ../CLUEstering/lib/) endif() include(CheckLanguage) # check if CUDA is available check_language(CUDA) -if (CMAKE_CUDA_COMPILER) +if(CMAKE_CUDA_COMPILER) # enable CUDA enable_language(CUDA) set(CMAKE_CUDA_HOST_COMPILER ${CMAKE_CUDA_COMPILER}) # set the CUDA standard if(NOT DEFINED CMAKE_CUDA_STANDARD) - set(CMAKE_CUDA_STANDARD 17) - set(CMAKE_CUDA_STANDARD_REQUIRED ON) + set(CMAKE_CUDA_STANDARD 17) + set(CMAKE_CUDA_STANDARD_REQUIRED ON) endif() set(CMAKE_CUDA_FLAGS "-Wall -Wextra -g -O2") # compile nvidia gpu async module pybind11_add_module(CLUE_GPU_CUDA SHARED - ./CLUEstering/alpaka/BindingModules/binding_gpu_cuda.cu) + ./CLUEstering/alpaka/BindingModules/binding_gpu_cuda.cu) target_link_libraries(CLUE_Convolutional_Kernels PRIVATE ${Boost_LIBRARIES}) target_include_directories(CLUE_Convolutional_Kernels PRIVATE ${Boost_PATH}) # set the cuda architectures set_target_properties(CLUE_GPU_CUDA PROPERTIES CUDA_ARCHITECTURES - "50;60;61;62;70") + "50;60;61;62;70") # alpaka compilation flags target_compile_options( - CLUE_GPU_CUDA - PRIVATE -DALPAKA_ACC_GPU_CUDA_PRESENT -DALPAKA_ACC_GPU_CUDA_ENABLED - -DALPAKA_ACC_GPU_CUDA_ASYNC_BACKEND) + CLUE_GPU_CUDA + PRIVATE -DALPAKA_ACC_GPU_CUDA_PRESENT -DALPAKA_ACC_GPU_CUDA_ENABLED + -DALPAKA_ACC_GPU_CUDA_ASYNC_BACKEND) # nvcc compilation flags target_compile_options( - CLUE_GPU_CUDA PRIVATE --expt-relaxed-constexpr -gencode - arch=compute_61,code=[sm_61,compute_61]) + CLUE_GPU_CUDA PRIVATE --expt-relaxed-constexpr -gencode + arch=compute_61,code=[sm_61,compute_61]) set_target_properties(CLUE_GPU_CUDA PROPERTIES LIBRARY_OUTPUT_DIRECTORY - ../CLUEstering/) + ../CLUEstering/lib/) endif() # check if HIP is available check_language(HIP) -if (CMAKE_HIP_COMPILER) +if(CMAKE_HIP_COMPILER) # enable HIP enable_language(HIP) set(CMAKE_HIP_HOST_COMPILER ${CMAKE_HIP_COMPILER}) # look for the hip package folder find_package(hip) - message("${hip_INCLUDE_DIRS}") - message("${CMAKE_HIP_COMPILER}") set(CMAKE_CXX_COMPILER "hipcc") pybind11_add_module(CLUE_GPU_HIP SHARED - ./CLUEstering/alpaka/BindingModules/binding_gpu_hip.cc) + ./CLUEstering/alpaka/BindingModules/binding_gpu_hip.cc) target_link_libraries(CLUE_Convolutional_Kernels PRIVATE ${Boost_LIBRARIES}) target_include_directories(CLUE_Convolutional_Kernels PRIVATE ${Boost_PATH}) - target_compile_options(CLUE_GPU_HIP PRIVATE -DALPAKA_ACC_GPU_HIP_PRESENT - -DALPAKA_ACC_GPU_HIP_ENABLED - -DALPAKA_ACC_GPU_HIP_ASYNC_BACKEND) + target_compile_options( + CLUE_GPU_HIP + PRIVATE -DALPAKA_ACC_GPU_HIP_PRESENT -DALPAKA_ACC_GPU_HIP_ENABLED + -DALPAKA_ACC_GPU_HIP_ASYNC_BACKEND) target_include_directories(CLUE_GPU_HIP PRIVATE ${hip_INCLUDE_DIRS}) - target_include_directories(CLUE_GPU_HIP PRIVATE ${hip_INCLUDE_DIRS}/../hiprand/include) - target_include_directories(CLUE_GPU_HIP PRIVATE ${hip_INCLUDE_DIRS}/../rocrand/include) + target_include_directories(CLUE_GPU_HIP + PRIVATE ${hip_INCLUDE_DIRS}/../hiprand/include) + target_include_directories(CLUE_GPU_HIP + PRIVATE ${hip_INCLUDE_DIRS}/../rocrand/include) set_target_properties(CLUE_GPU_HIP PROPERTIES LIBRARY_OUTPUT_DIRECTORY - ../CLUEstering/) + ../CLUEstering/lib/) endif() diff --git a/setup.py b/setup.py index 949e8423..e0db4123 100644 --- a/setup.py +++ b/setup.py @@ -21,7 +21,7 @@ long_description_content_type='text/markdown', packages=find_packages(), install_requires=['scikit-learn','numpy','matplotlib','pandas'], - package_data={'': ['*.so']}, + package_data={'': ['lib/*.so']}, keywords=['Python','Clustering','Binding'], python_requires='>=3.7', classifiers=[ From b39ae0f06871424ee0ec135bfe2e56f1510ef8cf Mon Sep 17 00:00:00 2001 From: sbaldu Date: Tue, 23 Jan 2024 16:47:23 +0100 Subject: [PATCH 08/16] Change extension of cuda binding file to `.cu` --- ...inding_gpu_cuda.cc => binding_gpu_cuda.cu} | 87 +++++++++++++------ 1 file changed, 60 insertions(+), 27 deletions(-) rename CLUEstering/alpaka/BindingModules/{binding_gpu_cuda.cc => binding_gpu_cuda.cu} (71%) diff --git a/CLUEstering/alpaka/BindingModules/binding_gpu_cuda.cc b/CLUEstering/alpaka/BindingModules/binding_gpu_cuda.cu similarity index 71% rename from CLUEstering/alpaka/BindingModules/binding_gpu_cuda.cc rename to CLUEstering/alpaka/BindingModules/binding_gpu_cuda.cu index da464650..15cea948 100644 --- a/CLUEstering/alpaka/BindingModules/binding_gpu_cuda.cc +++ b/CLUEstering/alpaka/BindingModules/binding_gpu_cuda.cu @@ -49,28 +49,39 @@ namespace alpaka_cuda_async { // Running the clustering algorithm // switch (Ndim) { [[unlikely]] case (1) : - return run1(dc, rhoc, outlier, pPBin, coords, weights, kernel, queue_, block_size); + /* return run1(dc, rhoc, outlier, pPBin, coords, weights, queue_); */ + break; [[likely]] case (2) : return run2(dc, rhoc, outlier, pPBin, coords, weights, kernel, queue_, block_size); + break; [[likely]] case (3) : - return run3(dc, rhoc, outlier, pPBin, coords, weights, kernel, queue_, block_size); + /* return run3(dc, rhoc, outlier, pPBin, coords, weights, queue_); */ + break; [[unlikely]] case (4) : - return run4(dc, rhoc, outlier, pPBin, coords, weights, kernel, queue_, block_size); + /* return run4(dc, rhoc, outlier, pPBin, coords, weights, queue_); */ + break; [[unlikely]] case (5) : - return run5(dc, rhoc, outlier, pPBin, coords, weights, kernel, queue_, block_size); + /* return run5(dc, rhoc, outlier, pPBin, coords, weights, queue_); */ + break; [[unlikely]] case (6) : - return run6(dc, rhoc, outlier, pPBin, coords, weights, kernel, queue_, block_size); + /* return run6(dc, rhoc, outlier, pPBin, coords, weights, queue_); */ + break; [[unlikely]] case (7) : - return run7(dc, rhoc, outlier, pPBin, coords, weights, kernel, queue_, block_size); + /* return run7(dc, rhoc, outlier, pPBin, coords, weights, queue_); */ + break; [[unlikely]] case (8) : - return run8(dc, rhoc, outlier, pPBin, coords, weights, kernel, queue_, block_size); + /* return run8(dc, rhoc, outlier, pPBin, coords, weights, queue_); */ + break; [[unlikely]] case (9) : - return run9(dc, rhoc, outlier, pPBin, coords, weights, kernel, queue_, block_size); + /* return run9(dc, rhoc, outlier, pPBin, coords, weights, queue_); */ + break; [[unlikely]] case (10) : - return run10(dc, rhoc, outlier, pPBin, coords, weights, kernel, queue_, block_size); + /* return run10(dc, rhoc, outlier, pPBin, coords, weights, queue_); */ + break; [[unlikely]] default: std::cout << "This library only works up to 10 dimensions\n"; return {}; + break; } } @@ -92,28 +103,39 @@ namespace alpaka_cuda_async { // Running the clustering algorithm // switch (Ndim) { [[unlikely]] case (1) : - return run1(dc, rhoc, outlier, pPBin, coords, weights, kernel, queue_, block_size); + /* return run1(dc, rhoc, outlier, pPBin, coords, weights, queue_); */ + break; [[likely]] case (2) : return run2(dc, rhoc, outlier, pPBin, coords, weights, kernel, queue_, block_size); + break; [[likely]] case (3) : - return run3(dc, rhoc, outlier, pPBin, coords, weights, kernel, queue_, block_size); + /* return run3(dc, rhoc, outlier, pPBin, coords, weights, queue_); */ + break; [[unlikely]] case (4) : - return run4(dc, rhoc, outlier, pPBin, coords, weights, kernel, queue_, block_size); + /* return run4(dc, rhoc, outlier, pPBin, coords, weights, queue_); */ + break; [[unlikely]] case (5) : - return run5(dc, rhoc, outlier, pPBin, coords, weights, kernel, queue_, block_size); + /* return run5(dc, rhoc, outlier, pPBin, coords, weights, queue_); */ + break; [[unlikely]] case (6) : - return run6(dc, rhoc, outlier, pPBin, coords, weights, kernel, queue_, block_size); + /* return run6(dc, rhoc, outlier, pPBin, coords, weights, queue_); */ + break; [[unlikely]] case (7) : - return run7(dc, rhoc, outlier, pPBin, coords, weights, kernel, queue_, block_size); + /* return run7(dc, rhoc, outlier, pPBin, coords, weights, queue_); */ + break; [[unlikely]] case (8) : - return run8(dc, rhoc, outlier, pPBin, coords, weights, kernel, queue_, block_size); + /* return run8(dc, rhoc, outlier, pPBin, coords, weights, queue_); */ + break; [[unlikely]] case (9) : - return run9(dc, rhoc, outlier, pPBin, coords, weights, kernel, queue_, block_size); + /* return run9(dc, rhoc, outlier, pPBin, coords, weights, queue_); */ + break; [[unlikely]] case (10) : - return run10(dc, rhoc, outlier, pPBin, coords, weights, kernel, queue_, block_size); + /* return run10(dc, rhoc, outlier, pPBin, coords, weights, queue_); */ + break; [[unlikely]] default: std::cout << "This library only works up to 10 dimensions\n"; return {}; + break; } } @@ -135,28 +157,39 @@ namespace alpaka_cuda_async { // Running the clustering algorithm // switch (Ndim) { [[unlikely]] case (1) : - return run1(dc, rhoc, outlier, pPBin, coords, weights, kernel, queue_, block_size); + /* return run1(dc, rhoc, outlier, pPBin, coords, weights, queue_); */ + break; [[likely]] case (2) : return run2(dc, rhoc, outlier, pPBin, coords, weights, kernel, queue_, block_size); + break; [[likely]] case (3) : - return run3(dc, rhoc, outlier, pPBin, coords, weights, kernel, queue_, block_size); + /* return run3(dc, rhoc, outlier, pPBin, coords, weights, queue_); */ + break; [[unlikely]] case (4) : - return run4(dc, rhoc, outlier, pPBin, coords, weights, kernel, queue_, block_size); + /* return run4(dc, rhoc, outlier, pPBin, coords, weights, queue_); */ + break; [[unlikely]] case (5) : - return run5(dc, rhoc, outlier, pPBin, coords, weights, kernel, queue_, block_size); + /* return run5(dc, rhoc, outlier, pPBin, coords, weights, queue_); */ + break; [[unlikely]] case (6) : - return run6(dc, rhoc, outlier, pPBin, coords, weights, kernel, queue_, block_size); + /* return run6(dc, rhoc, outlier, pPBin, coords, weights, queue_); */ + break; [[unlikely]] case (7) : - return run7(dc, rhoc, outlier, pPBin, coords, weights, kernel, queue_, block_size); + /* return run7(dc, rhoc, outlier, pPBin, coords, weights, queue_); */ + break; [[unlikely]] case (8) : - return run8(dc, rhoc, outlier, pPBin, coords, weights, kernel, queue_, block_size); + /* return run8(dc, rhoc, outlier, pPBin, coords, weights, queue_); */ + break; [[unlikely]] case (9) : - return run9(dc, rhoc, outlier, pPBin, coords, weights, kernel, queue_, block_size); + /* return run9(dc, rhoc, outlier, pPBin, coords, weights, queue_); */ + break; [[unlikely]] case (10) : - return run10(dc, rhoc, outlier, pPBin, coords, weights, kernel, queue_, block_size); + /* return run10(dc, rhoc, outlier, pPBin, coords, weights, queue_); */ + break; [[unlikely]] default: std::cout << "This library only works up to 10 dimensions\n"; return {}; + break; } } From d26a664497e3a062be47bd74ab2a5001ab7981a0 Mon Sep 17 00:00:00 2001 From: sbaldu Date: Wed, 24 Jan 2024 08:29:14 +0100 Subject: [PATCH 09/16] Don't use `.cu` files --- .../{binding_gpu_cuda.cu => binding_gpu_cuda.cc} | 0 CMakeLists.txt | 7 +++++-- 2 files changed, 5 insertions(+), 2 deletions(-) rename CLUEstering/alpaka/BindingModules/{binding_gpu_cuda.cu => binding_gpu_cuda.cc} (100%) diff --git a/CLUEstering/alpaka/BindingModules/binding_gpu_cuda.cu b/CLUEstering/alpaka/BindingModules/binding_gpu_cuda.cc similarity index 100% rename from CLUEstering/alpaka/BindingModules/binding_gpu_cuda.cu rename to CLUEstering/alpaka/BindingModules/binding_gpu_cuda.cc diff --git a/CMakeLists.txt b/CMakeLists.txt index 6d6cbf0d..d2eda83a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -90,9 +90,12 @@ if(CMAKE_CUDA_COMPILER) set(CMAKE_CUDA_FLAGS "-Wall -Wextra -g -O2") - # compile nvidia gpu async module + # compile nvidia gpu async module set(CMAKE_CXX_COMPILER "/opt/cuda/bin/nvcc") + set_source_files_properties( + ./CLUEstering/alpaka/BindingModules/binding_gpu_cuda.cc PROPERTIES LANGUAGE + CUDA) pybind11_add_module(CLUE_GPU_CUDA SHARED - ./CLUEstering/alpaka/BindingModules/binding_gpu_cuda.cu) + ./CLUEstering/alpaka/BindingModules/binding_gpu_cuda.cc) target_link_libraries(CLUE_Convolutional_Kernels PRIVATE ${Boost_LIBRARIES}) target_include_directories(CLUE_Convolutional_Kernels PRIVATE ${Boost_PATH}) # set the cuda architectures From 9aa5355c46d5f2fff60a3c5c2a4c92e76154b105 Mon Sep 17 00:00:00 2001 From: sbaldu Date: Wed, 24 Jan 2024 11:02:59 +0100 Subject: [PATCH 10/16] Small tweaks to CMake file and formatting --- CMakeLists.txt | 42 +++++++++++++++++++++++++++++++----------- 1 file changed, 31 insertions(+), 11 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index d2eda83a..668ae1d8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -10,10 +10,13 @@ set(CMAKE_CXX_FLAGS "-Wall -Wextra -g -O2") # include alpaka extern subfolder include_directories(extern/alpaka/include) # include pybind11 extern subfolder +set(PYBIND11_FINDPYTHON ON) +set(PYBIND11_PYTHON_VERSION ">=3.8") add_subdirectory(extern/pybind11) find_package(Boost 1.75.0) +# if boost is not found, t's fetched from the official boost repository if(NOT Boost_FOUND) include(FetchContent) FetchContent_Declare( @@ -30,34 +33,41 @@ else() set(Boost_PATH ${Boost_INCLUDE_DIRS}) endif() -# compile convolutional kernel module +# Convolutional Kernels compile convolutional kernel module pybind11_add_module(CLUE_Convolutional_Kernels SHARED ./CLUEstering/alpaka/BindingModules/binding_kernels.cc) +# link boost target_link_libraries(CLUE_Convolutional_Kernels PRIVATE ${Boost_LIBRARIES}) target_include_directories(CLUE_Convolutional_Kernels PRIVATE ${Boost_PATH}) +# alpaka build flags target_compile_options( CLUE_Convolutional_Kernels PRIVATE -DALPAKA_HOST_ONLY -DALPAKA_ACC_CPU_B_SEQ_T_SEQ_PRESENT -DALPAKA_ACC_CPU_B_SEQ_T_SEQ_ENABLED -DALPAKA_ACC_CPU_B_SEQ_T_SEQ_SYNC_BACKEND) +# set output directory set_target_properties(CLUE_Convolutional_Kernels PROPERTIES LIBRARY_OUTPUT_DIRECTORY ../CLUEstering/lib/) -# compile cpu serial module +# CPU Serial compile cpu serial module pybind11_add_module(CLUE_CPU_Serial SHARED ./CLUEstering/alpaka/BindingModules/binding_cpu.cc) +# link boost target_link_libraries(CLUE_CPU_Serial PRIVATE ${Boost_LIBRARIES}) target_include_directories(CLUE_CPU_Serial PRIVATE ${Boost_PATH}) +# alpaka build flags target_compile_options( CLUE_CPU_Serial PRIVATE -DALPAKA_HOST_ONLY -DALPAKA_ACC_CPU_B_SEQ_T_SEQ_PRESENT -DALPAKA_ACC_CPU_B_SEQ_T_SEQ_ENABLED -DALPAKA_ACC_CPU_B_SEQ_T_SEQ_SYNC_BACKEND) +# set output directory set_target_properties(CLUE_CPU_Serial PROPERTIES LIBRARY_OUTPUT_DIRECTORY ../CLUEstering/lib/) find_package(TBB) +# CPU TBB if(TBB_FOUND) # compile cpu tbb module pybind11_add_module(CLUE_CPU_TBB SHARED @@ -74,9 +84,11 @@ if(TBB_FOUND) ../CLUEstering/lib/) endif() -include(CheckLanguage) # check if CUDA is available +include(CheckLanguage) check_language(CUDA) + +# GPU CUDA if(CMAKE_CUDA_COMPILER) # enable CUDA enable_language(CUDA) @@ -90,18 +102,20 @@ if(CMAKE_CUDA_COMPILER) set(CMAKE_CUDA_FLAGS "-Wall -Wextra -g -O2") - # compile nvidia gpu async module set(CMAKE_CXX_COMPILER "/opt/cuda/bin/nvcc") + # compile the file with .cc extension using nvcc set_source_files_properties( ./CLUEstering/alpaka/BindingModules/binding_gpu_cuda.cc PROPERTIES LANGUAGE CUDA) + # compile gpu cuda module pybind11_add_module(CLUE_GPU_CUDA SHARED ./CLUEstering/alpaka/BindingModules/binding_gpu_cuda.cc) + # link boost target_link_libraries(CLUE_Convolutional_Kernels PRIVATE ${Boost_LIBRARIES}) target_include_directories(CLUE_Convolutional_Kernels PRIVATE ${Boost_PATH}) # set the cuda architectures set_target_properties(CLUE_GPU_CUDA PROPERTIES CUDA_ARCHITECTURES "50;60;61;62;70") - # alpaka compilation flags + # alpaka build flags target_compile_options( CLUE_GPU_CUDA PRIVATE -DALPAKA_ACC_GPU_CUDA_PRESENT -DALPAKA_ACC_GPU_CUDA_ENABLED @@ -110,11 +124,12 @@ if(CMAKE_CUDA_COMPILER) target_compile_options( CLUE_GPU_CUDA PRIVATE --expt-relaxed-constexpr -gencode arch=compute_61,code=[sm_61,compute_61]) + # set output directory set_target_properties(CLUE_GPU_CUDA PROPERTIES LIBRARY_OUTPUT_DIRECTORY ../CLUEstering/lib/) endif() -# check if HIP is available +# GPU HIP check if HIP is available check_language(HIP) if(CMAKE_HIP_COMPILER) # enable HIP @@ -124,20 +139,25 @@ if(CMAKE_HIP_COMPILER) # look for the hip package folder find_package(hip) - set(CMAKE_CXX_COMPILER "hipcc") + set(hip_BASE "${hip_INCLUDE_DIRS}/..") + # set the hipcc compiler + set(CMAKE_CXX_COMPILER "${hip_BASE}/bin/hipcc") + # compile gpu hip module pybind11_add_module(CLUE_GPU_HIP SHARED ./CLUEstering/alpaka/BindingModules/binding_gpu_hip.cc) + # link boost target_link_libraries(CLUE_Convolutional_Kernels PRIVATE ${Boost_LIBRARIES}) target_include_directories(CLUE_Convolutional_Kernels PRIVATE ${Boost_PATH}) + # alpaka build flags target_compile_options( CLUE_GPU_HIP PRIVATE -DALPAKA_ACC_GPU_HIP_PRESENT -DALPAKA_ACC_GPU_HIP_ENABLED -DALPAKA_ACC_GPU_HIP_ASYNC_BACKEND) + # link hip-rand libraries target_include_directories(CLUE_GPU_HIP PRIVATE ${hip_INCLUDE_DIRS}) - target_include_directories(CLUE_GPU_HIP - PRIVATE ${hip_INCLUDE_DIRS}/../hiprand/include) - target_include_directories(CLUE_GPU_HIP - PRIVATE ${hip_INCLUDE_DIRS}/../rocrand/include) + target_include_directories(CLUE_GPU_HIP PRIVATE ${hip_BASE}/hiprand/include) + target_include_directories(CLUE_GPU_HIP PRIVATE ${hip_BASE}/rocrand/include) + # set output directory set_target_properties(CLUE_GPU_HIP PROPERTIES LIBRARY_OUTPUT_DIRECTORY ../CLUEstering/lib/) endif() From 479b4551cec7ecca8bb4d16a07957afab248087b Mon Sep 17 00:00:00 2001 From: sbaldu Date: Wed, 24 Jan 2024 11:09:03 +0100 Subject: [PATCH 11/16] Fix typos --- .../alpaka/BindingModules/binding_gpu_cuda.cc | 87 ++++++------------- 1 file changed, 27 insertions(+), 60 deletions(-) diff --git a/CLUEstering/alpaka/BindingModules/binding_gpu_cuda.cc b/CLUEstering/alpaka/BindingModules/binding_gpu_cuda.cc index 15cea948..da464650 100644 --- a/CLUEstering/alpaka/BindingModules/binding_gpu_cuda.cc +++ b/CLUEstering/alpaka/BindingModules/binding_gpu_cuda.cc @@ -49,39 +49,28 @@ namespace alpaka_cuda_async { // Running the clustering algorithm // switch (Ndim) { [[unlikely]] case (1) : - /* return run1(dc, rhoc, outlier, pPBin, coords, weights, queue_); */ - break; + return run1(dc, rhoc, outlier, pPBin, coords, weights, kernel, queue_, block_size); [[likely]] case (2) : return run2(dc, rhoc, outlier, pPBin, coords, weights, kernel, queue_, block_size); - break; [[likely]] case (3) : - /* return run3(dc, rhoc, outlier, pPBin, coords, weights, queue_); */ - break; + return run3(dc, rhoc, outlier, pPBin, coords, weights, kernel, queue_, block_size); [[unlikely]] case (4) : - /* return run4(dc, rhoc, outlier, pPBin, coords, weights, queue_); */ - break; + return run4(dc, rhoc, outlier, pPBin, coords, weights, kernel, queue_, block_size); [[unlikely]] case (5) : - /* return run5(dc, rhoc, outlier, pPBin, coords, weights, queue_); */ - break; + return run5(dc, rhoc, outlier, pPBin, coords, weights, kernel, queue_, block_size); [[unlikely]] case (6) : - /* return run6(dc, rhoc, outlier, pPBin, coords, weights, queue_); */ - break; + return run6(dc, rhoc, outlier, pPBin, coords, weights, kernel, queue_, block_size); [[unlikely]] case (7) : - /* return run7(dc, rhoc, outlier, pPBin, coords, weights, queue_); */ - break; + return run7(dc, rhoc, outlier, pPBin, coords, weights, kernel, queue_, block_size); [[unlikely]] case (8) : - /* return run8(dc, rhoc, outlier, pPBin, coords, weights, queue_); */ - break; + return run8(dc, rhoc, outlier, pPBin, coords, weights, kernel, queue_, block_size); [[unlikely]] case (9) : - /* return run9(dc, rhoc, outlier, pPBin, coords, weights, queue_); */ - break; + return run9(dc, rhoc, outlier, pPBin, coords, weights, kernel, queue_, block_size); [[unlikely]] case (10) : - /* return run10(dc, rhoc, outlier, pPBin, coords, weights, queue_); */ - break; + return run10(dc, rhoc, outlier, pPBin, coords, weights, kernel, queue_, block_size); [[unlikely]] default: std::cout << "This library only works up to 10 dimensions\n"; return {}; - break; } } @@ -103,39 +92,28 @@ namespace alpaka_cuda_async { // Running the clustering algorithm // switch (Ndim) { [[unlikely]] case (1) : - /* return run1(dc, rhoc, outlier, pPBin, coords, weights, queue_); */ - break; + return run1(dc, rhoc, outlier, pPBin, coords, weights, kernel, queue_, block_size); [[likely]] case (2) : return run2(dc, rhoc, outlier, pPBin, coords, weights, kernel, queue_, block_size); - break; [[likely]] case (3) : - /* return run3(dc, rhoc, outlier, pPBin, coords, weights, queue_); */ - break; + return run3(dc, rhoc, outlier, pPBin, coords, weights, kernel, queue_, block_size); [[unlikely]] case (4) : - /* return run4(dc, rhoc, outlier, pPBin, coords, weights, queue_); */ - break; + return run4(dc, rhoc, outlier, pPBin, coords, weights, kernel, queue_, block_size); [[unlikely]] case (5) : - /* return run5(dc, rhoc, outlier, pPBin, coords, weights, queue_); */ - break; + return run5(dc, rhoc, outlier, pPBin, coords, weights, kernel, queue_, block_size); [[unlikely]] case (6) : - /* return run6(dc, rhoc, outlier, pPBin, coords, weights, queue_); */ - break; + return run6(dc, rhoc, outlier, pPBin, coords, weights, kernel, queue_, block_size); [[unlikely]] case (7) : - /* return run7(dc, rhoc, outlier, pPBin, coords, weights, queue_); */ - break; + return run7(dc, rhoc, outlier, pPBin, coords, weights, kernel, queue_, block_size); [[unlikely]] case (8) : - /* return run8(dc, rhoc, outlier, pPBin, coords, weights, queue_); */ - break; + return run8(dc, rhoc, outlier, pPBin, coords, weights, kernel, queue_, block_size); [[unlikely]] case (9) : - /* return run9(dc, rhoc, outlier, pPBin, coords, weights, queue_); */ - break; + return run9(dc, rhoc, outlier, pPBin, coords, weights, kernel, queue_, block_size); [[unlikely]] case (10) : - /* return run10(dc, rhoc, outlier, pPBin, coords, weights, queue_); */ - break; + return run10(dc, rhoc, outlier, pPBin, coords, weights, kernel, queue_, block_size); [[unlikely]] default: std::cout << "This library only works up to 10 dimensions\n"; return {}; - break; } } @@ -157,39 +135,28 @@ namespace alpaka_cuda_async { // Running the clustering algorithm // switch (Ndim) { [[unlikely]] case (1) : - /* return run1(dc, rhoc, outlier, pPBin, coords, weights, queue_); */ - break; + return run1(dc, rhoc, outlier, pPBin, coords, weights, kernel, queue_, block_size); [[likely]] case (2) : return run2(dc, rhoc, outlier, pPBin, coords, weights, kernel, queue_, block_size); - break; [[likely]] case (3) : - /* return run3(dc, rhoc, outlier, pPBin, coords, weights, queue_); */ - break; + return run3(dc, rhoc, outlier, pPBin, coords, weights, kernel, queue_, block_size); [[unlikely]] case (4) : - /* return run4(dc, rhoc, outlier, pPBin, coords, weights, queue_); */ - break; + return run4(dc, rhoc, outlier, pPBin, coords, weights, kernel, queue_, block_size); [[unlikely]] case (5) : - /* return run5(dc, rhoc, outlier, pPBin, coords, weights, queue_); */ - break; + return run5(dc, rhoc, outlier, pPBin, coords, weights, kernel, queue_, block_size); [[unlikely]] case (6) : - /* return run6(dc, rhoc, outlier, pPBin, coords, weights, queue_); */ - break; + return run6(dc, rhoc, outlier, pPBin, coords, weights, kernel, queue_, block_size); [[unlikely]] case (7) : - /* return run7(dc, rhoc, outlier, pPBin, coords, weights, queue_); */ - break; + return run7(dc, rhoc, outlier, pPBin, coords, weights, kernel, queue_, block_size); [[unlikely]] case (8) : - /* return run8(dc, rhoc, outlier, pPBin, coords, weights, queue_); */ - break; + return run8(dc, rhoc, outlier, pPBin, coords, weights, kernel, queue_, block_size); [[unlikely]] case (9) : - /* return run9(dc, rhoc, outlier, pPBin, coords, weights, queue_); */ - break; + return run9(dc, rhoc, outlier, pPBin, coords, weights, kernel, queue_, block_size); [[unlikely]] case (10) : - /* return run10(dc, rhoc, outlier, pPBin, coords, weights, queue_); */ - break; + return run10(dc, rhoc, outlier, pPBin, coords, weights, kernel, queue_, block_size); [[unlikely]] default: std::cout << "This library only works up to 10 dimensions\n"; return {}; - break; } } From 3178d402876aa0e1b32c408c70717fe113f37924 Mon Sep 17 00:00:00 2001 From: sbaldu Date: Fri, 26 Jan 2024 22:20:04 +0100 Subject: [PATCH 12/16] Remove pybind11 from toml --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 11fac177..73d0e7b1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["setuptools>=42","wheel","pybind11","pathlib"] +requires = ["setuptools>=42","wheel","pathlib"] build-backend = "setuptools.build_meta" [tool.autopep8] From 3aef2d7ab6ee297ba8d793ecc6e459558818c3e6 Mon Sep 17 00:00:00 2001 From: sbaldu Date: Fri, 26 Jan 2024 22:22:27 +0100 Subject: [PATCH 13/16] Fix packaging and installation --- CMakeLists.txt | 13 +++++++------ MANIFEST.in | 7 ++++++- setup.py | 27 ++++++++++++++------------- 3 files changed, 27 insertions(+), 20 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 668ae1d8..454523f1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -46,8 +46,9 @@ target_compile_options( -DALPAKA_ACC_CPU_B_SEQ_T_SEQ_ENABLED -DALPAKA_ACC_CPU_B_SEQ_T_SEQ_SYNC_BACKEND) # set output directory -set_target_properties(CLUE_Convolutional_Kernels - PROPERTIES LIBRARY_OUTPUT_DIRECTORY ../CLUEstering/lib/) +set_target_properties( + CLUE_Convolutional_Kernels PROPERTIES LIBRARY_OUTPUT_DIRECTORY + ./lib/CLUEstering/lib/) # CPU Serial compile cpu serial module pybind11_add_module(CLUE_CPU_Serial SHARED @@ -63,7 +64,7 @@ target_compile_options( -DALPAKA_ACC_CPU_B_SEQ_T_SEQ_SYNC_BACKEND) # set output directory set_target_properties(CLUE_CPU_Serial PROPERTIES LIBRARY_OUTPUT_DIRECTORY - ../CLUEstering/lib/) + ./lib/CLUEstering/lib/) find_package(TBB) @@ -81,7 +82,7 @@ if(TBB_FOUND) -DALPAKA_ACC_CPU_B_TBB_T_SEQ_ASYNC_BACKEND) target_link_libraries(CLUE_CPU_TBB PRIVATE TBB::tbb) set_target_properties(CLUE_CPU_TBB PROPERTIES LIBRARY_OUTPUT_DIRECTORY - ../CLUEstering/lib/) + ./lib/CLUEstering/lib/) endif() # check if CUDA is available @@ -126,7 +127,7 @@ if(CMAKE_CUDA_COMPILER) arch=compute_61,code=[sm_61,compute_61]) # set output directory set_target_properties(CLUE_GPU_CUDA PROPERTIES LIBRARY_OUTPUT_DIRECTORY - ../CLUEstering/lib/) + ./lib/CLUEstering/lib/) endif() # GPU HIP check if HIP is available @@ -159,5 +160,5 @@ if(CMAKE_HIP_COMPILER) target_include_directories(CLUE_GPU_HIP PRIVATE ${hip_BASE}/rocrand/include) # set output directory set_target_properties(CLUE_GPU_HIP PROPERTIES LIBRARY_OUTPUT_DIRECTORY - ../CLUEstering/lib/) + ./lib/CLUEstering/lib/) endif() diff --git a/MANIFEST.in b/MANIFEST.in index b435f7e2..5f0a3bfc 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1 +1,6 @@ -graft CLUEstering/include +graft CLUEstering/alpaka/ +graft extern/ +include CMakeLists.txt + +exclude tests/*.py +exclude build/* diff --git a/setup.py b/setup.py index e0db4123..f6e9d10a 100644 --- a/setup.py +++ b/setup.py @@ -1,14 +1,15 @@ import os +import sys from pathlib import Path -from setuptools import Extension, setup, find_packages +from setuptools import setup __version__ = "2.0.0" this_directory = Path(__file__).parent long_description = (this_directory/'README.md').read_text() -print("Now we build with cmake") -os.system("cmake -B build && make -C build") +if sys.argv[1] != 'sdist': + os.system("cmake -B build && make -C build") setup( name="CLUEstering", @@ -16,19 +17,19 @@ author="Simone Balducci", author_email="simone.balducci00@gmail.com", description='''A library that generalizes the original 2-dimensional CLUE - algorithm made at CERN.''', - long_description=long_description, - long_description_content_type='text/markdown', - packages=find_packages(), - install_requires=['scikit-learn','numpy','matplotlib','pandas'], - package_data={'': ['lib/*.so']}, - keywords=['Python','Clustering','Binding'], - python_requires='>=3.7', - classifiers=[ + algorithm made at CERN.''', + long_description=long_description, + long_description_content_type='text/markdown', + packages=['CLUEstering'], + install_requires=['scikit-learn','numpy','matplotlib','pandas'], + package_data={'': []}, + keywords=['Python','Clustering','Binding'], + python_requires='>=3.7', + classifiers=[ 'Intended Audience :: Developers', 'Programming Language :: Python :: 3', 'Operating System :: Unix', 'Operating System :: MacOS :: MacOS X', 'Operating System :: Microsoft :: Windows', - ] + ] ) From f84c3fec5cec028851712ce2c9b3f6270c98f6b2 Mon Sep 17 00:00:00 2001 From: sbaldu Date: Mon, 29 Jan 2024 08:48:54 +0100 Subject: [PATCH 14/16] Link shared object to `CLUEstering/lib folder` --- CMakeLists.txt | 36 +++++++++++++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 454523f1..f0be7194 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -16,7 +16,7 @@ add_subdirectory(extern/pybind11) find_package(Boost 1.75.0) -# if boost is not found, t's fetched from the official boost repository +# if boost is not found, it's fetched from the official boost repository if(NOT Boost_FOUND) include(FetchContent) FetchContent_Declare( @@ -33,6 +33,9 @@ else() set(Boost_PATH ${Boost_INCLUDE_DIRS}) endif() +# create lib directory in CLUEstering folder +execute_process(COMMAND mkdir -p ./CLUEstering/lib) + # Convolutional Kernels compile convolutional kernel module pybind11_add_module(CLUE_Convolutional_Kernels SHARED ./CLUEstering/alpaka/BindingModules/binding_kernels.cc) @@ -49,6 +52,13 @@ target_compile_options( set_target_properties( CLUE_Convolutional_Kernels PROPERTIES LIBRARY_OUTPUT_DIRECTORY ./lib/CLUEstering/lib/) +# create link of shared object to lib folder inside CLUEstering directory +file(GLOB CONV_MODULE + ./build/lib/CLUEstering/lib/CLUE_Convolutional_Kernels*.so) +foreach(CONV_MODULE ${CONV_MODULE}) + execute_process(COMMAND ln -sf ${CONV_MODULE} + ./CLUEstering/lib/CLUE_Convolutional_Kernels.so) +endforeach() # CPU Serial compile cpu serial module pybind11_add_module(CLUE_CPU_Serial SHARED @@ -65,6 +75,12 @@ target_compile_options( # set output directory set_target_properties(CLUE_CPU_Serial PROPERTIES LIBRARY_OUTPUT_DIRECTORY ./lib/CLUEstering/lib/) +# create link of shared object to lib folder inside CLUEstering directory +file(GLOB CONV_MODULE ./build/lib/CLUEstering/lib/CLUE_CPU_Serial*.so) +foreach(CONV_MODULE ${CONV_MODULE}) + execute_process(COMMAND ln -sf ${CONV_MODULE} + ./CLUEstering/lib/CLUE_CPU_Serial.so) +endforeach() find_package(TBB) @@ -83,6 +99,12 @@ if(TBB_FOUND) target_link_libraries(CLUE_CPU_TBB PRIVATE TBB::tbb) set_target_properties(CLUE_CPU_TBB PROPERTIES LIBRARY_OUTPUT_DIRECTORY ./lib/CLUEstering/lib/) + # create link of shared object to lib folder inside CLUEstering directory + file(GLOB CONV_MODULE ./build/lib/CLUEstering/lib/CLUE_CPU_TBB*.so) + foreach(CONV_MODULE ${CONV_MODULE}) + execute_process(COMMAND ln -sf ${CONV_MODULE} + ./CLUEstering/lib/CLUE_CPU_TBB.so) + endforeach() endif() # check if CUDA is available @@ -128,6 +150,12 @@ if(CMAKE_CUDA_COMPILER) # set output directory set_target_properties(CLUE_GPU_CUDA PROPERTIES LIBRARY_OUTPUT_DIRECTORY ./lib/CLUEstering/lib/) + # create link of shared object to lib folder inside CLUEstering directory + file(GLOB CONV_MODULE ./build/lib/CLUEstering/lib/CLUE_GPU_CUDA*.so) + foreach(CONV_MODULE ${CONV_MODULE}) + execute_process(COMMAND ln -sf ${CONV_MODULE} + ./CLUEstering/lib/CLUE_GPU_CUDA.so) + endforeach() endif() # GPU HIP check if HIP is available @@ -161,4 +189,10 @@ if(CMAKE_HIP_COMPILER) # set output directory set_target_properties(CLUE_GPU_HIP PROPERTIES LIBRARY_OUTPUT_DIRECTORY ./lib/CLUEstering/lib/) + # create link of shared object to lib folder inside CLUEstering directory + file(GLOB CONV_MODULE ./build/lib/CLUEstering/lib/CLUE_GPU_HIP*.so) + foreach(CONV_MODULE ${CONV_MODULE}) + execute_process(COMMAND ln -sf ${CONV_MODULE} + ./CLUEstering/lib/CLUE_GPU_HIP.so) + endforeach() endif() From 0f38c8b777f4fdb5e5d606544e9f8ef5ed9a9534 Mon Sep 17 00:00:00 2001 From: sbaldu Date: Wed, 31 Jan 2024 12:28:38 +0100 Subject: [PATCH 15/16] Delete copy of `CLUEstering.py` --- .../alpaka/BindingModules/CLUEstering.py | 925 ------------------ .../{alpaka/BindingModules => }/sissa.csv | 0 2 files changed, 925 deletions(-) delete mode 100644 CLUEstering/alpaka/BindingModules/CLUEstering.py rename CLUEstering/{alpaka/BindingModules => }/sissa.csv (100%) diff --git a/CLUEstering/alpaka/BindingModules/CLUEstering.py b/CLUEstering/alpaka/BindingModules/CLUEstering.py deleted file mode 100644 index 19cdfbbe..00000000 --- a/CLUEstering/alpaka/BindingModules/CLUEstering.py +++ /dev/null @@ -1,925 +0,0 @@ -""" -Density based clustering algorithm developed at CERN. -""" - -from dataclasses import dataclass -from glob import glob -import random as rnd -from math import sqrt -import time -import types -from typing import Union -import matplotlib.pyplot as plt -import numpy as np -import pandas as pd -from sklearn.datasets import make_blobs -from sklearn.preprocessing import StandardScaler -import CLUE_Convolutional_Kernels as clue_kernels -import CLUE_CPU_Serial as cpu_serial -tbb_found = False -if len(glob('./CLUE_CPU_TBB*.so')) != 0: - import CLUE_CPU_TBB as cpu_tbb - tbb_found = True -cuda_found = False -if len(glob('./CLUE_GPU_CUDA*.so')) != 0: - import CLUE_GPU_CUDA as gpu_cuda - cuda_found = True -hip_found = False -if len(glob('./CLUE_GPU_HIP*.so')) != 0: - import CLUE_GPU_HIP as gpu_hip - hip_found = True - -def test_blobs(n_samples: int, n_dim: int , n_blobs: int = 4, mean: float = 0, - sigma: float = 0.5, x_max: float = 30, y_max: float = 30) -> pd.DataFrame: - """ - Returns a dataframe containing randomly generated 2-dimensional or 3-dimensional blobs. - - This functions serves as a tool for generating a random dataset to test the library. - - Parameters - ---------- - n_samples : int - The number of points in the dataset. - n_dim : int - The number of dimensions. - n_blobs : int, optional - The number of blobs that should be produced. By default it is set to 4. - mean : float, optional - The mean of the gaussian distribution of the z values. - sigma : float, optional - The standard deviation of the gaussian distribution of the z values. - x_max : float, optional - Limit of the space where the blobs are created in the x direction. - y_max : float, optional - Limit of the space where the blobs are created in the y direction. - - Returns - ------- - Pandas DataFrame - DataFrame containing n_blobs gaussian blobs. - """ - - if n_blobs < 0: - raise ValueError('Wrong parameter value. The number of blobs must be positive.') - if sigma < 0.: - raise ValueError("Wrong parameter value. The mean and sigma of the blobs" - + " cannot be negative.") - if n_dim > 3: - raise ValueError("Wrong number of dimensions. Blobs can only be generated" - + " in 2 or 3 dimensions.") - centers = [] - if n_dim == 2: - data = {'x0': np.array([]), 'x1': np.array([]), 'weight': np.array([])} - centers = [[x_max * rnd.random(), y_max * rnd.random()] for _ in range(n_blobs)] - blob_data = make_blobs(n_samples=n_samples, centers=np.array(centers))[0] - - data['x0'] = blob_data.T[0] - data['x1'] = blob_data.T[1] - data['weight'] = np.full(shape=len(blob_data.T[0]), fill_value=1) - - - return pd.DataFrame(data) - if n_dim == 3: - data = {'x0': [], 'x1': [], 'x2': [], 'weight': []} - sqrt_samples = int(sqrt(n_samples)) - z_values = np.random.normal(mean, sigma,sqrt_samples) - centers = [[x_max * rnd.random(), y_max * rnd.random()] for _ in range(n_blobs)] - - for value in z_values: # for every z value, a layer is generated. - blob_data = make_blobs(n_samples=sqrt_samples, centers=np.array(centers))[0] - data['x0'] = np.concatenate([data['x0'], blob_data.T[0]]) - data['x1'] = np.concatenate([data['x1'], blob_data.T[1]]) - data['x2'] = np.concatenate([data['x2'], np.full(shape=sqrt_samples, - fill_value=value)]) - data['weight'] = np.concatenate([data['weight'], np.full(shape=sqrt_samples, - fill_value=1)]) - - return pd.DataFrame(data) - -@dataclass() -class clustering_data: - """ - Container characterizing the data used for clustering. - - Attributes - ---------- - coords : np.ndarray - Spatially normalized data coordinates in the coordinate system used for clustering. - original_coords : np.ndarray - Data coordinates in the original coordinate system provided by the user. - weight : np.ndarray - Weight values of the data points. - domain_ranges : list - List containing the ranges of the domains for every coordinate. - n_dim : int - Number of dimensions. - n_points : int - Number of points in the clustering data. - """ - - coords : np.ndarray - original_coords : np.ndarray - weight : np.ndarray - n_dim : int - n_points : int - -@dataclass(eq=False) -class cluster_properties: - """ - Container of the data resulting from the clusterization of the input data. - - Attributes - ---------- - n_clusters : int - Number of clusters constructed. - cluster_ids : np.ndarray - Array containing the cluster_id of each point. - is_seed : np.ndarray - Array of integers containing '1' if a point is a seed and '0' if it isn't - cluster_points : np.ndarray - Array containing, for each cluster, the list of point_ids corresponding to the - clusters bolonging to that cluster. - points_per_cluster : np.ndarray - Array containing the number of points belonging to each cluster. - output_df : pd.DataFrame - Dataframe containing is_seed and cluster_ids as columns. - """ - - n_clusters : int - cluster_ids : np.ndarray - is_seed : np.ndarray - cluster_points : np.ndarray - points_per_cluster : np.ndarray - output_df : pd.DataFrame - - def __eq__(self, other): - if self.n_clusters != other.n_clusters: - return False - if not (self.cluster_ids == other.cluster_ids).all(): - return False - if not (self.is_seed == other.is_seed).all(): - return False - - return True - - -class clusterer: - """ - Class representing a wrapper for the methods using in the process of clustering using - the CLUE algorithm. - - Attributes - ---------- - dc_ : float - Spatial parameter indicating how large is the region over which the local density of - each point is calculated. - rhoc : float - Parameter representing the density threshold value which divides seeds and - outliers. - - Points with a density lower than rhoc can't be seeds, can only be followers - or outliers. - outlier : float - Multiplicative increment of dc_ for getting the region over which the followers of a - point are searched. - - While dc_ determines the size of the search box in which the neighbors of a point are - searched when calculating its local density, when looking for followers while trying - to find potential seeds the size of the search box is given by dm = dc_ * outlier. - ppbin : int - Average number of points to be found in each tile. - kernel : Algo.kernel - Convolution kernel used to calculate the local density of the points. - clust_data : clustering_data - Container of the data used by the clustering algorithm. - clust_prop : cluster_properties - Container of the data produced as output of the algorithm. - elapsed_time : int - Execution time of the algorithm, expressed in nanoseconds. - """ - - def __init__(self, dc_: float, rhoc_: float, outlier_: float, ppbin: int = 10): - self.dc_ = dc_ - self.rhoc = rhoc_ - self.outlier = outlier_ - self.ppbin = ppbin - - # Initialize attributes - ## Data containers - self.clust_data = None - self.scaler = StandardScaler() - - ## Kernel for calculation of local density - self.kernel = clue_kernels.FlatKernel(0.5) - - ## Output attributes - self.clust_prop = None - self.elapsed_time = 0. - - def _read_array(self, input_data: Union[list,np.ndarray]) -> None: - """ - Reads data provided with lists or np.ndarrays - - Attributes - ---------- - input_data : list, np.ndarray - The coordinates and weight values of the data points - - Modified attributes - ------------------- - clust_data : clustering_data - Properties of the input data - - Returns - ------- - None - """ - - if len(input_data) < 2 or len(input_data) > 10: - raise ValueError("Inadequate data. The data must contain at least one coordinate" + - " and the weight.") - self.clust_data = clustering_data(np.asarray(input_data[:-1]), - np.copy(np.asarray(input_data[:-1])), - np.asarray(input_data[-1]), - len(input_data[:-1]), - len(input_data[-1])) - - def _read_string(self, input_data: str) -> Union[pd.DataFrame,None]: - """ - Reads data provided by passing a string containing the path to a csv file - - Attributes - ---------- - input_data : str - The path to the csv file containing the input data - - Modified attributes - ------------------- - None - - Returns - ------------------- - pd.DataFrame - Dataframe containing the input data - """ - - if not input_data.endswith('.csv'): - raise ValueError('Wrong type of file. The file is not a csv file.') - df_ = pd.read_csv(input_data) - return df_ - - def _read_dict_df(self, input_data: Union[dict,pd.DataFrame]) -> pd.DataFrame: - """ - Reads data provided using dictionaries or pandas dataframes - - Attributes - ---------- - input_data : dict, pd.DataFrame - The coordinates and weight values of the data points - - Modified attributes - ------------------- - None - - Returns - ------------------- - pd.DataFrame - Dataframe containing the input data - """ - - df_ = pd.DataFrame(input_data, copy=False) - return df_ - - def _handle_dataframe(self, df_: pd.DataFrame) -> None: - """ - Constructs the clust_data attribute from the dataframe produced by the - _read_string or _read_dict_df methods - - Modified attributes - ------------------- - clust_data : clustering_data - Properties of the input data - - Returns - ------- - None - """ - - # Check that the user provided the weights - if 'weight' not in df_.columns: - raise ValueError("Inadequate data. The input dataframe must" - + " contain a weight column.") - - coordinate_columns = [col for col in df_.columns if col[0] == 'x'] - - # Check that the dimensionality of the dataset is adequate - if len(df_.columns) < 2: - raise ValueError("Inadequate data. The data must contain" - + " at least one coordinate and the weight.") - if len(coordinate_columns) > 10: - raise ValueError("Inadequate data. The maximum number of" - + " dimensions supported is 10.") - n_dim = len(coordinate_columns) - n_points = len(df_.index) - coords = df_.iloc[:, 0:-1].to_numpy() - - self.clust_data = clustering_data(coords, - np.copy(coords), - np.asarray(df_['weight']), - n_dim, - n_points) - - def _rescale(self) -> None: - """ - Normalizes the input data using a standard scaler - - Modified attributes - ------------------- - clust_data.coords : np.ndarray - Array containing the coordinates and weight values of the data points - - Returns - ------- - None - """ - - for dim in range(self.clust_data.n_dim): - self.clust_data.coords.T[dim] = \ - self.scaler.fit_transform(self.clust_data.coords.T[dim].reshape(-1, 1)).reshape(1, -1)[0] - - def read_data(self, - input_data: Union[pd.DataFrame,str,dict,list,np.ndarray]) -> None: - """ - Reads the data in input and fills the class members containing the coordinates - of the points, the weight, the number of dimensions and the number of points. - - Parameters - ---------- - input_data : pandas dataframe - The dataframe should contain one column for every - coordinate, each one called 'x*', and one column for the weight. - input_data : string - The string should contain the full path to a csv file containing - the data. - input_data : dict - input_data : array_like - The list or numpy array should contain a list of lists for the - coordinates and a list for the weight. - kwargs : tuples - Tuples corresponding to the domain of any periodic variables. The - keyword should be the keyword of the corrispoding variable. - - Modified attributes - ------------------- - coords : ndarray - Point coordinates used for clustering, spatially normalized. - original_coords : ndarray - Point coordinates in the original coordinate system used by the user. - weight : ndarray - Weights of all the points. - domain_ranges : list of Algo.domain_t - List of the domains for each coordinate. - n_dim : int - The number of dimensions in which we are calculating the clusters. - n_points : int - The number of points in the dataset. - - Returns - ------- - None - """ - - # lists and np ndarrays - if isinstance(input_data, (list, np.ndarray)): - self._read_array(input_data) - - # path to .csv file or pandas dataframe - if isinstance(input_data, (str)): - df = self._read_string(input_data) - self._handle_dataframe(df) - - if isinstance(input_data, (dict, pd.DataFrame)): - df = self._read_dict_df(input_data) - self._handle_dataframe(df) - - # Rescale the coordinates with a standard scaler - self._rescale() - - def change_coordinates(self, **kwargs: types.FunctionType) -> None: - """ - Change the coordinate system - - Parameters - ---------- - kwargs : function objects - The functions for the change of coordinates. - The keywords of the arguments are the coordinates names (x0, x1, ...). - - Modifies attributes - ------------------- - coords : ndarray - Coordinates used for clustering converted in the chosen coordinate system. - - Returns - ------- - None - """ - - # Change the coordinate system - for coord, func in kwargs.items(): - self.clust_data.coords[int(coord[1])] = func(self.clust_data.original_coords) - - # Normalize the coordinate with a standard scaler - self.clust_data.coords[int(coord[1])] = \ - self.scaler.fit_transform( - self.clust_data.coords[int(coord[1])].reshape(-1, 1) - ).reshape(1, -1)[0] - - def choose_kernel(self, - choice: str, - parameters: Union[list,None] = None, - function: types.FunctionType = lambda: 0) -> None: - """ - Changes the kernel used in the calculation of local density. The default kernel - is a flat kernel with parameter 0.5 - - Parameters - ---------- - choice : string - The type of kernel that you want to choose (flat, exp, gaus or custom). - parameters : array_like, optional - List of the parameters needed by the kernels. - The flat kernel requires one, the exponential requires two - (amplitude and mean), the gaussian requires three (amplitude, - mean and standard deviation) and the custom doesn't require any. - function : function object, optional - Function that should be used as kernel when the custom kernel is chosen. - - Modified attributes - ------------------- - kernel : Algo.kernel - - Return - ------ - None - """ - - if choice == "flat": - if len(parameters) != 1: - raise ValueError("Wrong number of parameters. The flat kernel" - + " requires 1 parameter.") - self.kernel = CLUE_Convolutional_Kernels.FlatKernel(parameters[0]) - elif choice == "exp": - if len(parameters) != 2: - raise ValueError("Wrong number of parameters. The exponential" - + " kernel requires 2 parameters.") - self.kernel = CLUE_Convolutional_Kernels.ExponentialKernel(parameters[0], parameters[1]) - elif choice == "gaus": - if len(parameters) != 3: - raise ValueError("Wrong number of parameters. The gaussian" + - " kernel requires 3 parameters.") - self.kernel = CLUE_Convolutional_Kernels.GaussinKernel(parameters[0], parameters[1], parameters[2]) - elif choice == "custom": - if len(parameters) != 0: - raise ValueError("Wrong number of parameters. Custom kernels" - + " requires 0 parameters.") - else: - raise ValueError("Invalid kernel. The allowed choices for the" - + " kernels are: flat, exp, gaus and custom.") - - def list_devices(self, backend: str = "all") -> None: - """ - Lists the devices available for the chosen backend. - - Parameters - ---------- - backend : string, optional - The backend for which the devices are listed. The allowed values are - 'all', 'cpu serial', 'cpu tbb' and 'gpu cuda'. - The default value is 'all'. - - Raises - ------ - ValueError : If the backend is not valid. - """ - - if backend == "all": - cpu_serial.listDevices('cpu serial') - if tbb_found: - cpu_tbb.listDevices('cpu tbb') - if cuda_found: - gpu_cuda.listDevices('gpu cuda') - if hip_found: - gpu_hip.listDevices('gpu hip') - elif backend == "cpu serial": - cpu_serial.listDevices(backend) - elif backend == "cpu tbb": - if tbb_found: - cpu_tbb.listDevices(backend) - else: - print("TBB module not found. Please re-compile the library and try again.") - elif backend == "gpu cuda": - if cuda_found: - gpu_cuda.listDevices(backend) - else: - print("CUDA module not found. Please re-compile the library and try again.") - elif backend == "gpu hip": - if hip_found: - gpu_hip.listDevices(backend) - else: - print("HIP module not found. Please re-compile the library and try again.") - else: - raise ValueError("Invalid backend. The allowed choices for the" - + " backend are: all, cpu serial, cpu tbb and gpu cuda.") - - def run_clue(self, - backend: str = "cpu serial", - block_size: int = 1024, - device_id: int = 0, - verbose: bool = False) -> None: - """ - Executes the CLUE clustering algorithm. - - Parameters - ---------- - verbose : bool, optional - The verbose option prints the execution time of runCLUE and the number - of clusters found. - - Modified attributes - ------------------- - cluster_ids : ndarray - Contains the cluster_id corresponding to every point. - is_seed : ndarray - For every point the value is 1 if the point is a seed or an - outlier and 0 if it isn't. - n_clusters : int - Number of clusters reconstructed. - cluster_points : ndarray of lists - Contains, for every cluster, the list of points associated to id. - points_per_cluster : ndarray - Contains the number of points associated to every cluster. - - Return - ------ - None - """ - - start = time.time_ns() - if backend == "cpu serial": - cluster_id_is_seed = cpu_serial.mainRun(self.dc_, self.rhoc, self.outlier, self.ppbin, - self.clust_data.coords, self.clust_data.weight, - self.kernel, self.clust_data.n_dim, block_size, - device_id) - elif backend == "cpu tbb": - if tbb_found: - cluster_id_is_seed = cpu_tbb.mainRun(self.dc_, self.rhoc, self.outlier, - self.ppbin, self.clust_data.coords, - self.clust_data.weight, self.kernel, - self.clust_data.n_dim, block_size, - device_id) - else: - print("TBB module not found. Please re-compile the library and try again.") - - elif backend == "gpu cuda": - if cuda_found: - cluster_id_is_seed = gpu_cuda.mainRun(self.dc_, self.rhoc, self.outlier, - self.ppbin, self.clust_data.coords, - self.clust_data.weight, self.kernel, - self.clust_data.n_dim, block_size, - device_id) - else: - print("CUDA module not found. Please re-compile the library and try again.") - - elif backend == "gpu hip": - if hip_found: - cluster_id_is_seed = gpu_hip.mainRun(self.dc_, self.rhoc, self.outlier, - self.ppbin, self.clust_data.coords, - self.clust_data.weight, self.kernel, - self.clust_data.n_dim, block_size, - device_id) - else: - print("HIP module not found. Please re-compile the library and try again.") - - finish = time.time_ns() - cluster_ids = np.array(cluster_id_is_seed[0]) - is_seed = np.array(cluster_id_is_seed[1]) - n_clusters = len(np.unique(cluster_ids)) - - cluster_points = [[] for _ in range(n_clusters)] - for i in range(self.clust_data.n_points): - cluster_points[cluster_ids[i]].append(i) - - points_per_cluster = np.array([len(clust) for clust in cluster_points]) - - data = {'cluster_ids': cluster_ids, 'is_seed': is_seed} - output_df = pd.DataFrame(data) - - self.clust_prop = cluster_properties(n_clusters, - cluster_ids, - is_seed, - np.asarray(cluster_points, dtype=object), - points_per_cluster, - output_df) - - self.elapsed_time = (finish - start)/(10**6) - if verbose: - print(f'CLUE executed in {self.elapsed_time} ms') - print(f'Number of clusters found: {self.clust_prop.n_clusters}') - - def input_plotter(self, plot_title: str='', title_size: float = 16, - x_label: str = 'x', y_label: str = 'y', z_label: str = 'z', - label_size: float = 16, pt_size: float = 1, pt_colour: str = 'b', - grid: bool = True, grid_style: str = '--', grid_size: float = 0.2, - x_ticks=None, y_ticks=None, z_ticks=None, - **kwargs) -> None: - """ - Plots the points in input. - - Parameters - ---------- - plot_title : string, optional - Title of the plot. - title_size : float, optional - Size of the plot title. - x_label : string, optional - Label on x-axis. - y_label : string, optional - Label on y-axis. - z_label : string, optional - Label on z-axis. - label_size : int, optional - Fontsize of the axis labels. - pt_size : int, optional - Size of the points in the plot. - pt_colour : string, optional - Colour of the points in the plot. - grid : bool, optional - If true displays grids in the plot. - grid_style : string, optional - Style of the grid. - grid_size : float, optional - Linewidth of the plot grid. - x_ticks : list, optional - List of ticks for the x axis. - y_ticks : list, optional - List of ticks for the y axis. - z_ticks : list, optional - List of ticks for the z axis. - kwargs : function objects, optional - Functions for converting the used coordinates to cartesian coordinates. - The keywords of the arguments are the coordinates names (x0, x1, ...). - - Modified attributes - ------------------- - None - - Returns - ------- - None - """ - - # Convert the used coordinates to cartesian coordiantes - cartesian_coords = np.copy(self.clust_data.original_coords) - for coord, func in kwargs.items(): - cartesian_coords[int(coord[1])] = func(self.clust_data.original_coords) - - if self.clust_data.n_dim == 2: - plt.scatter(cartesian_coords[0], - cartesian_coords[1], - s=pt_size, - color=pt_colour) - - # Customization of the plot title - plt.title(plot_title, fontsize=title_size) - - # Customization of axis labels - plt.xlabel(x_label, fontsize=label_size) - plt.ylabel(y_label, fontsize=label_size) - - # Customization of the grid - if grid: - plt.grid(linestyle=grid_style, linewidth=grid_size) - - # Customization of axis ticks - if x_ticks is not None: - plt.xticks(x_ticks) - if y_ticks is not None: - plt.yticks(y_ticks) - - plt.show() - if self.clust_data.n_dim >= 3: - fig = plt.figure() - ax_ = fig.add_subplot(projection='3d') - ax_.scatter(cartesian_coords[0], - cartesian_coords[1], - cartesian_coords[2], - s=pt_size, - color=pt_colour) - - # Customization of the plot title - ax_.set_title(plot_title, fontsize=title_size) - - # Customization of axis labels - ax_.set_xlabel(x_label, fontsize=label_size) - ax_.set_ylabel(y_label, fontsize=label_size) - ax_.set_zlabel(z_label, fontsize=label_size) - - # Customization of the grid - if grid: - plt.grid(linestyle=grid_style, linewidth=grid_size) - - # Customization of axis ticks - if x_ticks is not None: - ax_.set_xticks(x_ticks) - if y_ticks is not None: - ax_.set_yticks(y_ticks) - if z_ticks is not None: - ax_.set_zticks(z_ticks) - - plt.show() - - def cluster_plotter(self, plot_title: str = '', title_size: float = 16, - x_label: str = 'x', y_label: str = 'y', z_label: str = 'z', - label_size: float = 16, outl_size: float = 10, pt_size: float = 10, - seed_size: float = 25, grid: bool = True, grid_style: str = '--', - grid_size: float = 0.2, x_ticks=None, y_ticks=None, z_ticks=None, - **kwargs) -> None: - """ - Plots the clusters with a different colour for every cluster. - - The points assigned to a cluster are printed as points, the seeds - as stars and the outliers as little grey crosses. - - Parameters - ---------- - plot_title : string, optional - Title of the plot - title_size : float, optional - Size of the plot title - x_label : string, optional - Label on x-axis - y_label : string, optional - Label on y-axis - z_label : string, optional - Label on z-axis - label_size : int, optional - Fontsize of the axis labels - outl_size : int, optional - Size of the outliers in the plot - pt_size : int, optional - Size of the points in the plot - seed_size : int, optional - Size of the seeds in the plot - grid : bool, optional - f true displays grids in the plot - grid_style : string, optional - Style of the grid - grid_size : float, optional - Linewidth of the plot grid - x_ticks : list, optional - List of ticks for the x axis. - y_ticks : list, optional - List of ticks for the y axis. - z_ticks : list, optional - List of ticks for the z axis. - kwargs : function objects, optional - Functions for converting the used coordinates to cartesian coordinates. - The keywords of the arguments are the coordinates names (x0, x1, ...). - - Modified attributes - ------------------- - None - - Returns - ------- - None - """ - - # Convert the used coordinates to cartesian coordiantes - cartesian_coords = np.copy(self.clust_data.original_coords.T) - for coord, func in kwargs.items(): - cartesian_coords[int(coord[1])] = func(self.clust_data.original_coords) - - if self.clust_data.n_dim == 2: - data = {'x0': cartesian_coords[0], - 'x1': cartesian_coords[1], - 'cluster_ids': self.clust_prop.cluster_ids, - 'isSeed': self.clust_prop.is_seed} - df_ = pd.DataFrame(data) - - max_clusterid = max(df_["cluster_ids"]) - - df_out = df_[df_.cluster_ids == -1] # Outliers - plt.scatter(df_out.x0, df_out.x1, s=outl_size, marker='x', color='0.4') - for i in range(0, max_clusterid+1): - dfi = df_[df_.cluster_ids == i] # ith cluster - plt.scatter(dfi.x0, dfi.x1, s=pt_size, marker='.') - df_seed = df_[df_.isSeed == 1] # Only Seeds - plt.scatter(df_seed.x0, df_seed.x1, s=seed_size, color='r', marker='*') - - # Customization of the plot title - plt.title(plot_title, fontsize=title_size) - - # Customization of axis labels - plt.xlabel(x_label, fontsize=label_size) - plt.ylabel(y_label, fontsize=label_size) - - # Customization of the grid - if grid: - plt.grid(linestyle=grid_style, linewidth=grid_size) - - # Customization of axis ticks - if x_ticks is not None: - plt.xticks(x_ticks) - if y_ticks is not None: - plt.yticks(y_ticks) - - plt.show() - if self.clust_data.n_dim == 3: - data = {'x0': cartesian_coords[0], - 'x1': cartesian_coords[1], - 'x2': cartesian_coords[2], - 'cluster_ids': self.clust_prop.cluster_ids, - 'isSeed': self.clust_prop.is_seed} - df_ = pd.DataFrame(data) - - max_clusterid = max(df_["cluster_ids"]) - fig = plt.figure() - ax_ = fig.add_subplot(projection='3d') - - df_out = df_[df_.cluster_ids == -1] - ax_.scatter(df_out.x0, df_out.x1, df_out.x2, s=outl_size, color = 'grey', marker = 'x') - for i in range(0, max_clusterid+1): - dfi = df_[df_.cluster_ids == i] - ax_.scatter(dfi.x0, dfi.x1, dfi.x2, s=pt_size, marker = '.') - - df_seed = df_[df_.isSeed == 1] # Only Seeds - ax_.scatter(df_seed.x0, df_seed.x1, df_seed.x2, s=seed_size, color = 'r', marker = '*') - - # Customization of the plot title - ax_.set_title(plot_title, fontsize=title_size) - - # Customization of axis labels - ax_.set_xlabel(x_label, fontsize=label_size) - ax_.set_ylabel(y_label, fontsize=label_size) - ax_.set_zlabel(z_label, fontsize=label_size) - - # Customization of the grid - if grid: - plt.grid(linestyle=grid_style, linewidth=grid_size) - - # Customization of axis ticks - if x_ticks is not None: - ax_.set_xticks(x_ticks) - if y_ticks is not None: - ax_.set_yticks(y_ticks) - if z_ticks is not None: - ax_.set_zticks(z_ticks) - - plt.show() - - def to_csv(self, output_folder: str, file_name: str) -> None: - """ - Creates a file containing the coordinates of all the points, their cluster_ids and is_seed. - - Parameters - ---------- - output_folder : string - Full path to the desired ouput folder. - file_name : string - Name of the file, with the '.csv' suffix. - - Modified attributes - ------------------- - None - - Returns - ------- - None - """ - - out_path = output_folder + file_name - data = {} - for i in range(self.clust_data.n_dim): - data['x' + str(i)] = self.clust_data.coords.T[i] - data['cluster_ids'] = self.clust_prop.cluster_ids - data['is_seed'] = self.clust_prop.is_seed - - df_ = pd.DataFrame(data) - df_.to_csv(out_path,index=False) - -if __name__ == "__main__": - c = clusterer(0.4,5,1.) - c.read_data('./sissa.csv') - c.run_clue(backend="cpu serial", verbose=True) - # c.run_clue(backend="cpu tbb", verbose=True) - # c.run_clue(backend="gpu cuda", verbose=True) - # c.run_clue(backend="gpu hip", verbose=True) - c.cluster_plotter() - # c.to_csv('./','sissa_output_tbb.csv') - c.list_devices('cpu serial') - c.list_devices('cpu tbb') - c.list_devices('gpu cuda') - c.list_devices('gpu hip') - c.list_devices() diff --git a/CLUEstering/alpaka/BindingModules/sissa.csv b/CLUEstering/sissa.csv similarity index 100% rename from CLUEstering/alpaka/BindingModules/sissa.csv rename to CLUEstering/sissa.csv From d77beadcf5fe2ebcb409248ff7c83a0f5c68092c Mon Sep 17 00:00:00 2001 From: sbaldu Date: Mon, 5 Feb 2024 11:26:34 +0100 Subject: [PATCH 16/16] Cleaner check for existence of modules --- CLUEstering/CLUEstering.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/CLUEstering/CLUEstering.py b/CLUEstering/CLUEstering.py index 3bdcaa81..85aa9997 100644 --- a/CLUEstering/CLUEstering.py +++ b/CLUEstering/CLUEstering.py @@ -14,24 +14,21 @@ import pandas as pd from sklearn.datasets import make_blobs from sklearn.preprocessing import StandardScaler -from os.path import dirname, join +from os.path import dirname, exists, join path = dirname(__file__) import sys sys.path.insert(1, join(path, 'lib')) import CLUE_Convolutional_Kernels as clue_kernels import CLUE_CPU_Serial as cpu_serial -tbb_found = False -if len(glob(join(path, 'lib/CLUE_CPU_TBB*.so'))) != 0: +tbb_found = exists(*glob(join(path, 'lib/CLUE_CPU_TBB*.so'))) +if tbb_found: import CLUE_CPU_TBB as cpu_tbb - tbb_found = True -cuda_found = False -if len(glob(join(path, 'lib/CLUE_GPU_CUDA*.so'))) != 0: +cuda_found = exists(*glob(join(path, 'lib/CLUE_CPU_TBB*.so'))) +if cuda_found: import CLUE_GPU_CUDA as gpu_cuda - cuda_found = True -hip_found = False -if len(glob(join(path, 'lib/CLUE_GPU_HIP*.so'))) != 0: +hip_found = exists(*glob(join(path, 'lib/CLUE_CPU_TBB*.so'))) +if hip_found: import CLUE_GPU_HIP as gpu_hip - hip_found = True def test_blobs(n_samples: int, n_dim: int , n_blobs: int = 4, mean: float = 0, sigma: float = 0.5, x_max: float = 30, y_max: float = 30) -> pd.DataFrame: