Add support for SYCL/Data Parallel C++ (DPCPP)

.gitignore

@@ -9,6 +9,11 @@ host/inc/stringify.h
 final_population_run*
 device/stringify_tmp
 KernelProgramBuildInfo.txt
+run.sh
+run_batch.sh
+gdb.txt
+performdocking.h
+performdocking.cpp
 
 # ===================
 # C gitignore

Makefile

@@ -1,4 +1,5 @@
 # AutoDock-GPU Makefile
+# Copyright (C) 2022 Intel Corporation
 
 # ------------------------------------------------------
 # Note that environment variables must be defined
@@ -20,9 +21,11 @@
 # in any other case, OpenCL will be used
 # OpenCL GPU path can be explicitly used with
 # DEVICE=OCLGPU
+# Choose Xe/DPC++ Device
+# DEVICE=XeGPU
 # ------------------------------------------------------
 # Choose OpenCL device
-# Valid values: CPU, GPU, CUDA, OCLGPU
+# Valid values: CPU, GPU, CUDA, OCLGPU, XeGPU
 
 ifeq ($(DEVICE), $(filter $(DEVICE),GPU CUDA))
 TEST_CUDA := $(shell ./test_cuda.sh nvcc "$(GPU_INCLUDE_PATH)" "$(GPU_LIBRARY_PATH)")
@@ -37,9 +40,23 @@ override DEVICE:=GPU
 export
 include Makefile.Cuda
 else
+ifeq ($(DEVICE),XeGPU)
+override DEVICE:=GPU
+include Makefile.dpcpp
+export
+else
+# run SYCL version on NVidia GPU
+ifeq ($(DEVICE),NvGPU)
+override DEVICE:=GPU
+PLATFORM=NvGPU
+include Makefile.dpcpp
+export
+else
 ifeq ($(DEVICE),OCLGPU)
 override DEVICE:=GPU
 export
 endif
 include Makefile.OpenCL
 endif
+endif
+endif

Makefile.Cuda

@@ -15,7 +15,7 @@ NVCC = nvcc
 CPP = g++
 UNAME := $(shell uname)
 
-TARGETS = 52 60 61 70
+TARGETS = 52 60 61 70 80 90
 CUDA_TARGETS=$(foreach target,$(TARGETS),-gencode arch=compute_$(target),code=sm_$(target))
 
 ifeq ($(DEVICE), CPU)
@@ -171,8 +171,8 @@ check-env-all: check-env-dev check-env-cpu check-env-gpu
 
 GIT_VERSION := $(shell ./version_string.sh)
 
-CFLAGS+=-DVERSION=\"$(GIT_VERSION)\"
-TOOL_CFLAGS+=-DVERSION=\"$(GIT_VERSION)\"
+CFLAGS+=-DAD_VERSION=\"$(GIT_VERSION)\"
+TOOL_CFLAGS+=-DAD_VERSION=\"$(GIT_VERSION)\"
 
 # ------------------------------------------------------
 

Makefile.OpenCL

@@ -235,8 +235,9 @@ check-env-all: check-env-dev check-env-cpu check-env-gpu
 
 GIT_VERSION := $(shell ./version_string.sh)
 
-CFLAGS+=-DVERSION=\"$(GIT_VERSION)\"
-TOOL_CFLAGS+=-DVERSION=\"$(GIT_VERSION)\"
+CFLAGS+=-DAD_VERSION=\"$(GIT_VERSION)\"
+TOOL_CFLAGS+=-DAD_VERSION=\"$(GIT_VERSION)\"
+
 
 # ------------------------------------------------------
 

Makefile.dpcpp

@@ -0,0 +1,286 @@
+## AutoDock-GPU DPCPP Makefile
+# Copyright (C) 2022 Intel Corporation
+
+# ------------------------------------------------------
+# Note that environment variables must be defined
+# before compiling
+# DEVICE?
+# if DEVICE=CPU: CPU_INCLUDE_PATH?, CPU_LIBRARY_PATH?
+# if DEVICE=GPU: GPU_INCLUDE_PATH?, GPU_LIBRARY_PATH?
+
+# Choose Xe/DPC++ Device
+# DEVICE=XeGPU
+# ------------------------------------------------------
+# Choose OpenCL device
+# Valid values: CPU, GPU
+
+DPCPP = icpx -fsycl
+CPP = icpx -fsycl
+UNAME := $(shell uname)
+
+# Assume $ONEAPIPATH/setvars.sh has been run
+
+
+ifeq ($(DEVICE), CPU)
+	DEV =-DCPU_DEVICE
+else ifeq ($(DEVICE), GPU)
+	DEV =-DGPU_DEVICE
+	ifeq ($(PLATFORM), NvGPU)
+		DPCPP=clang++
+		CPP=clang++
+		IFLAGS=-fsycl -fsycl-targets=nvptx64-nvidia-cuda -fgpu-inline-threshold=100000 -Xsycl-target-backend --cuda-gpu-arch=sm_80 \
+		-Xcuda-ptxas --verbose -Xcuda-ptxas --maxrregcount=64 \
+		--cuda-path=${CUDA_TOOLKIT_ROOT_DIR}
+	endif
+endif
+
+# ------------------------------------------------------
+# Project directories
+# opencl_lvs: wrapper for OpenCL APIs
+COMMON_DIR=./common
+HOST_INC_DIR=./host/inc
+HOST_SRC_DIR=./host/src
+KRNL_DIR=./dpcpp
+KCMN_DIR=$(COMMON_DIR)
+BIN_DIR=./bin
+LIB_XeGPU = kernels.dp.o
+
+TARGET := autodock
+TOOL_TARGET := adgpu_analysis
+
+IFLAGS+=-cl-fast-relaxed-math -cl-single-precision-constant
+IFLAGS+=-I$(COMMON_DIR) -I$(HOST_INC_DIR) -I$(KRNL_DIR)
+# set DPCT_INCUDE_PATH in the environment if DPCT helper functions are used for SYCL on non Intel platforms
+ifdef DPCT_INCLUDE_PATH
+	IFLAGS+=-I$(DPCT_INCLUDE_PATH)
+endif
+LFLAGS=-Wl,-rpath=$(GPU_LIBRARY_PATH):$(CPU_LIBRARY_PATH)
+CFLAGS= $(IFLAGS) $(LFLAGS)
+TOOL_CFLAGS=-std=c++17 -I$(COMMON_DIR) -I$(HOST_INC_DIR)
+
+ifeq ($(DEVICE), CPU)
+	TARGET:=$(TARGET)_cpu
+else ifeq ($(DEVICE), GPU)
+	NWI=-DN64WI
+	TARGET:=$(TARGET)_xegpu
+endif
+
+ifeq ($(OVERLAP), ON)
+	PIPELINE=-DUSE_PIPELINE -fopenmp
+endif
+
+
+BIN := $(wildcard $(TARGET)*)
+
+# ------------------------------------------------------
+# Number of work-items (wi)
+# Valid values: 32, 64, 128, 256
+NUMWI=
+
+ifeq ($(NUMWI), 8)
+	NWI=-DN8WI
+	TARGET:=$(TARGET)_8wi
+else ifeq ($(NUMWI), 16)
+	NWI=-DN16WI
+	TARGET:=$(TARGET)_16wi
+else ifeq ($(NUMWI), 32)
+	NWI=-DN32WI
+	TARGET:=$(TARGET)_32wi
+else ifeq ($(NUMWI), 64)
+	NWI=-DN64WI
+	TARGET:=$(TARGET)_64wi
+else ifeq ($(NUMWI), 128)
+	NWI=-DN128WI
+	TARGET:=$(TARGET)_128wi
+else ifeq ($(NUMWI), 256)
+		NWI=-DN256WI
+		TARGET:=$(TARGET)_256wi
+else ifeq ($(NUMWI), 512)
+		NWI=-DN512WI
+		TARGET:=$(TARGET)_512wi
+else ifeq ($(NUMWI), 1024)
+		NWI=-DN1024WI
+		TARGET:=$(TARGET)_1024wi
+else
+	ifeq ($(DEVICE), CPU)
+		NWI=-DN16WI
+		TARGET:=$(TARGET)_16wi
+	else ifeq ($(DEVICE), GPU)
+		NWI=-DN64WI
+		TARGET:=$(TARGET)_64wi
+	endif
+endif
+
+# ------------------------------------------------------
+# Configuration
+# FDEBUG (full) : enables debugging on both host + device
+# LDEBUG (light): enables debugging on host
+# RELEASE
+CONFIG=RELEASE
+#CONFIG=FDEBUG
+
+ifeq ($(CONFIG),FDEBUG)
+	OPT =-g -Wall -DDOCK_DEBUG
+	ifeq ($(DEVICE), GPU)
+	# for AOT compile and debug
+	#	OPT+=-fsycl-targets=spir64_gen-unknown-unknown-sycldevice -Xs "-device ats -internal_options -cl-kernel-debug-enable -options -cl-opt-disable"
+	# for AOT compile
+	#	OPT+=-fsycl-targets=spir64_gen-unknown-unknown-sycldevice -Xs "-device ats"
+	endif    
+else ifeq ($(CONFIG),FDEBUG_VTUNE)
+	OPT =-g -Wall -DDOCK_DEBUG -gline-tables-only -fdebug-info-for-profiling
+else ifeq ($(CONFIG),LDEBUG_VTUNE)
+	OPT =-g -Wall -gline-tables-only -fdebug-info-for-profiling	
+else ifeq ($(CONFIG),LDEBUG)
+	OPT =-g -Wall 
+else ifeq ($(CONFIG),RELEASE)
+	OPT =-O3
+	ifeq ($(PLATFORM),PVC)
+		OPT+=-fsycl-targets=spir64_gen -Xs "-device 0x0BD6 -revision_id 7"
+	else ifeq ($(PLATFORM),ICX)
+		OPT+=-fsycl-targets=spir64_gen -Xs "-device icllp"
+	endif
+else
+	OPT =
+endif
+
+# ------------------------------------------------------
+# Reproduce results (remove randomness)
+REPRO=NO
+
+ifeq ($(REPRO),YES)
+	REP =-DREPRO
+else
+	REP =
+endif
+# ------------------------------------------------------
+
+#all: otool odock
+all: odock
+
+check-env-dev:
+	@if test -z "$$DEVICE"; then \
+		echo "Please set DEVICE to either CPU, GPU, CUDA, XeGPU, or OCLGPU to build docking software."; \
+		exit 1; \
+	else \
+		if [ "$$DEVICE" = "CPU" ]; then \
+			echo "DEVICE is set to $$DEVICE"; \
+		else \
+			if [ "$$DEVICE" = "GPU" ]; then \
+				echo "DEVICE is set to $$DEVICE"; \
+			else \
+				echo "DEVICE value is invalid. Please set DEVICE to either CPU, GPU, XeGPU or OCLGPU"; \
+				exit 1; \
+			fi; \
+		fi; \
+	fi; \
+	echo " "
+
+check-env-cpu:
+	@if test -z "$$CPU_INCLUDE_PATH"; then \
+		echo "CPU_INCLUDE_PATH is undefined"; \
+	else \
+		echo "CPU_INCLUDE_PATH is set to $$CPU_INCLUDE_PATH"; \
+	fi; \
+	if test -z "$$CPU_LIBRARY_PATH"; then \
+		echo "CPU_LIBRARY_PATH is undefined"; \
+	else \
+		echo "CPU_LIBRARY_PATH is set to $$CPU_LIBRARY_PATH"; \
+	fi; \
+	echo " "
+
+check-env-gpu:
+	@if test -z "$$GPU_INCLUDE_PATH"; then \
+		echo "GPU_INCLUDE_PATH is undefined"; \
+	else \
+		echo "GPU_INCLUDE_PATH is set to $$GPU_INCLUDE_PATH"; \
+	fi; \
+	if test -z "$$GPU_LIBRARY_PATH"; then \
+		echo "GPU_LIBRARY_PATH is undefined"; \
+	else \
+		echo "GPU_LIBRARY_PATH is set to $$GPU_LIBRARY_PATH"; \
+	fi; \
+	echo " "
+
+check-env-all: check-env-dev check-env-cpu check-env-gpu
+
+# ------------------------------------------------------
+# Printing out its git version hash
+
+GIT_VERSION := $(shell ./version_string.sh)
+
+CFLAGS+=-DAD_VERSION=\"$(GIT_VERSION)\"
+TOOL_CFLAGS+=-DAD_VERSION=\"$(GIT_VERSION)\"
+
+# ------------------------------------------------------
+
+link-code:
+	ln -sf performdocking.h.dpcpp $(HOST_INC_DIR)/performdocking.h
+	ln -sf performdocking.cpp.dpcpp $(HOST_SRC_DIR)/performdocking.cpp
+
+unlink-code:
+	rm -f $(HOST_INC_DIR)/performdocking.h $(HOST_SRC_DIR)/performdocking.cpp
+
+kernels: $(KERNEL_SRC)
+	$(DPCPP) $(NWI) $(REP) $(DPCPP_FLAGS) $(IFLAGS) $(OPT) $(DPCPP_INCLUDES) -c $(KRNL_DIR)/kernels.dp.cpp
+
+otool: unlink-code
+	@echo "Building" $(TOOL_TARGET) "..."
+	$(CPP) \
+	$(shell ls $(HOST_SRC_DIR)/*.cpp) \
+	$(TOOL_CFLAGS) \
+	-o$(BIN_DIR)/$(TOOL_TARGET) \
+	$(PIPELINE) $(OPT) -DTOOLMODE $(REP)
+
+odock: check-env-all kernels link-code
+	@echo "Building" $(TARGET) "..."
+	$(CPP) \
+	$(shell ls $(HOST_SRC_DIR)/*.cpp) \
+	$(CFLAGS) \
+	$(LIB_XeGPU) \
+	-o$(BIN_DIR)/$(TARGET) \
+	$(DEV) $(NWI) $(PIPELINE) $(OPT) $(DD) $(REP) $(KFLAGS)
+
+# Example
+# 1ac8: for testing gradients of translation and rotation genes
+# 7cpa: for testing gradients of torsion genes (15 torsions) 
+# 3tmn: for testing gradients of torsion genes (1 torsion)
+
+PDB      := 3ce3
+NRUN     := 100
+NGEN     := 27000
+POPSIZE  := 150
+TESTNAME := test
+TESTLS   := sw
+
+test: odock
+	$(BIN_DIR)/$(TARGET) \
+	-ffile ./input/$(PDB)/derived/$(PDB)_protein.maps.fld \
+	-lfile ./input/$(PDB)/derived/$(PDB)_ligand.pdbqt \
+	-nrun $(NRUN) \
+	-ngen $(NGEN) \
+	-psize $(POPSIZE) \
+	-resnam $(TESTNAME) \
+	-gfpop 0 \
+	-lsmet $(TESTLS)
+
+ASTEX_PDB := 2bsm
+ASTEX_NRUN:= 10
+ASTEX_POPSIZE := 10
+ASTEX_TESTNAME := test_astex
+ASTEX_LS := sw
+
+astex: odock
+	$(BIN_DIR)/$(TARGET) \
+	-ffile ./input_tsri/search-set-astex/$(ASTEX_PDB)/protein.maps.fld \
+	-lfile ./input_tsri/search-set-astex/$(ASTEX_PDB)/flex-xray.pdbqt \
+	-nrun $(ASTEX_NRUN) \
+	-psize $(ASTEX_POPSIZE) \
+	-resnam $(ASTEX_TESTNAME) \
+	-gfpop 1 \
+	-lsmet $(ASTEX_LS)
+
+#	$(BIN_DIR)/$(TARGET) -ffile ./input_tsri/search-set-astex/$(ASTEX_PDB)/protein.maps.fld -lfile ./input_tsri/search-set-astex/$(ASTEX_PDB)/flex-xray.pdbqt -nrun $(ASTEX_NRUN) -psize $(ASTEX_POPSIZE) -resnam $(ASTEX_TESTNAME) -gfpop 1 | tee ./input_tsri/search-set-astex/intrapairs/$(ASTEX_PDB)_intrapair.txt
+
+clean:
+	rm -f $(HOST_INC_DIR)/performdocking.h $(HOST_SRC_DIR)/performdocking.cpp