Skip to content

Commit

Permalink
2.0.0-rc updates (#20)
Browse files Browse the repository at this point in the history
Update to newer FLAMEGPU2 api (2.0.0-rc), and update benchmark data to V100 CUDA 11.0 executed on Bessemer.

Also includes A100 data for an earlier build, but the nodes were not available for 2.0.0-rc benchmarking.
  • Loading branch information
ptheywood committed Jan 27, 2023
1 parent 1e2ad4a commit 18f4baa
Show file tree
Hide file tree
Showing 72 changed files with 673,207 additions and 122 deletions.
4 changes: 2 additions & 2 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
# Images etc.

*.png
*.csv

*.eps
*.pdf

# Model output files
*.xml
Expand Down
29 changes: 18 additions & 11 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -1,21 +1,31 @@
cmake_minimum_required(VERSION VERSION 3.18 FATAL_ERROR)
# Minimum CMake version 3.18 for CUDA --std=c++17
cmake_minimum_required(VERSION 3.18...3.25 FATAL_ERROR)

# Name the project and set languages
project(circles-benchmarking CUDA CXX)
# Optionaly set the version of flamegpu which should be used, ideally a tag (i.e. `v2.0.0-rc`) or branch name, or potentially a commit hash.
set(FLAMEGPU_VERSION "v2.0.0-rc" CACHE STRING "FLAMEGPU/FLAMEGPU2 git branch or tag to use")
# If the above version is a hash instead, also set FLAMEGPU_VERSION_ALLOW_HASH to ON
# set(FLAMEGPU_VERSION_ALLOW_HASH "ON")

# Optionaly set the version of flamegpu which should be used.
# Use a branch name such as `master`, or a tagged version such as `v2.0.0-alpha`
set(FLAMEGPU_VERSION "v2.0.0-alpha.2" CACHE STRING "Git branch or tag to use")
# Manually specify the FLAMEGPU_VISUALISATION option to provide it prior to original configuration and allow the default to be overridden in the downstream project
option(FLAMEGPU_VISUALISATION "Enable FLAMEGPU visualisation support" OFF)

# Our core dependency is FLAMEGPU2 lib, first lets find it
include(${CMAKE_CURRENT_LIST_DIR}/cmake/flamegpu2.cmake)

# Include common rules.
# Handle CMAKE_CUDA_ARCHITECTURES gracefully, passign the project name for code-injection
include(${FLAMEGPU_ROOT}/cmake/CUDAArchitectures.cmake)
flamegpu_init_cuda_architectures(PROJECT circles-benchmark)

# Name the project and set languages, this must be done after flamegpu_init_cuda_architectures
project(circles-benchmark CUDA CXX)

# Include common rules from the FLAMEGPU/FLAMEGPU2 repositories CMake
include(${FLAMEGPU_ROOT}/cmake/common.cmake)

# Define output location of binary files
SET(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/bin/${CMAKE_BUILD_TYPE}/)


# Prepare list of source files
# Can't do this automatically, as CMake wouldn't know when to regen (as CMakeLists.txt would be unchanged)
SET(ALL_SRC
Expand All @@ -28,11 +38,8 @@ SET(ALL_SRC
${CMAKE_CURRENT_SOURCE_DIR}/src/main.cu
)

# Option to enable/disable building the static library
option(VISUALISATION "Enable visualisation support" OFF)

# Add the executable and set required flags for the target
add_flamegpu_executable("${PROJECT_NAME}" "${ALL_SRC}" "${FLAMEGPU_ROOT}" "${PROJECT_BINARY_DIR}" TRUE)
flamegpu_add_executable("${PROJECT_NAME}" "${ALL_SRC}" "${FLAMEGPU_ROOT}" "${PROJECT_BINARY_DIR}" TRUE)

# Also set as startup project
set_property(DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" PROPERTY VS_STARTUP_PROJECT "${PROJECT_NAME}")
Expand Down
40 changes: 19 additions & 21 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -11,36 +11,35 @@ For a more complete description of the model, see:

Three experiments are undertaken within this benchmark. There is a range of raw data in the [`sample/data`](sample/data) directory with a description of the machine configurations used to generate it in each directory.

The results below are from the V100 runs on the Bessemer HPC system at the University of Sheffield. Job submission scripts are included in the [`scripts/slurm`]() folder.
The results below are from the V100 runs on the Bessemer HPC system at the University of Sheffield. Job submission scripts are included in the [`scripts/slurm`](scripts/slurm/) directory.

A combined figure for use in publication is shown below. For details please refer to the publication.

[![Combined Benchmark Figure](sample/figures/v100-470.82.01/alpha.2-v100-11.0-beltsoff/paper_figure.png)](sample/figures/v100-470.82.01/alpha.2-v100-11.0-beltsoff/paper_figure.png)
[![Combined Benchmark Figure](sample/figures/v100-515.65.01/2.0.0-rc-v100-11.0-beltsoff/paper_figure.png)](sample/figures/v100-515.65.01/2.0.0-rc-v100-11.0-beltsoff/paper_figure.png)

### Fixed Density Benchmark

+ Communication Radius is fixed to `2.0`
+ Agent Density is fixed to `1` agent per unit of volume
+ Environment Volume is varied, with values of up to `1000000` units of volume
+ 4 Implementations are compared
+ Bruteforce messaging
+ Bruteforce messaging with RTC (run time compilation)
+ Spatial3D messaging
+ Spatial3D messaging with RTC (run time compilation)
+ Bruteforce messaging
+ Bruteforce messaging with RTC (run time compilation)
+ Spatial3D messaging
+ Spatial3D messaging with RTC (run time compilation)

[![Fixed Density Benchmark](sample/figures/v100-470.82.01/alpha.2-v100-11.0-beltsoff/fixed-density--volume--step-s--model--all.png)](sample/figures/v100-470.82.01/alpha.2-v100-11.0-beltsoff/fixed-density--volume--step-s--model--all.png)
[![Fixed Density Benchmark Zoomed](sample/figures/v100-470.82.01/alpha.2-v100-11.0-beltsoff/fixed-density--volume--step-s--model--zoomed.png)](sample/figures/v100-470.82.01/alpha.2-v100-11.0-beltsoff/fixed-density--volume--step-s--model--zoomed.png)
[![Fixed Density Benchmark](sample/figures/v100-515.65.01/2.0.0-rc-v100-11.0-beltsoff/fixed-density--volume--step-s--model--all.png)](sample/figures/v100-515.65.01/2.0.0-rc-v100-11.0-beltsoff/fixed-density--volume--step-s--model--all.png)
[![Fixed Density Benchmark Zoomed](sample/figures/v100-515.65.01/2.0.0-rc-v100-11.0-beltsoff/fixed-density--volume--step-s--model--zoomed.png)](sample/figures/v100-515.65.01/2.0.0-rc-v100-11.0-beltsoff/fixed-density--volume--step-s--model--zoomed.png)

### Variable Density Benchmark


+ Communication Radius is fixed to `2.0`
+ Agent Density is varied per unit of volume, from `1` to `4`
+ Environment Volume is varied upto `~ 500000` units of volume.
+ Environment Volume is varied up to `~ 500000` units of volume.
+ A single implementation is benchmarked
+ Spatial3D messaging with RTC (run time compilation)
+ Spatial3D messaging with RTC (run time compilation)

[![variable-density volume](sample/figures/v100-470.82.01/alpha.2-v100-11.0-beltsoff/variable-density--volume--step-s--density--3drtc.png)](sample/figures/v100-470.82.01/alpha.2-v100-11.0-beltsoff/variable-density--volume--step-s--density--3drtc.png)
[![variable-density volume](sample/figures/v100-515.65.01/2.0.0-rc-v100-11.0-beltsoff/variable-density--volume--step-s--density--3drtc.png)](sample/figures/v100-515.65.01/2.0.0-rc-v100-11.0-beltsoff/variable-density--volume--step-s--density--3drtc.png)

### Variable Communication Radius Benchmark

Expand All @@ -51,8 +50,7 @@ A combined figure for use in publication is shown below. For details please refe
+ Environment width is fixed at 40
+ Agent Density is `1.0f`


[![Variable Communication Radius Benchmark](sample/figures/v100-470.82.01/alpha.2-v100-11.0-beltsoff/comm-radius--lineplot-spatial3D-bruteforce-rtc-only.png)](sample/figures/v100-470.82.01/alpha.2-v100-11.0-beltsoff/comm-radius--lineplot-spatial3D-bruteforce-rtc-only.png)
[![Variable Communication Radius Benchmark](sample/figures/v100-515.65.01/2.0.0-rc-v100-11.0-beltsoff/comm-radius--lineplot-spatial3D-bruteforce-rtc-only.png)](sample/figures/v100-515.65.01/2.0.0-rc-v100-11.0-beltsoff/comm-radius--lineplot-spatial3D-bruteforce-rtc-only.png)

### Variable Sort Period Benchmark

Expand All @@ -63,15 +61,15 @@ A combined figure for use in publication is shown below. For details please refe
+ Environment width is fixed at 40
+ Agent density is `1.0f`

[![variable-sort-period](sample/figures/v100-470.82.01/alpha.2-v100-11.0-beltsoff/sort-period--lineplot--sort_period--mean_s_step_mean--model-comm_radius.png)](sample/figures/v100-470.82.01/alpha.2-v100-11.0-beltsoff/sort-period--lineplot--sort_period--mean_s_step_mean--model-comm_radius.png)
[![variable-sort-period](sample/figures/v100-515.65.01/2.0.0-rc-v100-11.0-beltsoff/sort-period--lineplot--sort_period--mean_s_step_mean--model-comm_radius.png)](sample/figures/v100-515.65.01/2.0.0-rc-v100-11.0-beltsoff/sort-period--lineplot--sort_period--mean_s_step_mean--model-comm_radius.png)

## Building and Running the Benchmark

Detail of dependencies and the `cmake` build process are described in full in the [FLAMEGPU2-example-template Repo](https://github.com/FLAMEGPU/FLAMEGPU2-example-template) and are not repeated here. The benchmark should be built with seatbelts off (e.g. `-DSEATBELTS=OFF` passed to the `cmake` configuration step) to disable additional run-time checks and optionally disabling Python Swig support which is not needed for this model(-DBUILD_SWIG_PYTHON=OFF ). E.g. for Volta (`SM_70`) GPUs under Linux.
Detail of dependencies and the `cmake` build process are described in full in the [FLAMEGPU2-example-template Repo](https://github.com/FLAMEGPU/FLAMEGPU2-example-template) and are not repeated here. The benchmark should be built with seatbelts off (e.g. `-DFLAMEGPU_SEATBELTS=OFF` passed to the `cmake` configuration step) to disable additional run-time checks. E.g. for Volta (`SM_70`) GPUs under Linux.

```bash
# Configure
cmake . -B build -DCMAKE_BUILD_TYPE=Release -DSEATBELTS=OFF -DBUILD_SWIG_PYTHON=OFF -DCUDA_ARCH=70
cmake . -B build -DCMAKE_BUILD_TYPE=Release -DFLAMEGPU_SEATBELTS=OFF -DCMAKE_CUDA_ARCHITECTURES=70
# Build
cmake --build build -j`nproc`
```
Expand All @@ -80,13 +78,13 @@ cmake --build build -j`nproc`

```bash
cd build
./bin/Release/circles-benchmarking
./bin/Release/circles-benchmark
```

This will produce a number of `.csv` files in the `build` directory.

Note: The `FLAMEGPU2_INC_DIR` environment variable may need to be set to `./_deps/flamegpu2-src/include/` for run-time compilation (RTC) to succeed if the source directory is not automatically found.


## Plotting Results

Individual figures can be generated from data in CSV files via a python script `plot.py`. Alternatively a combined figure used for publication can be produced using `plot_publication.py`.
Expand All @@ -113,11 +111,11 @@ python3 plot.py build -o build/figures
The sample figures were generated from the root directory using

```bash
python3 plot.py sample/data/v100-470.82.01/alpha.2-v100-11.0-beltsoff -o sample/figures/v100-470.82.01/alpha.2-v100-11.0-beltsoff
python3 plot.py -i sample/data/v100-515.65.01/2.0.0-rc-v100-11.0-beltsoff -o sample/figures/v100-515.65.01/2.0.0-rc-v100-11.0-beltsoff
```

The publication figure was generated using

```bash
python3 plot_publication.py sample/data/v100-470.82.01/alpha.2-v100-11.0-beltsoff -o sample/figures/v100-470.82.01/alpha.2-v100-11.0-beltsoff
python3 plot_publication.py -i sample/data/v100-515.65.01/2.0.0-rc-v100-11.0-beltsoff -o sample/figures/v100-515.65.01/2.0.0-rc-v100-11.0-beltsoff
```
110 changes: 68 additions & 42 deletions cmake/flamegpu2.cmake
Original file line number Diff line number Diff line change
@@ -1,49 +1,75 @@
include(FetchContent)
cmake_policy(SET CMP0079 NEW)

# If a FLAMEGPU_VERSION has not been defined, set it to the default option.
if(NOT DEFINED FLAMEGPU_VERSION OR FLAMEGPU_VERSION STREQUAL "")
set(FLAMEGPU_VERSION "master" CACHE STRING "Git branch or tag to use")
endif()

# Allow users to switch to forks with relative ease.

if(NOT DEFINED FLAMEGPU_REPOSITORY OR FLAMEGPU_REPOSITORY STREQUAL "")
set(FLAMEGPU_REPOSITORY "https://github.com/FLAMEGPU/FLAMEGPU2.git" CACHE STRING "Remote Git Repository for FLAME GPU 2+")
endif()

# Always use most recent, simply recommend users that they may wish to do otherwise
FetchContent_Declare(
flamegpu2
GIT_REPOSITORY ${FLAMEGPU_REPOSITORY}
GIT_TAG ${FLAMEGPU_VERSION}
GIT_SHALLOW 1
GIT_PROGRESS ON
# UPDATE_DISCONNECTED ON
)

# Fetch and populate the content if required.
FetchContent_GetProperties(flamegpu2)
if(NOT flamegpu2_POPULATED)
FetchContent_Populate(flamegpu2)
# If overridden by the user, attempt to use that
if (FLAMEGPU_ROOT)
# Look for the main flamegpu.h header to get the abs path, but only look relative to the hints/paths, no cmake defaults. Do not cache the result.
set(FLAMEGPU_INCLUDE_HEADER_FILE include/flamegpu/flamegpu.h)
find_path(FLAMEGPU_ROOT_ABS
NAMES
${FLAMEGPU_INCLUDE_HEADER_FILE}
HINTS
${FLAMEGPU_ROOT}
PATHS
${FLAMEGPU_ROOT}
NO_DEFAULT_PATH
NO_CACHE
)
# If found, use the local flamegpu, otherwise error.
if(FLAMEGPU_ROOT_ABS)
# If the correct flamegpu root was found, output a successful status message
message(STATUS "Found FLAMEGPU_ROOT: ${FLAMEGPU_ROOT_ABS} (${FLAMEGPU_ROOT})")
# update the value to the non abs version, in local and parent scope.
set(FLAMEGPU_ROOT "${FLAMEGPU_ROOT_ABS}")
# And set up the flamegpu build
add_subdirectory(${FLAMEGPU_ROOT_ABS} ${CMAKE_CURRENT_BINARY_DIR}/_deps/flamegpu2-build)
else()
# Send a fatal error if the flamegpu root passed is invalid.
message(FATAL_ERROR "Invalid FLAMEGPU_ROOT '${FLAMEGPU_ROOT}'.\nFLAMEGPU_ROOT must be a valid directory containing '${FLAMEGPU_INCLUDE_HEADER_FILE}'")
endif()
else()
# If a FLAMEGPU_VERSION has not been defined, set it to the default option.
if(NOT DEFINED FLAMEGPU_VERSION OR FLAMEGPU_VERSION STREQUAL "")
set(FLAMEGPU_VERSION "master" CACHE STRING "Git branch or tag to use")
endif()

# If the FLAME GPU version is a hash not a branch or tag,
if(NOT DEFINED FLAMEGPU_VERSION_ALLOW_HASH OR FLAMEGPU_VERSION_ALLOW_HASH STREQUAL "")
set(FLAMEGPU_VERSION_ALLOW_HASH "OFF" CACHE BOOL "Boolean to enable use of a git hash in FLAMEGPU_VERSION. This will slow down CMake configuration.")
endif()

# Now disable extra bells/whistles and add it as s dependency
set(BUILD_ALL_EXAMPLES OFF CACHE BOOL "-")
set(BUILD_TESTS OFF CACHE BOOL "-")
mark_as_advanced(FORCE BUILD_FLAMEGPU2)
mark_as_advanced(FORCE BUILD_ALL_EXAMPLES)
mark_as_advanced(FORCE BUILD_EXAMPLE_HOST_FUNCTIONS)
mark_as_advanced(FORCE BUILD_EXAMPLE_GAME_OF_LIFE)
mark_as_advanced(FORCE BUILD_EXAMPLE_CIRCLES_BRUTE_FORCE)
mark_as_advanced(FORCE BUILD_EXAMPLE_CIRCLES_SPATIAL_3D)
mark_as_advanced(FORCE BUILD_EXAMPLE_RTC_EXAMPLE)
mark_as_advanced(FORCE BUILD_TESTS)
# Allow users to switch to forks with relative ease.
if(NOT DEFINED FLAMEGPU_REPOSITORY OR FLAMEGPU_REPOSITORY STREQUAL "")
set(FLAMEGPU_REPOSITORY "https://github.com/FLAMEGPU/FLAMEGPU2.git" CACHE STRING "Remote Git Repository for FLAME GPU 2+")
endif()

# CMake does not support simple negation, so map to a differnt non cache variable to invert to the correct truthyness for GIT_SHALLOW
set(USE_GIT_SHALLOW "ON")
if(FLAMEGPU_VERSION_ALLOW_HASH)
set(USE_GIT_SHALLOW OFF)
endif()
# Declare the fetch content target usign the above optional variables
FetchContent_Declare(
flamegpu2
GIT_REPOSITORY ${FLAMEGPU_REPOSITORY}
GIT_TAG ${FLAMEGPU_VERSION}
GIT_SHALLOW ${USE_GIT_SHALLOW}
GIT_PROGRESS ON
# UPDATE_DISCONNECTED ON
)
unset(USE_GIT_SHALLOW)

# Add the subdirectory
add_subdirectory(${flamegpu2_SOURCE_DIR} ${flamegpu2_BINARY_DIR})
# Fetch and populate the content if required.
FetchContent_GetProperties(flamegpu2)
if(NOT flamegpu2_POPULATED)
FetchContent_Populate(flamegpu2)
mark_as_advanced(FORCE BUILD_TESTS)
# Add the subdirectory
add_subdirectory(${flamegpu2_SOURCE_DIR} ${flamegpu2_BINARY_DIR})
# Add flamegpu2' expected location to the prefix path.
set(CMAKE_PREFIX_PATH "${CMAKE_PREFIX_PATH};${flamegpu2_SOURCE_DIR}/cmake")
endif()

# Add flamegpu2' expected location to the prefix path.
set(CMAKE_PREFIX_PATH "${CMAKE_PREFIX_PATH};${flamegpu2_SOURCE_DIR}/cmake")
message(STATUS "Found FLAMEGPU2 ${flamegpu2_SOURCE_DIR}")
set(FLAMEGPU_ROOT ${flamegpu2_SOURCE_DIR})
endif()
message(STATUS ${flamegpu2_SOURCE_DIR})
set(FLAMEGPU_ROOT ${flamegpu2_SOURCE_DIR})
2 changes: 1 addition & 1 deletion plot.py
Original file line number Diff line number Diff line change
Expand Up @@ -126,7 +126,7 @@ def validate_args(args):
if args.output_dir is not None:
p = pathlib.Path(args.output_dir)
try:
p.mkdir(exist_ok=True)
p.mkdir(exist_ok=True, parents=True)
except Exception as e:
print(f"Error: Could not create output directory {p}: {e}")
valid = False
Expand Down
8 changes: 4 additions & 4 deletions plot_publication.py
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ def validate_args(args):
if args.output_dir is not None:
p = pathlib.Path(args.output_dir)
try:
p.mkdir(exist_ok=True)
p.mkdir(exist_ok=True, parents=True)
except Exception as e:
print(f"Error: Could not create output directory {p}: {e}")
valid = False
Expand Down Expand Up @@ -149,15 +149,15 @@ def main():

# Plot radius comparison
rad_df = rad_df.query("model == 'circles_spatial3D' or model == 'circles_bruteforce'")
plt_df_rad = sns.lineplot(x='comm_radius_as_percentage_env_width', y='s_step_mean', hue='model', data=rad_df, ax=ax['p3'], palette=custom_palette, ci="sd")
plt_df_rad = sns.lineplot(x='comm_radius_as_percentage_env_width', y='s_step_mean', hue='model', style='model', data=rad_df, ax=ax['p3'], palette=custom_palette, ci="sd")
plt_df_rad.ticklabel_format(style='plain', axis='x') # no scientific notation
plt_df_rad.set(xlabel='r as % of W', ylabel='Step time (s)')
ax['p3'].set_title(label='C', loc='left', fontweight="bold")
ax['p3'].legend().set_visible(False)

# plot data volume
messages_df = rad_df.query("model == 'circles_spatial3D'")
plt_df_rad = sns.lineplot(x='comm_radius_as_percentage_env_width', y='mean_message_count', hue='model', data=messages_df, ax=ax['p4'], palette=custom_palette, ci="sd")
plt_df_rad = sns.lineplot(x='comm_radius_as_percentage_env_width', y='mean_message_count', hue='model', style='model', data=messages_df, ax=ax['p4'], palette=custom_palette, ci="sd")
plt_df_rad.set(xlabel='r as % of W', ylabel='Messages / step')
ax['p4'].set_title(label='D', loc='left', fontweight="bold")
ax['p4'].legend().set_visible(False)
Expand All @@ -171,7 +171,7 @@ def main():
plot_for_com_radius = 8
max_sort_period_to_plot = 20
sort_df = sort_df.query("sort_period <= " + str(max_sort_period_to_plot) + " and comm_radius == " + str(plot_for_com_radius) + " and (model == 'circles_spatial3D' or model == 'circles_spatial3D_rtc')")
plt_df_sort = sns.lineplot(x='sort_period', y='s_step_mean', hue='model', data=sort_df, ax=ax['p5'], palette=custom_palette, ci="sd")
plt_df_sort = sns.lineplot(x='sort_period', y='s_step_mean', hue='model', style='model', data=sort_df, ax=ax['p5'], palette=custom_palette, ci="sd")
plt_df_sort.ticklabel_format(style='plain', axis='x') # no scientific notation
plt_df_sort.set(xlabel='Sort period (steps)', ylabel='Step time (s)')
ax['p5'].set_title(label='E', loc='left', fontweight="bold")
Expand Down
Loading

0 comments on commit 18f4baa

Please sign in to comment.