diff --git a/README.md b/README.md index 03183a4..24aae89 100644 --- a/README.md +++ b/README.md @@ -2,15 +2,16 @@ -# LightSaber [![License](https://img.shields.io/github/license/lsds/LightSaber.svg?branch=master)](https://github.com/lsds/LightSaber/blob/master/LICENCE.md) +# LightSaber [![License](https://img.shields.io/github/license/lsds/LightSaber.svg?branch=master)](https://github.com/lsds/LightSaber/blob/master/LICENCE.md) LightSaber is a stream processing engine that balances parallelism and incremental processing when -executing window aggregation queries on multi-core CPUs. LightSaber operates on in-order +executing window aggregation queries on multi-core CPUs. LightSaber operates on in-order streams of data and achieves up to an order of magnitude higher throughput compared to existing systems. ### Getting started The `prepare-software.sh` script will guide you through the installation and compilation process of our system locally. +The script is tested on **Ubuntu 18.04.5 LTS**. ``` $ git clone https://github.com/lsds/LightSaber.git @@ -72,7 +73,7 @@ Variables in **SystemConf.h** configure the LightSaber runtime. Each of them als ###### --threads _N_ -Sets the number of CPU worker threads (`WORKER_THREADS` variable). The default value is `1`. **CPU worker threads are pinned to physical cores**. The threads are pinned to core ids based on the underlying hardware (e.g., if there are multiple sockets with n cores each, the first n threads are pinned in the first socket and so on). +Sets the number of CPU worker threads (`WORKER_THREADS` variable). The default value is `1`. **CPU worker threads are pinned to physical cores**. The threads are pinned to core ids based on the underlying hardware (e.g., if there are multiple sockets with n cores each, the first n threads are pinned in the first socket and so on). ###### --slots _N_ @@ -94,10 +95,6 @@ Sets the intermediate result buffer size in bytes (`UNBOUNDED_BUFFER_SIZE` varia Hash table size (in number of buckets): hash tables hold partial window aggregate results (`HASH_TABLE_SIZE` variable with the default value 512). -###### --throughput-monitor-interval _N_ - -Sets the query throughput matrix update interval, in msec (`THROUGHPUT_MONITOR_INTERVAL` variable). The default value is `1000` i.e. 1 sec. - ###### --performance-monitor-interval _N_ Sets the performance monitor interval, in msec (`PERFORMANCE_MONITOR_INTERVAL` variable). The default value is `1000`, i.e. 1 sec. Controls how often LightSaber prints on standard output performance statistics such as throughput and latency. @@ -108,4 +105,4 @@ Determines whether LightSaber should measure task latency or not (`LATENCY_ON` v ###### To enable NUMA-aware scheduling -Set the HAVE_NUMA flag in the respective CMakeLists.txt (e.g., in test/benchmarks/applications/CMakeLists.txt) and recompile the code. +Set the HAVE_NUMA flag in the respective CMakeLists.txt (e.g., in test/benchmarks/applications/CMakeLists.txt) and recompile the code. \ No newline at end of file diff --git a/scripts/prepare-software.sh b/scripts/prepare-software.sh index e092d6b..9b6f664 100755 --- a/scripts/prepare-software.sh +++ b/scripts/prepare-software.sh @@ -7,6 +7,33 @@ cd sudo apt-get update sudo apt-get upgrade +# Stuff +sudo apt-get install g++ build-essential python-dev autotools-dev libicu-dev libbz2-dev aptitude \ + libevent-dev \ + libdouble-conversion-dev \ + libgoogle-glog-dev \ + libgflags-dev \ + libiberty-dev \ + liblz4-dev \ + liblzma-dev \ + libsnappy-dev \ + make \ + zlib1g-dev \ + binutils-dev \ + libjemalloc-dev \ + libssl-dev \ + pkg-config + +# Boost +sudo apt-get install libboost-all-dev + +# TBB +sudo apt-get install libtbb-dev + +# Used for disk profiling +sudo apt install bpfcc-tools +# sudo biolatency-bpfcc -D 30 1 + # CMake 3.16 sudo apt remove --purge --auto-remove cmake version=3.16 @@ -32,29 +59,12 @@ cd doxygen mkdir build cd build cmake -G "Unix Makefiles" .. -make +make -j$(nproc) sudo make install echo 'export PATH=/usr/lib/ccache:$PATH' >> $HOME/.profile sudo ln -s /usr/local/bin/doxygen /usr/bin/doxygen cd -# Stuff -sudo apt-get install g++ build-essential g++ python-dev autotools-dev libicu-dev build-essential libbz2-dev aptitude \ - libevent-dev \ - libdouble-conversion-dev \ - libgoogle-glog-dev \ - libgflags-dev \ - libiberty-dev \ - liblz4-dev \ - liblzma-dev \ - libsnappy-dev \ - make \ - zlib1g-dev \ - binutils-dev \ - libjemalloc-dev \ - libssl-dev \ - pkg-config - # LLVM 9 git clone https://github.com/llvm/llvm-project.git cd llvm-project @@ -65,27 +75,27 @@ cmake -DLLVM_ENABLE_PROJECTS=clang -DCMAKE_BUILD_TYPE=Release \ -DBUILD_SHARED_LIBS=ON -DCLANG_INCLUDE_DOCS=OFF -DCLANG_INCLUDE_TESTS=OFF \ -DCLANG_INSTALL_SCANBUILD=OFF -DCLANG_INSTALL_SCANVIEW=OFF -DCLANG_PLUGIN_SUPPORT=OFF \ -DLLVM_TARGETS_TO_BUILD=X86 -G "Unix Makefiles" ../llvm -make +make -j$(nproc) sudo make install echo 'export LLVM_HOME=$(pwd)' >> $HOME/.profile echo 'export PATH=$LLVM_HOME/bin:$PATH' >> $HOME/.profile echo 'export LIBRARY_PATH=$LLVM_HOME/lib:$LIBRARY_PATH' >> $HOME/.profile +source $HOME/.profile sudo rm /etc/ld.so.cache sudo ldconfig +sudo ln -s /usr/local/bin/clang++ /usr/lib/ccache/clang++ +sudo ln -s /usr/local/bin/clang /usr/lib/ccache/clang cd -# Boost -sudo apt-get install libboost-all-dev - -# TBB -sudo apt-get install libtbb-dev - # Google Test sudo apt-get install libgtest-dev cd /usr/src/gtest sudo cmake CMakeLists.txt -sudo make -sudo cp *.a /usr/lib +sudo make -j$(nproc) +# which one is the correct? +sudo cp ./lib/*.a /usr/lib +sudo cp *.a /usr/lib/ +sudo mkdir /usr/local/lib/gtest sudo ln -s /usr/lib/libgtest.a /usr/local/lib/gtest/libgtest.a sudo ln -s /usr/lib/libgtest_main.a /usr/local/lib/gtest/libgtest_main.a cd @@ -96,7 +106,7 @@ cd benchmark mkdir build cd build cmake .. -DCMAKE_BUILD_TYPE=RELEASE -DBENCHMARK_DOWNLOAD_DEPENDENCIES=ON -make +make -j$(nproc) sudo make install cd @@ -111,4 +121,4 @@ cd #cd $HOME/LightSaber/build/test/benchmarks/microbenchmarks/ #./TestProjection -echo "All done..." +echo "All done..." \ No newline at end of file diff --git a/test/benchmarks/applications/BenchmarkQuery.h b/test/benchmarks/applications/BenchmarkQuery.h index 3b31252..b09ed80 100644 --- a/test/benchmarks/applications/BenchmarkQuery.h +++ b/test/benchmarks/applications/BenchmarkQuery.h @@ -58,6 +58,8 @@ class BenchmarkQuery { } else if (strcmp(argv[i], "--parallel-merge") == 0) { SystemConf::getInstance().PARALLEL_MERGE_ON = (strcasecmp(argv[j], "true") == 0 || std::atoi(argv[j]) != 0); + } else if (strcmp(argv[i], "--performance-monitor-interval") == 0) { + SystemConf::getInstance().PERFORMANCE_MONITOR_INTERVAL = std::stoul(argv[j]); } else if (strcmp(argv[i], "--latency") == 0) { SystemConf::getInstance().LATENCY_ON = (strcasecmp(argv[j], "true") == 0 || std::atoi(argv[j]) != 0);