Initial commit

.github/workflows/ci.yml

@@ -0,0 +1,28 @@
+name: CI 
+
+on: [push]
+
+jobs:
+  build:
+    runs-on: ubuntu-22.04
+
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python 3.12
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.12'
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        make install
+        pip install mypy pylint pytest
+    - name: Setup mypy
+      run: |
+        pip install mypy
+        mypy ./wke || true
+        mypy --install-types --non-interactive
+    - name: Unit Tests
+      run: make test
+    - name: Lint Checks 
+      run: make lint

.github/workflows/publish.yml

@@ -0,0 +1,27 @@
+name: "Publish"
+
+on:
+  release:
+    types: [created]
+
+jobs:
+  pypi-publish:
+    name: Upload release to PyPI
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/p/wke
+
+    permissions:
+      id-token: write  # IMPORTANT: this permission is mandatory for trusted publishing
+
+    steps:
+    - uses: actions/checkout@v4
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install build 
+    - name: Generate package distributions
+      run:  make package
+    - name: Publish package to PyPI
+      uses: pypa/gh-action-pypi-publish@release/v1

.gitignore

@@ -0,0 +1,5 @@
+__pycache__
+logs/
+dist/
+build/
+wke.egg-info/

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Kai Mast
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

Makefile

@@ -0,0 +1,24 @@
+.PHONY: cloc lint install
+
+SRC_DIR=./wke
+
+all: test lint install
+
+clean:
+	rm -rf ${SRC_DIR}/__pycache__
+
+cloc:
+	cloc ${SRC_DIR}
+
+lint:
+	pylint ${SRC_DIR}
+	mypy ${SRC_DIR}
+
+test:
+	pytest tests
+
+package:
+	python3 -m build
+
+install:
+	pip install .

NEWS

@@ -0,0 +1,2 @@
+v0.1:
+    * Initial release 

README.md

@@ -0,0 +1,37 @@
+# wke: A benchmarking tool for distributed systems  
+
+This tool can deploy and measure the behavior of distributed systems. However, it is not a general framework for running things on remote machines but tailored to the specific task of running experiments/benchmarks.
+
+See [this blogpost](https://kaimast.com/notes/wickie/) for a detailed description of wke.
+
+## Installation
+
+You need a recent version of Python (the scripts were only tested with python3.10 and above) and `pip`.
+
+Then, simply run `make install` to install the wke library and command.
+
+## Clusters and Configurations
+
+A *cluster* is a set of machines (physical or virtual) that will be used to run experiments.
+Clusters groups machines into *classes*. Each class should represent a role in your distributed system.
+For example, a simple benchmark might have two classes: clients and servers.
+
+A *configuration* is a set of scripts for a specific project (or variant of a project). You can set up multiple configurations for a single cluster.
+
+Each configuration resides in a dedicated folder that has the following layout:
+
+### {CONFIG}/config.toml
+
+A TOML file that describes all targets and settings specific to that configuration. Targets are scripts that can be run on machines of the cluster.
+
+### {CONFIG}/targets
+
+Targets are scripts that execute as part of your experiment or to set up the experiment. For example, it could be script that runs your server process, or one that issues client requests.
+
+You will execute them using the `wke run` command.
+
+### {CONFIG}/preludes
+
+Preludes are not targets, but can help reducing boilerplate code in a target. For example, they can be used to export environment variables before running experiments.
+
+In most cases as single engine file should suffice to run your experiments.

pyproject.toml

@@ -0,0 +1,32 @@
+[project]
+name = "wke"
+version = "0.1"
+dependencies=['paramiko', 'pandas', 'seaborn']
+requires-python=">=3.11"
+readme = "README.md"
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "License :: OSI Approved :: MIT License",
+]
+
+
+[project.urls]
+Homepage = "https://github.com/kaimast/wke"
+Issues = "https://github.com/kaimast/wke/issues"
+
+[metadata]
+author = "Kai Mast"
+author_email = "kai@kaimast.com"
+
+[build-system]
+requires = ["setuptools"]
+build-backend = "setuptools.build_meta"
+
+[project.scripts]
+wke = "wke.bin.cmd:main"
+
+[tool.setuptools.packages.find]
+where = ["."]
+
+[tool.setuptools.package-data]
+"*" = ["py.typed"]

test-files/configs/basic/config.toml

@@ -0,0 +1,26 @@
+[config]
+default-prelude = "home-runner"
+
+[ubuntu]
+required-packages = ["htop", "nload"]
+
+[preludes]
+home-runner = "Sets up $PATH to point to ~/.local/bin"
+
+[targets]
+# You can define targets without any metadata or arguments
+install-tokio = []
+
+# The most concise way to specify arguments is as a list of lists
+benchmark-tokio = [["num-operations", 10000]]
+
+[targets.setup-rust]
+# But you can also define targets in a more verbose
+# and human-readable way
+arguments = [
+    { name="channel", default="stable" }, 
+    { name="profile", default="minimal" }
+]
+about = "Install the rust toolchain"
+
+

test-files/configs/basic/preludes/home-runner

@@ -0,0 +1,3 @@
+#! /bin/bash
+export PATH=${PATH}:${HOME}/.local/bin:/usr/local/bin
+export RUST_BACKTRACE=1

test-files/configs/basic/targets/benchmark_tokio.py

@@ -0,0 +1,2 @@
+#! /bin/env python3
+print("Just another test script")

test-files/configs/basic/targets/install-tokio

@@ -0,0 +1,2 @@
+#! /bin/bash
+echo "Just a test script"

test-files/configs/cluster.toml

@@ -0,0 +1,9 @@
+[cluster]
+username="cskama"
+workdir="/workdir"
+
+[machines]
+node1 = ["128.105.144.176","10.1.1.1"]
+node2 = ["128.105.144.200","10.1.1.2"]
+node3 = ["128.105.144.174","10.1.1.4"]
+node4 = ["128.105.144.183","10.1.1.3"]

test-files/configs/inherit/config.toml

@@ -0,0 +1,11 @@
+[config]
+inherits = "basic"
+
+[targets]
+install-smol = []
+
+# If you do not set a default value for an argument, it is required
+benchmark-smol = [["num-operations"]]
+
+[targets.setup-rust]
+arguments = [["channel", "nightly"]]

test-files/configs/inherit/targets/benchmark_smol.py

@@ -0,0 +1,2 @@
+#! /bin/env python3
+print("Another script, but for smol")

test-files/experiments/basic.toml

@@ -0,0 +1,7 @@
+[experiment]
+num_iterations=2
+
+[parameters]
+workload="hello-world"
+num-txns=1000
+worker-type=["docker", "open-lambda", "faasm"]

test-files/experiments/exponential-range.toml

@@ -0,0 +1,7 @@
+[experiment]
+num_iterations=5
+
+[parameters]
+workload="hello-world"
+num-txns={ base=10, start=2, end=10, step-size=2 }
+worker-type="docker"

test-files/experiments/multi_subparams.toml

@@ -0,0 +1,22 @@
+# Tests double-nested subparameters
+
+[experiment]
+num_iterations=2
+
+[parameters]
+workload="hello-world"
+sub-parameters=[["foo", "bar"],["txns100", "txns200"]]
+
+[foo]
+num-clients=10
+worker-type="docker"
+
+[bar]
+num-clients=20
+worker-type="open-lambda"
+
+[txns100]
+num-txns=100
+
+[txns200]
+num-txns=200

test-files/experiments/single_subparam.toml

@@ -0,0 +1,11 @@
+[experiment]
+num_iterations=2
+
+[parameters]
+workload="hello-world"
+sub-parameters=["foo"]
+
+[foo]
+num-clients={ start=20, end=100, step-size=80 }
+num-txns=[100]
+worker-type="docker"

test-files/experiments/subparams.toml

@@ -0,0 +1,16 @@
+[experiment]
+num_iterations=2
+
+[parameters]
+workload="hello-world"
+sub-parameters=["foo", "bar"]
+
+[foo]
+num-clients=10
+num-txns=[100,200]
+worker-type="docker"
+
+[bar]
+num-clients=20
+num-txns=[1000,2000]
+worker-type="open-lambda"

test-files/experiments/subparams_and_range.toml

@@ -0,0 +1,15 @@
+[experiment]
+num_iterations=2
+
+[parameters]
+workload="hello-world"
+sub-parameters=["foo", "bar"]
+num-txns=[100,200]
+
+[foo]
+num-clients=10
+worker-type="docker"
+
+[bar]
+num-clients=20
+worker-type="open-lambda"

test-files/experiments/subparams_and_range_flipped.toml

@@ -0,0 +1,15 @@
+[experiment]
+num_iterations=2
+
+[parameters]
+workload="hello-world"
+num-txns=[100,200]
+sub-parameters=["foo", "bar"]
+
+[foo]
+num-clients=10
+worker-type="docker"
+
+[bar]
+num-clients=20
+worker-type="open-lambda"

test-files/experiments/subparams_nested.toml

@@ -0,0 +1,27 @@
+# Tests double-nested subparameters
+
+[experiment]
+num_iterations=2
+
+[parameters]
+workload="hello-world"
+sub-parameters=["foo", "bar"]
+
+[foo]
+num-clients=10
+worker-type="docker"
+sub-parameters=["txns100", "txns200"]
+
+[bar]
+num-clients=20
+worker-type="open-lambda"
+sub-parameters=["txns1000"]
+
+[txns100]
+num-txns=100
+
+[txns200]
+num-txns=200
+
+[txns1000]
+num-txns=1000