hornbill (early stage)

docker/base-hornbill/Dockerfile

@@ -0,0 +1,66 @@
+ARG HORNBILL_IMAGE="hstreamdb/hornbill:latest"
+FROM ${HORNBILL_IMAGE}
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+ARG USE_CHINA_MIRROR
+# Mirror
+RUN if [ "$USE_CHINA_MIRROR" = true ] ; then sed -i 's/archive.ubuntu.com/mirrors.ustc.edu.cn/g' /etc/apt/sources.list ; fi
+
+# Basic system stuff
+RUN apt-get -qy update && \
+    apt-get -qy --no-install-recommends install \
+        apt-transport-https
+
+# Install packages
+RUN apt-get -qy update && \
+    apt-get -qy --no-install-recommends install \
+        dos2unix \
+        openssh-server \
+        pwgen
+
+# Install Jepsen deps
+RUN apt-get -qy --no-install-recommends install \
+        build-essential \
+        bzip2 \
+        ca-certificates \
+        curl \
+        dirmngr \
+        dnsutils \
+        faketime \
+        iproute2 \
+        iptables \
+        iputils-ping \
+        logrotate \
+        man \
+        man-db \
+        netcat \
+        net-tools \
+        ntpdate \
+        psmisc \
+        python3 \
+        rsyslog \
+        sudo \
+        tar \
+        tcpdump \
+        unzip \
+        vim \
+        wget \
+        # control deps
+        git \
+        gnuplot-nox \
+        graphviz \
+        htop \
+        libjna-java \
+        leiningen \
+        openjdk-21-jdk-headless \
+        openssh-client \
+        screen && \
+    rm -rf /var/lib/apt/lists/* && apt-get clean
+
+# Config SSH for Jepsen connection
+COPY ./init-ssh.sh /usr/local/bin/init-ssh
+
+EXPOSE 22
+
+RUN chmod +x /usr/local/bin/init-ssh

docker/base-hornbill/init-ssh.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+# Configure sshd
+sed -i "s/#PermitRootLogin prohibit-password/PermitRootLogin yes/g" /etc/ssh/sshd_config
+
+# Start ssh server
+mkdir -p /run/sshd
+/usr/sbin/sshd &
+
+# We add our hostname to the shared volume, so that control can find us
+echo "Adding hostname to shared volume" >> /var/log/jepsen-setup.log
+# We do a little dance to get our hostname (random hex), IP, then use DNS to
+# get a proper container name.
+#HOSTNAME=`hostname`
+#IP=`getent hosts "${HOSTNAME}" | awk '{ print $1 }'`
+#NAME=`dig +short -x "${IP}" | cut -f 1 -d .`
+#echo "${NAME}" >> /var/jepsen/shared/nodes
+echo `hostname` >> /var/jepsen/shared/nodes
+
+# We make sure that root's authorized keys are ready
+echo "Setting up root's authorized_keys" >> /var/log/jepsen-setup.log
+mkdir -p /root/.ssh
+chmod 700 /root/.ssh
+cp /run/secrets/authorized_keys /root/.ssh/
+chmod 600 /root/.ssh/authorized_keys

docker/control-hornbill/Dockerfile

@@ -0,0 +1,90 @@
+ARG BASE_IMAGE="jepsen-hornbill:base"
+FROM ${BASE_IMAGE}
+
+ADD ./bashrc /root/.bashrc
+ADD ./init-ssh-control.sh /init-ssh-control.sh
+RUN dos2unix /init-ssh-control.sh /root/.bashrc \
+    && chmod +x /init-ssh-control.sh
+
+# Proxy
+ARG arg_http_proxy
+ARG arg_https_proxy
+ENV env_http_proxy=$arg_http_proxy
+ENV env_https_proxy=$arg_https_proxy
+
+CMD /init-ssh-control.sh && \
+    eval `ssh-agent` && \
+    ssh-add /root/.ssh/id_rsa && \
+    cd /home/Work && \
+    ### Use proxy (if set) to download deps, then unset proxy for running tests
+    export http_proxy=$env_http_proxy https_proxy=$env_https_proxy && \
+    lein deps && \
+    unset http_proxy && \
+    unset https_proxy && \
+    ####################### start test #######################
+    lein with-profile kafka run test \
+    #-------------- basic options --------------#
+    --db hstream \
+    --workload queue \
+    --sub-via subscribe \
+    --time-limit 180 \
+    --final-time-limit 120 \
+    --key-dist uniform \
+    --key-count 32 \
+    ##<<<
+    # This value is enough. A larger one can make elle exhausted.
+    --concurrency 10 \
+    --rate 10 \
+    ##>>>
+    ##<<<
+    # When writes of a key exceed the limit, the generator will pick
+    # a new key (which can be larger than 'key-count'!!!), which is
+    # very confusing. So we set a very large limit here.
+    --max-writes-per-key 102400 \
+    ##>>>
+    #-------------- kafka client options --------------#
+    ##<<<
+    # The default number of partitions is 1, which may be different
+    # from the test. This can make some partitions unusable. So we
+    # never create topics automatically.
+    --disable-server-auto-create-topics \
+    ##>>>
+    ##<<<
+    # We disable producer retry now to avoid dupicate writes.
+    # It may be turned on in the future to perform more tests...
+    --retries 0 \
+    ##>>>
+    #-------------- txns options --------------#
+    ##<<<
+    # We do not support txns and idempotent for now.
+    # So we disable it.
+    --no-txn \
+    --no-idempotence \
+    --no-server-idempotence \
+    ##>>>
+    #-------------- nemesis options --------------#
+    --nemesis none \
+    --nemesis-interval 15 \
+    --crash-clients true \
+    --tcpdump \
+    # --crash-client-interval 30 \ # FIXME: parse error, what happened?
+    # --tcpdump \ # TODO: sadly, tcpdump is only available for redpanda.
+    #-------------- default options --------------#
+    ##<<<
+    # The following opts is client default (3.7.0) but we
+    # specify them explicitly here for clarity.
+    --auto-offset-reset latest \
+    --enable-auto-commit \
+    --enable-server-auto-create-topics \
+    --isolation-level read_uncommitted \
+    --acks all && \
+    ##>>>
+    ##<<<
+    # For kafka test, the following options are hard-coded,
+    # see workload/queue.clj and workload/list_append.clj.
+    # - replication-factor = 3
+    # - partition-count = 4
+    # - poll-ms = 100
+    # - ... TODO
+    ##>>>
+    exit

docker/control-hornbill/bashrc

@@ -0,0 +1,2 @@
+eval $(ssh-agent) &> /dev/null
+ssh-add /root/.ssh/id_rsa &> /dev/null

docker/control-hornbill/init-ssh-control.sh

@@ -0,0 +1,21 @@
+#!/bin/sh
+
+sleep 1 && \
+
+: "${SSH_PRIVATE_KEY?SSH_PRIVATE_KEY is empty, please use up.sh}"
+: "${SSH_PUBLIC_KEY?SSH_PUBLIC_KEY is empty, please use up.sh}"
+
+if [ ! -f ~/.ssh/known_hosts ]; then
+    mkdir -m 700 ~/.ssh
+    echo $SSH_PRIVATE_KEY | perl -p -e 's/↩/\n/g' > ~/.ssh/id_rsa
+    chmod 600 ~/.ssh/id_rsa
+    echo $SSH_PUBLIC_KEY > ~/.ssh/id_rsa.pub
+    echo > ~/.ssh/known_hosts
+    # Get nodes list
+    sort -V /var/jepsen/shared/nodes > ~/nodes
+    # Scan SSH keys
+    while read node; do
+      ssh-keyscan -t rsa $node >> ~/.ssh/known_hosts
+      ssh-keyscan -t ed25519 $node >> ~/.ssh/known_hosts
+    done <~/nodes
+fi

docker/docker-compose-hornbill.yml

@@ -0,0 +1,151 @@
+version: '3.7'
+x-node:
+  &hserver-node
+  build: ./node-hornbill
+  env_file:
+    - ./secret/node.env
+    - ./secret/control.env
+  secrets:
+    - authorized_keys
+  tty: true
+  tmpfs:
+    - /run:size=100M
+    - /run/lock:size=100M
+  volumes:
+    - "jepsen-hornbill-shared:/var/jepsen/shared"
+    - "/sys/fs/cgroup:/sys/fs/cgroup:ro"
+    - "/tmp:/tmp:rw"
+  cap_add:
+    - ALL
+  ports:
+    - ${JEPSEN_PORT:-22}
+
+volumes:
+  jepsen-hornbill-shared:
+
+secrets:
+  authorized_keys:
+    file: ./secret/authorized_keys
+
+networks:
+  jepsen-hornbill-network:
+    ipam:
+      config:
+        - subnet: 172.20.0.0/24
+
+services:
+  fdb:
+    container_name: jepsen-hornbill-fdb
+    hostname: fdb
+    build: ./fdb-hornbill
+    env_file: ./secret/node.env
+    secrets:
+      - authorized_keys
+    tmpfs:
+      - /run:size=100M
+      - /run/lock:size=100M
+    expose:
+      - "4500"
+    networks:
+      jepsen-hornbill-network:
+        ipv4_address: 172.20.0.6
+    cap_add:
+      - ALL
+    volumes:
+      - "jepsen-hornbill-shared:/var/jepsen/shared"
+      - "/sys/fs/cgroup:/sys/fs/cgroup:ro"
+      - "/tmp:/tmp:rw"
+
+  meta:
+    container_name: jepsen-hornbill-meta
+    hostname: meta
+    depends_on:
+      - fdb
+    build: ./meta-hornbill
+    env_file: ./secret/node.env
+    secrets:
+      - authorized_keys
+    tmpfs:
+      - /run:size=100M
+      - /run/lock:size=100M
+    expose:
+      - "22"
+      - "8964"
+    networks:
+      jepsen-hornbill-network:
+        ipv4_address: 172.20.0.7
+    cap_add:
+      - ALL
+    volumes:
+      - "jepsen-hornbill-shared:/var/jepsen/shared"
+      - "/sys/fs/cgroup:/sys/fs/cgroup:ro"
+      - "/tmp:/tmp:rw"
+
+  control:
+    container_name: jepsen-hornbill-control
+    hostname: control-hornbill
+    depends_on:
+      - meta
+      - hserver-1
+      - hserver-2
+      - hserver-3
+      - hserver-4
+      - hserver-5
+    build: ./control-hornbill
+    env_file: ./secret/control.env
+    privileged: true
+    expose:
+      - "22"
+      - "8080"
+    networks:
+      jepsen-hornbill-network:
+        ipv4_address: 172.20.0.8
+    volumes:
+      - "jepsen-hornbill-shared:/var/jepsen/shared"
+      - "/home/commelina/.m2/:/root/.m2"
+
+  hserver-1:
+    << : *hserver-node
+    container_name: jepsen-hornbill-n1
+    hostname: n1
+    networks:
+      jepsen-hornbill-network:
+        ipv4_address: 172.20.0.11
+    depends_on:
+      - meta
+  hserver-2:
+    << : *hserver-node
+    container_name: jepsen-hornbill-n2
+    hostname: n2
+    networks:
+      jepsen-hornbill-network:
+        ipv4_address: 172.20.0.12
+    depends_on:
+      - meta
+  hserver-3:
+    << : *hserver-node
+    container_name: jepsen-hornbill-n3
+    hostname: n3
+    networks:
+      jepsen-hornbill-network:
+        ipv4_address: 172.20.0.13
+    depends_on:
+      - meta
+  hserver-4:
+    << : *hserver-node
+    container_name: jepsen-hornbill-n4
+    hostname: n4
+    networks:
+      jepsen-hornbill-network:
+        ipv4_address: 172.20.0.14
+    depends_on:
+      - meta
+  hserver-5:
+    << : *hserver-node
+    container_name: jepsen-hornbill-n5
+    hostname: n5
+    networks:
+      jepsen-hornbill-network:
+        ipv4_address: 172.20.0.15
+    depends_on:
+      - meta

docker/fdb-hornbill/Dockerfile

@@ -0,0 +1,8 @@
+FROM foundationdb/foundationdb:7.1.61
+
+COPY ./start.sh /usr/local/bin/start.sh
+RUN chmod +x /usr/local/bin/start.sh
+
+EXPOSE 22 4500
+
+ENTRYPOINT ["/usr/bin/tini", "--", "/usr/local/bin/start.sh"]

docker/fdb-hornbill/start.sh

@@ -0,0 +1,29 @@
+FDB_CLUSTER_FILE="/etc/foundationdb/fdb.cluster"
+FDB_PORT=4500
+FDB_NETWORKING_MODE="container"
+fdbcli="/usr/bin/fdbcli"
+
+## start fdb server
+/var/fdb/scripts/fdb.bash >> /tmp/$HOSTNAME.log 2>&1 &
+
+## wait & configure
+# Attempt to connect. Configure the database if necessary.
+if ! $fdbcli -C $FDB_CLUSTER_FILE --exec status --timeout 1 ; then
+    config="configure new single ssd; status"
+    if ! $fdbcli -C $FDB_CLUSTER_FILE --exec "$config" --timeout 10 ; then
+        echo "Unable to configure new FDB cluster."
+        exit 1
+    fi
+fi
+
+echo "Can now connect to docker-based FDB cluster using $FDB_CLUSTER_FILE."
+
+echo "Create hstream tenant"
+config="configure tenant_mode=optional_experimental; createtenant hstream;"
+if ! $fdbcli -C $FDB_CLUSTER_FILE --exec "$config" --timeout 10 ; then
+    echo "Create hstream tenant failed"
+fi
+
+## Make sure the process doesn't exit
+## FIXME: This makes the script block forever, which is bad for jepsen nemesis
+tail -f /dev/null