Merge pull request #1 from reecetech/feature/initilisation

(feat): Initialisation

.dockerignore

@@ -0,0 +1,6 @@
+.github
+configs
+LICENSE
+README.md
+build/*
+ebs-bootstrap*

.github/workflows/build-and-test.yml

@@ -0,0 +1,18 @@
+name: Build and Test 🔨
+on: [push]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        architecture: [amd64, arm64]
+    name: Build and Test (${{ matrix.architecture }}) 🔨
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+        with:
+          platforms: ${{ matrix.architecture }}
+      - name: Build and Test 🔨
+        run: ./build/docker.sh --architecture ${{ matrix.architecture }}

.gitignore

@@ -0,0 +1 @@
+/ebs-bootstrap*

Dockerfile

@@ -0,0 +1,22 @@
+# syntax=docker/dockerfile:1
+FROM golang:1.21
+
+# Set destination for COPY
+WORKDIR /app
+
+# Copy go source code
+COPY ./ ./
+
+# Install dependencies
+RUN go mod download
+
+# Build application
+RUN go build cmd/ebs-bootstrap.go
+
+# Test application
+RUN go test ./...
+
+# ebs-bootstrap cannot run in docker as it needs to interact
+# with the raw devices of the host. Therefore docker must be
+# exclusively used to build the binary in host architecture agnostic manner
+CMD ["tail", "-f", "/dev/null"]

README.md

@@ -0,0 +1,174 @@
+# EBS Bootstrap
+
+## Build
+
+`ebs-bootstrap` can be built locally regardless of the architecture of the host machine. This is facilitated by a multi-architecture Docker build process. The currently supported architechtures are `linux/amd64` and `linux/arm64`.
+
+```bash
+# Specific Architecture
+./build/docker.sh --architecture arm64
+ls -la
+... ebs-bootstrap_linux-arm64
+
+# All Architectures
+./build/docker.sh
+ls -la
+... ebs-bootstrap_linux-arm64
+... ebs-bootstrap_linux-x86_64
+```
+## Recommended Setup
+
+### `systemd`
+
+The ideal way of operating `ebs-bootstrap` is through a `systemd` service. This is so we can configure it as a `oneshot` service type that executes after the file system is ready and after `clout-init.service` writes any config files to disk. The latter is essential as `ebs-bootstrap` consumes a config file that is located at `/etc/ebs-boostrap/config.yml` by default. 
+
+`ExecStopPost=-...` con point torwards a script that is executed when the `ebs-bootstrap` service exits on either success or failure. This is a suitable place to include logic to notify a human operator that the configured devices failed their relevant healthchecks and the underlying application failed to launch in the process.
+
+```ini
+[Unit]
+Description=EBS Bootstrap
+After=local-fs.target cloud-init.service
+
+[Service]
+Type=oneshot
+RemainAfterExit=true
+StandardInput=null
+ExecStart=ebs-bootstrap
+PrivateMounts=no
+MountFlags=shared
+ExecStopPost=-/etc/ebs-bootstrap/post-hook.sh
+
+[Install]
+WantedBy=multi-user.target
+```
+
+```
+cat /etc/ebs-bootstrap/post-hook.sh
+#!/bin/sh
+if [ "${EXIT_STATUS}" = "0" ]; then
+    echo "🟢 Post Stop Hook: Success"
+else
+    echo "🔴 Post Stop Hook: Failure"
+fi
+```
+
+It is then possible to configure another `systemd` service to only start if the `ebs-bootstrap` service is successful. Certain databases support the ability to spread database chunks across multiple devices that need to be mounted to pre-defined directories with the correct ownership and permissions. In this particular use-case, the database could be configured as a `systemd` service that relies on the `ebs-bootstrap.service` to succeed before attempting to start. This can be achieved by specifiying `ebs-boostrap.service` as a dependency in the `Requires=` and `After=` parameters.
+
+```ini
+[Unit]
+Description=Example Database
+Wants=network-online.target
+Requires=ebs-bootstrap.service
+After=network.target network-online.target ebs-bootstrap.service
+
+[Service]
+Type=forking
+User=ec2-user
+Group=ec2-user
+ExecStart=/usr/bin/database start
+ExecStop=/usr/bin/database stop
+
+[Install]
+WantedBy=multi-user.target
+```
+
+### `cloud-init`
+
+`cloud-init` can be configured through EC2 User Data to write a config file to `/etc/ebs-boostrap/config.yml` through the `write_files` module. 
+
+The NVMe Driver, for Nitro-based EC2 Instances, has an [established behaviour](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/nvme-ebs-volumes.html) of dynamically renaming a block device, based on the order in which the device is attached to the EC2 instance. This order is unpredictable, thus it is recommended to label the volumes appropriately so that they can be referenced consistently in the `mounts` module. The `mounts` module is responsible for managing the `/etc/fstab` file, thus establishing a reliable mechanism for mounting external volumes at boot, independent of the block storage driver used by the EC2 instance.
+
+```yaml
+Resources:
+  Instance:
+    Type: AWS::EC2::Instance
+  ...
+  Volumes:
+    - Device: /dev/sdb
+      VolumeId: !Ref ExternalVolumeID
+  UserData:
+    Fn::Base64: !Sub
+      - |+
+        #cloud-config
+        write_files:
+          - content: |
+              global:
+                mode: healthcheck
+              devices:
+                /dev/sdb:
+                  fs: ${FileSystem}
+                  mount_point: /mnt/app
+                  owner: ec2-user
+                  group: ec2-user
+                  permissions: 755
+                  label: external-vol
+            path: /etc/ebs-bootstrap/config.yml
+        mounts:
+          - [ "LABEL=external-vol", "/mnt/app", "${FileSystem}", "${MountOptions}", "0", "2" ]
+      - FileSystem: ext4
+        MountOptions: defaults,nofail,x-systemd.device-timeout=5
+```
+
+## Config
+
+### `global`
+
+#### `mode`
+
+Specifies the mode that `ebs-bootstrap` operates in
+  - `healthcheck`
+    - Validate whether the state of a device matches its desired configuration
+    - Returns an exit code of `0` 🟢, if no changes are detected
+    - Returns an exit code of `1` 🔴, if changes are detected
+
+### `devices[*]`
+
+#### `fs`
+
+The **file system** that the device has been formatted to
+  - If an empty string is provided, all other device properties will be ignored
+
+#### `mount_point`
+
+The **mount point** that the device has been mounted to
+  - If an empty string is provided, `owner`, `group` and `permissions` will be ignored
+
+#### `owner`
+
+The **user** that has been assigned ownership of the mount point
+  - Supports both a user **ID** and the **name** of the user
+
+#### `group`
+
+The **group** that has been assigned ownership of the mount point
+  - Supports both a group **ID** and the **name** of the group
+
+#### `permissions`
+
+The **permissions** that has been assigned to the mount point
+  - Must be specified as a three digit octal: `755`, `644`, ...
+
+#### `label`
+
+The **label** assigned to the formatted device
+  - Labels are constrained to the limitations of the underlying file system.
+    - `ext4` file systems have a maximum label size of `16`
+    - `xfs` file systems have a maximum label size of `12`
+
+#### `mode`
+
+Provide a device-level **override** of a global `mode` property
+
+```yaml
+global:
+  mode: healthcheck
+devices:
+  /dev/xvdf:
+    fs: "xfs"
+    mount_point: "/ifmx/dev/root"
+    owner: 1000
+    group: 1000
+    permissions: 755
+    label: "external-vol"
+    mode: healthcheck
+```

build/docker.sh

@@ -0,0 +1,102 @@
+#!/bin/bash
+set -euo pipefail
+
+# https://stackoverflow.com/questions/59895/get-the-source-directory-of-a-bash-script-from-within-the-script-itself
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+cd "${SCRIPT_DIR}/.."
+
+IMAGE="ebs-bootstrap"
+ARCHS=("amd64" "arm64")
+
+function map_depedencies() {    
+    # Check if Mac-GNU alternative binaries are installed
+    getopt_cmd="getopt"
+
+    if [[ "$(uname)" == "Darwin" ]] ; then
+        getopt_cmd="$(brew --prefix)/opt/gnu-getopt/bin/getopt"
+        if [[ ! -x "$(type -P "${getopt_cmd}")" ]] ; then
+            echo >&2 "
+                ERROR: GNU-enhanced version of getopt not installed
+                    Run \"brew install gnu-getopt\""
+            exit 2
+        fi
+    fi
+}
+
+function get_docker_platform() {
+    arch="${1:-}"
+    if [ "${arch}" = 'arm64' ]; then
+        echo "linux/arm64"
+    elif [ "${arch}" = 'amd64' ]; then
+        echo "linux/amd64"
+    else
+        >&2 echo "🔴 Unsupported architecture: ${arch}"; exit 1
+    fi
+}
+
+function get_binary_name() {
+    arch="${1:-}"
+    if [ "${arch}" = 'arm64' ]; then
+        echo "ebs-bootstrap-linux-arm64"
+    elif [ "${arch}" = 'amd64' ]; then
+        echo "ebs-bootstrap-linux-x86_64"
+    else
+        >&2 echo "🔴 Unsupported architecture: ${arch}"; exit 1
+    fi
+}
+
+function docker_build() {
+    for arch in "${ARCHS[@]}"
+    do
+        docker_platform="$(get_docker_platform "${arch}")"
+        docker build . -t "${IMAGE}:${arch}" --platform "${docker_platform}" --no-cache
+        echo "🟢 Built image: ${IMAGE}:${arch}"
+    done
+}
+
+function copy_binaries() {
+    for arch in "${ARCHS[@]}"
+    do
+        name="$(get_binary_name "${arch}")"
+        id=$(docker create "${IMAGE}:${arch}")
+        # docker cp produces a tar stream
+        docker cp "$id:/app/ebs-bootstrap" - | tar xf - --transform "s/ebs-bootstrap/${name}/"
+        docker rm -v "$id"
+        echo "🟢 Built and copied binary: ${name}"
+    done
+}
+
+function main() {
+    docker_build
+    copy_binaries
+}
+
+map_depedencies
+
+ARGUMENT_LIST=(
+  "architecture"
+)
+
+opts=$("${getopt_cmd}" \
+  --longoptions "$(printf "%s:," "${ARGUMENT_LIST[@]}")" \
+  --name "$(basename "$0")" \
+  --options "" \
+  -- "$@"
+)
+
+eval set --"$opts"
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --architecture)
+      ARCHS=("${2}")
+      shift 2
+      ;;
+
+    *)
+      break
+      ;;
+  esac
+done
+
+main

cmd/ebs-bootstrap.go

@@ -0,0 +1,44 @@
+package main
+
+import (
+	"os"
+	"log"
+	"ebs-bootstrap/internal/config"
+	"ebs-bootstrap/internal/service"
+	"ebs-bootstrap/internal/utils"
+	"ebs-bootstrap/internal/state"
+)
+
+func main() {
+	// Disable Timetamp
+	log.SetFlags(0)
+	e := utils.NewExecRunner()
+	ds := &service.LinuxDeviceService{Runner: e}
+	ns := &service.AwsNVMeService{}
+	fs := &service.UnixFileService{}
+	dts := &service.EbsDeviceTranslator{DeviceService: ds, NVMeService: ns}
+
+	dt, err := dts.GetTranslator()
+	if err != nil {
+		log.Fatal(err)
+	}
+	config, err := config.New(os.Args, dt, fs)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	for name, device := range config.Devices {
+		d, err := state.NewDevice(name, ds, fs)
+		if err != nil {
+			log.Fatal(err)
+		}
+		err = d.Diff(config)
+		if err == nil {
+			log.Printf("🟢 %s: No changes detected", name)
+			continue
+		}
+		if device.Mode == "healthcheck" {
+			log.Fatal(err)
+		}
+	}
+}

configs/ebs-bootstrap.yml

@@ -0,0 +1,11 @@
+global:
+  mode: healthcheck
+devices:
+  /dev/xvdf:
+    fs: "xfs"
+    mount_point: "/mnt/app"
+    owner: 1000
+    group: 1000
+    permissions: 755
+    label: "external-vol"
+    mode: healthcheck

go.mod

@@ -0,0 +1,7 @@
+module ebs-bootstrap
+
+go 1.21
+
+require gopkg.in/yaml.v2 v2.4.0
+
+require github.com/google/go-cmp v0.6.0 // indirect

go.sum

@@ -0,0 +1,5 @@
+github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
+github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=