From 04a123d38fd891234a87c7bde9328e18a72ebaab Mon Sep 17 00:00:00 2001
From: Bruce Becker <brucellino@protonmail.com>
Date: Wed, 25 Oct 2023 00:43:28 +0200
Subject: [PATCH] feat(github-runner): add terraform for github runner (#64)

feat(github-runner): add terraform for github runner
chore: update promtail priority to 90
chore: add nomad format hook
fix(promtail): add an update stanza to promtail
fix(github-runner): redesign the nomad job template to use different orgs
Provide different job names and group names
---------
Signed-off-by: Bruce Becker <brucellino@protonmail.com>
---
 .pre-commit-config.yaml               |  11 ++
 github-runner/.terraform.lock.hcl     |  84 ++++++++++++
 github-runner/github-runner.nomad     |  69 ++++++++++
 github-runner/github-runner.nomad.tpl |  68 ++++++++++
 github-runner/main.tf                 | 112 ++++++++++++++++
 monitoring/monitoring.nomad           | 179 ++++++++++++++------------
 monitoring/promtail.nomad             |  84 ++++++------
 7 files changed, 482 insertions(+), 125 deletions(-)
 create mode 100644 github-runner/.terraform.lock.hcl
 create mode 100644 github-runner/github-runner.nomad
 create mode 100644 github-runner/github-runner.nomad.tpl
 create mode 100644 github-runner/main.tf

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 02deeba..a2b9638 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -26,5 +26,16 @@ repos:
     rev: v1.26.3
     hooks:
       - id: tfsec-system
+  - repo: local
+    hooks:
+      - id: format-jobspec
+        name: Format jobspec
+        language: system
+        entry: nomad
+        args:
+          - fmt
+        files: nomad
+        exclude: ".*tpl"
+
 ci:
   autoupdate_branch: main
diff --git a/github-runner/.terraform.lock.hcl b/github-runner/.terraform.lock.hcl
new file mode 100644
index 0000000..8d1a734
--- /dev/null
+++ b/github-runner/.terraform.lock.hcl
@@ -0,0 +1,84 @@
+# This file is maintained automatically by "terraform init".
+# Manual edits may be lost in future updates.
+
+provider "registry.terraform.io/hashicorp/http" {
+  version     = "3.4.0"
+  constraints = "~> 3.0"
+  hashes = [
+    "h1:h3URn6qAnP36OlSqI1tTuKgPL3GriZaJia9ZDrUvRdg=",
+    "zh:56712497a87bc4e91bbaf1a5a2be4b3f9cfa2384baeb20fc9fad0aff8f063914",
+    "zh:6661355e1090ebacab16a40ede35b029caffc279d67da73a000b6eecf0b58eba",
+    "zh:67b92d343e808b92d7e6c3bbcb9b9d5475fecfed0836963f7feb9d9908bd4c4f",
+    "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3",
+    "zh:86ebb9be9b685c96dbb5c024b55d87526d57a4b127796d6046344f8294d3f28e",
+    "zh:902be7cfca4308cba3e1e7ba6fc292629dfd150eb9a9f054a854fa1532b0ceba",
+    "zh:9ba26e0215cd53b21fe26a0a98c007de1348b7d13a75ae3cfaf7729e0f2c50bb",
+    "zh:a195c941e1f1526147134c257ff549bea4c89c953685acd3d48d9de7a38f39dc",
+    "zh:a7967b3d2a8c3e7e1dc9ae381ca753268f9fce756466fe2fc9e414ca2d85a92e",
+    "zh:bde56542e9a093434d96bea21c341285737c6d38fea2f05e12ba7b333f3e9c05",
+    "zh:c0306f76903024c497fd01f9fd9bace5854c263e87a97bc2e89dcc96d35ca3cc",
+    "zh:f9335a6c336171e85f8e3e99c3d31758811a19aeb21fa8c9013d427e155ae2a9",
+  ]
+}
+
+provider "registry.terraform.io/hashicorp/nomad" {
+  version     = "2.0.0"
+  constraints = "~> 2.0"
+  hashes = [
+    "h1:lIHIxA6ZmfyTGL3J9YIddhxlfit4ipSS09BLxkwo6L0=",
+    "zh:09b897d64db293f9a904a4a0849b11ec1e3fff5c638f734d82ae36d8dc044b72",
+    "zh:435cc106799290f64078ec24b6c59cb32b33784d609088638ed32c6d12121199",
+    "zh:7073444bd064e8c4ec115ca7d9d7f030cc56795c0a83c27f6668bba519e6849a",
+    "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3",
+    "zh:79d238c35d650d2d83a439716182da63f3b2767e72e4cbd0b69cb13d9b1aebfc",
+    "zh:7ef5f49344278fe0bbc5447424e6aa5425ff1821d010d944a444d7fa2c751acf",
+    "zh:92179091638c8ba03feef371c4361a790190f9955caea1fa59de2055c701a251",
+    "zh:a8a34398851761368eb8e7c171f24e55efa6e9fdbb5c455f6dec34dc17f631bc",
+    "zh:b38fd5338625ebace5a4a94cea1a28b11bd91995d834e318f47587cfaf6ec599",
+    "zh:b71b273a2aca7ad5f1e07c767b25b5a888881ba9ca93b30044ccc39c2937f03c",
+    "zh:cd14357e520e0f09fb25badfb4f2ee37d7741afdc3ed47c7bcf54c1683772543",
+    "zh:e05e025f4bb95138c3c8a75c636e97cd7cfd2fc1525b0c8bd097db8c5f02df6e",
+  ]
+}
+
+provider "registry.terraform.io/hashicorp/vault" {
+  version     = "3.21.0"
+  constraints = "~> 3.0"
+  hashes = [
+    "h1:QVDIGe1ZHq97ymVJlZw76h+bVxU+xvDYafyXYJdCJ+4=",
+    "zh:00ff2d3b7b4a516ab883640256f3b1b612faf55902cae5fd614ac546452308d7",
+    "zh:179074d94db888f1f30afd1567140b2c9f2ab5f1dfb3f110e15193a93b33963f",
+    "zh:1ebf2ba457eec518d0cf0302641fdaffef36dbae8726551241807c7a06e19544",
+    "zh:1ee696fc57284c75b94f45e9bd71f9d9dd040491f4d882f18c1f5b3dda3ffdfb",
+    "zh:3093f2fd2429a4aecb80bc4fe148cae63da9871d36fd0d5e84c621f1fa65e8c9",
+    "zh:43346defacf9051af4fe123185b9d8e796d145a9e037a432278b2b65f521214c",
+    "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3",
+    "zh:868000939a0e5027809941962cdfce4d0d3b8d02b212e91867aecf0031793381",
+    "zh:c7570f5409f9f647c5d70202ce64da3bb0a142e8dbe9e98a7bda0fde94886d72",
+    "zh:cb5e39cc981d61b530939ba1ff4804bc19e217f7317c0f691c69b9e2127cd596",
+    "zh:d3251b1b73d8b89c40587aee649daf10382613bb59a6615fc16df8838678733d",
+    "zh:d5434cd4d9028082745b7078f0deb67635258c7c1477ea9fdc4f9dd9fce233a7",
+  ]
+}
+
+provider "registry.terraform.io/integrations/github" {
+  version     = "5.40.0"
+  constraints = "~> 5.0"
+  hashes = [
+    "h1:pbFq74DVEMbu5xYUM2R2ouIjzlIA2WGg8u5jrmvecwk=",
+    "zh:02922b9eb54dcdbad524caaef7901a800759ae5d3a6c8cbdf934d4cfce395d5d",
+    "zh:282b9736c2afa9f4a7817d5da9ac0caeddb4edc085c7236b71f3ecbb539b2132",
+    "zh:49275a2a1b523da1794f0ea19dfc0f595d1ac8c711d24c6887bb672a5a571e01",
+    "zh:521bdef1fdd3211be6ba30edb3092acbed5974b62497d83028f0e8fd2d7bcd24",
+    "zh:62e8e8de673859a3c0c1e9fb0fd623e0012c8d9b44d45728ad1aa04d744b724c",
+    "zh:6fc1fbecf16b896f791d5708208295267f20cf4a62e393ecb93f70538306f30e",
+    "zh:6fef1d1c77ca2f77501a6ffd63640e0174905c7ae88aca516ede1a7263af8bc3",
+    "zh:923968a2eb3856db4e02b2e87dd0f09555531d09ad707c6dd53b1c9c3af0ff82",
+    "zh:95e409dfb3437171f66c97493c6ae9a75a7329c90a1d4c489663c6e3823d32af",
+    "zh:a45d0f1449bc2833974ba88e238a5ec0b41da64b18da626153ec0b650bf90e4c",
+    "zh:af5d8e506f3280ecf22824549c5b2b68ec047a0df9da0a19721b1a6303f77d5a",
+    "zh:ba03a3a81ebc68cc452930bca9289a5feb2b0d22d7997bea7f9e59bfd89babed",
+    "zh:d941df409e689f3deb06f912a57f2ee36ab223e1f48d395e3d43937e62a7fbea",
+    "zh:eaf71e9586b98c631218a67b1f1d7295ff224ba31b0c899e88e4aa217a160116",
+  ]
+}
diff --git a/github-runner/github-runner.nomad b/github-runner/github-runner.nomad
new file mode 100644
index 0000000..03025bd
--- /dev/null
+++ b/github-runner/github-runner.nomad
@@ -0,0 +1,69 @@
+variable "runner_version" {
+  description = "Version to use for the github runner.\nSee https://github.com/actions/runner/releases/"
+  default = "2.310.2"
+  type = string
+}
+
+variable "github_org" {
+  description = "Name of the github org we attach the runner to"
+  default = "SouthAfricaDigitalScience"
+  type = string
+}
+
+variable "token" {
+  description = "Github Personal Access Token"
+  default = "AAQEOZFGCRNN2DT7DBTYXMTEGKUB2"
+  type = string
+}
+job "github-runner" {
+  datacenters = ["dc1"]
+  group "main" {
+    task "configure" {
+      driver = "exec"
+      artifact {
+        source = "https://github.com/actions/runner/releases/download/v${var.runner_version}/actions-runner-linux-${attr.cpu.arch}-${var.runner_version}.tar.gz"
+      }
+      lifecycle {
+        hook = "prestart"
+        sidecar = false
+      }
+      config {
+        command = "/bin/bash"
+        args = [
+          "local/config.sh",
+          "--unattended",
+          "--url https://github.com/${var.github_org}",
+          "--token  ${var.token}",
+          "--labels test"
+        ]
+      }
+    }
+    task "run" {
+      env {
+        RUNNER_CFG_PAT = var.token
+      }
+      driver = "exec"
+      config {
+        command = "/bin/bash"
+        args = [
+          "local/run.sh"
+        ]
+      }
+    }
+    task "remove" {
+      lifecycle {
+        hook = "poststop"
+        sidecar = false
+      }
+      driver = "exec"
+      config {
+        command = "config.sh"
+        args = [
+          "remove",
+          "--token",
+          var.token
+        ]
+      }
+    }
+  }
+}
diff --git a/github-runner/github-runner.nomad.tpl b/github-runner/github-runner.nomad.tpl
new file mode 100644
index 0000000..bde043b
--- /dev/null
+++ b/github-runner/github-runner.nomad.tpl
@@ -0,0 +1,68 @@
+job "github-runner-${org}" {
+  name = "github-runner-${org}"
+  datacenters = ["dc1"]
+  group "${org}" {
+
+    task "configure" {
+      env {
+        RUNNER_CFG_PAT = "${token}"
+      }
+      lifecycle {
+        hook = "prestart"
+      }
+      driver = "exec"
+      artifact {
+        source = "https://github.com/actions/runner/releases/download/v${runner_version}/actions-runner-linux-$${attr.cpu.arch}-${runner_version}.tar.gz"
+        destination = "$${NOMAD_ALLOC_DIR}/actions-runner"
+        mode = "dir"
+      }
+      config {
+        command = "$${NOMAD_ALLOC_DIR}/actions-runner/config.sh"
+        args = [
+          "config.sh",
+          "--unattended",
+          "--url", "https://github.com/${org}",
+          "--token", "${token}",
+          "--labels", "hah",
+          "--ephemeral"
+        ]
+      }
+    }
+
+    task "launch" {
+      driver = "exec"
+      config {
+        command = "$${NOMAD_ALLOC_DIR}/actions-runner/run.sh"
+      }
+      scaling "cpu" {
+        enabled = true
+        min     = 100
+        max     = 150
+
+        policy {
+          cooldown = "5m"
+          evaluation_interval = "10s"
+          strategy "target-value" {
+            target = 2
+          }
+        }
+      }
+    }
+
+    task "remove" {
+      lifecycle {
+        hook = "poststop"
+        sidecar = false
+      }
+      driver = "exec"
+      config {
+        command = "$${NOMAD_ALLOC_DIR}/actions-runner/config.sh"
+        args = [
+          "remove",
+          "--token",
+          "${token}"
+        ]
+      }
+    } // remove task
+  } // task group
+}
diff --git a/github-runner/main.tf b/github-runner/main.tf
new file mode 100644
index 0000000..c94ce6e
--- /dev/null
+++ b/github-runner/main.tf
@@ -0,0 +1,112 @@
+terraform {
+  backend "consul" {
+    scheme = "http"
+    path   = "terraform/personal/github-runners"
+  }
+  required_providers {
+    vault = {
+      source  = "hashicorp/vault"
+      version = "~> 3.0"
+    }
+    github = {
+      source  = "integrations/github"
+      version = "~> 5.0"
+    }
+    http = {
+      source  = "hashicorp/http"
+      version = "~> 3.0"
+    }
+    nomad = {
+      source  = "hashicorp/nomad"
+      version = "~> 2.0"
+    }
+  }
+}
+
+variable "org_name" {
+  description = "Name of the Github organisation"
+  default     = "SouthAfricaDigitalScience"
+  sensitive   = false
+  type        = string
+}
+
+provider "vault" {
+  address = "http://sense:8200"
+}
+
+provider "nomad" {}
+
+data "vault_kv_secret_v2" "name" {
+  mount = "kv"
+  name  = "github"
+}
+
+provider "github" {
+  token = data.vault_kv_secret_v2.name.data.personal
+}
+
+data "github_organization" "sads" {
+  name = var.org_name
+}
+
+locals {
+  runners_api_url = "https://api.github.com/orgs/${var.org_name}/actions/runners"
+  headers = {
+    "Accept"               = "application/vnd.github+json"
+    "Authorization"        = "Bearer ${data.vault_kv_secret_v2.name.data.personal}"
+    "X-GitHub-Api-Version" = "2022-11-28"
+  }
+}
+
+provider "http" {}
+
+data "http" "runners" {
+  url             = local.runners_api_url
+  request_headers = local.headers
+  lifecycle {
+    postcondition {
+      condition     = contains([200], self.status_code)
+      error_message = "Error"
+    }
+  }
+}
+
+data "http" "runner_reg_token" {
+  url             = "${local.runners_api_url}/registration-token"
+  request_headers = local.headers
+  method          = "POST"
+  lifecycle {
+    postcondition {
+      condition     = contains([201, 204], self.status_code)
+      error_message = tostring(self.response_body)
+    }
+  }
+}
+
+resource "vault_kv_secret_v2" "runner_registration_token" {
+  mount = "kv"
+  name  = "github_runner"
+  # cas                 = 1
+  # delete_all_versions = true
+  data_json = data.http.runner_reg_token.response_body
+  custom_metadata {
+    data = {
+      created_by = "Terraform"
+    }
+  }
+}
+
+resource "nomad_job" "runner" {
+  jobspec = templatefile("github-runner.nomad.tpl", {
+    token          = jsondecode(vault_kv_secret_v2.runner_registration_token.data_json).token,
+    runner_version = "2.310.2",
+    org_name       = var.org_name
+  })
+}
+
+resource "github_actions_runner_group" "arm64" {
+  allows_public_repositories = false
+  name                       = "hashi-at-home"
+  visibility                 = "private"
+  # default                    = false
+}
diff --git a/monitoring/monitoring.nomad b/monitoring/monitoring.nomad
index b584985..1c94306 100644
--- a/monitoring/monitoring.nomad
+++ b/monitoring/monitoring.nomad
@@ -1,67 +1,68 @@
 variable "prom_version" {
-  default = "2.43.0"
-  type = string
+  default     = "2.43.0"
+  type        = string
   description = "Version of prometheus to use"
 }
 
 variable "prom_sha2" {
-  type = string
-  default = "79c4262a27495e5dff45a2ce85495be2394d3eecd51f0366c706f6c9c729f672" #pragma: allowlist secret
+  type        = string
+  default     = "79c4262a27495e5dff45a2ce85495be2394d3eecd51f0366c706f6c9c729f672" #pragma: allowlist secret
   description = "https://prometheus.io/download/"
 }
 
 variable "mimir_version" {
-  default = "2.8.0"
-  type = string
+  default     = "2.8.0"
+  type        = string
   description = "Version of mimir to use"
 }
 
 variable "mimir_sha2" {
-  type = string
-  default = "e7d2d401f616b185bded25cfe84f7b6543e169f4d0d8a36e19f7ba124848b712" #pragma: allowlist secret
+  type        = string
+  default     = "e7d2d401f616b185bded25cfe84f7b6543e169f4d0d8a36e19f7ba124848b712" #pragma: allowlist secret
   description = "https://prometheus.io/download/"
 }
 
 variable "grafana_version" {
-  type = string
-  default = "9.4.7"
+  type        = string
+  default     = "9.4.7"
   description = "Grafana version"
 }
 
 job "monitoring" {
   datacenters = ["dc1"]
   type        = "service"
+  priority    = "60"
   meta {
-    auto-backup = true
-    backup-schedule = "@daily"
+    auto-backup      = true
+    backup-schedule  = "@daily"
     backup-target-db = "postgres"
   }
   update {
     max_parallel = 2
     health_check = "checks"
-    canary = 1
+    canary       = 1
     auto_promote = true
-    auto_revert = true
+    auto_revert  = true
   }
 
   migrate {
-    max_parallel = 1
-    health_check = "checks"
+    max_parallel     = 1
+    health_check     = "checks"
     min_healthy_time = "30s"
     healthy_deadline = "10m"
   }
   constraint {
-     attribute = attr.cpu.arch
-     value     = "arm64"
+    attribute = attr.cpu.arch
+    value     = "arm64"
   }
 
   group "prometheus" {
     count = 1
-    volume "data" {
-      type      = "host"
-      read_only = false
-      source    = "scratch"
-    }
+    // volume "data" {
+    //   type      = "host"
+    //   read_only = false
+    //   source    = "scratch"
+    // }
     network {
       port "prometheus_ui" {}
     }
@@ -74,9 +75,9 @@ job "monitoring" {
     }
 
     reschedule {
-      delay = "5m"
+      delay          = "5m"
       delay_function = "fibonacci"
-      unlimited = true
+      unlimited      = true
     }
 
     ephemeral_disk {
@@ -93,10 +94,10 @@ job "monitoring" {
         }
       }
       template {
-        change_mode = "signal"
+        change_mode   = "signal"
         change_signal = "SIGHUP"
-        destination = "local/prometheus.yml"
-        data = file("templates/prometheus.yml.tpl")
+        destination   = "local/prometheus.yml"
+        data          = file("templates/prometheus.yml.tpl")
         wait {
           min = "10s"
           max = "20s"
@@ -104,9 +105,9 @@ job "monitoring" {
       }
 
       template {
-        change_mode = "noop"
-        destination = "local/node-rules.yml"
-        left_delimiter = "[["
+        change_mode     = "noop"
+        destination     = "local/node-rules.yml"
+        left_delimiter  = "[["
         right_delimiter = "]]"
         wait {
           min = "10s"
@@ -118,7 +119,7 @@ job "monitoring" {
 
       config {
         command = "local/prometheus-${var.prom_version}.linux-arm64/prometheus"
-        args    = [
+        args = [
           "--config.file=local/prometheus.yml",
           "--storage.tsdb.retention.size=1GB",
           "--storage.tsdb.retention.time=7d",
@@ -127,13 +128,13 @@ job "monitoring" {
           "--storage.tsdb.path=data"
         ]
       }
-      volume_mount {
-        volume      = "data"
-        destination = "data"
-        read_only   = false
-      }
+      // volume_mount {
+      //   volume      = "data"
+      //   destination = "data"
+      //   read_only   = false
+      // }
       resources {
-        cpu = 250
+        cpu    = 250
         memory = 400
       }
 
@@ -162,11 +163,11 @@ job "monitoring" {
 
   group "mimir" {
     count = 1
-    volume "data" {
-      type      = "host"
-      read_only = false
-      source    = "scratch"
-    }
+    // volume "data" {
+    //   type      = "host"
+    //   read_only = false
+    //   source    = "scratch"
+    // }
     network {
       port "mimir_ui" {}
     }
@@ -179,9 +180,9 @@ job "monitoring" {
     }
 
     reschedule {
-      delay = "5m"
+      delay          = "5m"
       delay_function = "fibonacci"
-      unlimited = true
+      unlimited      = true
     }
 
     ephemeral_disk {
@@ -190,8 +191,8 @@ job "monitoring" {
 
     task "mimir" {
       vault {
-        policies = ["read-only"]
-        change_mode = "restart"
+        policies      = ["read-only"]
+        change_mode   = "restart"
         change_signal = "SIGHUP"
       }
       artifact {
@@ -202,10 +203,10 @@ job "monitoring" {
         }
       }
       template {
-        change_mode = "signal"
+        change_mode   = "signal"
         change_signal = "SIGHUP"
-        destination = "local/mimir.yml"
-        data = file("templates/mimir.yml.tpl")
+        destination   = "local/mimir.yml"
+        data          = file("templates/mimir.yml.tpl")
         wait {
           min = "10s"
           max = "20s"
@@ -216,18 +217,18 @@ job "monitoring" {
 
       config {
         command = "local/mimir-linux-arm64"
-        args    = [
+        args = [
           "-server.http-listen-port=${NOMAD_PORT_mimir_ui}",
           "--config.file=local/mimir.yml"
         ]
       }
-      volume_mount {
-        volume      = "data"
-        destination = "data"
-        read_only   = false
-      }
+      // volume_mount {
+      //   volume      = "data"
+      //   destination = "data"
+      //   read_only   = false
+      // }
       resources {
-        cpu = 250
+        cpu    = 250
         memory = 400
       }
 
@@ -236,6 +237,8 @@ job "monitoring" {
         tags = ["urlprefix-/mimir strip=/mimir"]
         port = "mimir_ui"
 
+        provider = "consul"
+
         check {
           name     = "mimir_readiness check"
           type     = "http"
@@ -252,7 +255,7 @@ job "monitoring" {
     network {
       port "mysql_server" {
         static = 3306
-        to = 3306
+        to     = 3306
       }
       mode = "host"
     }
@@ -278,19 +281,19 @@ job "monitoring" {
       // }
 
       check {
-        type = "tcp"
-        name = "mysql_alive"
+        type     = "tcp"
+        name     = "mysql_alive"
         interval = "5s"
-        timeout = "2s"
-        port = "mysql_server"
+        timeout  = "2s"
+        port     = "mysql_server"
       }
     }
 
     restart {
       attempts = 1
       interval = "10m"
-      delay = "15s"
-      mode = "fail"
+      delay    = "15s"
+      mode     = "fail"
     }
 
     update {
@@ -304,8 +307,8 @@ job "monitoring" {
     }
 
     migrate {
-      max_parallel = 1
-      health_check = "checks"
+      max_parallel     = 1
+      health_check     = "checks"
       min_healthy_time = "30s"
       healthy_deadline = "10m"
     }
@@ -313,15 +316,15 @@ job "monitoring" {
     task "mysql" {
       driver = "podman"
       config {
-        image = "docker://arm64v8/mysql:oracle"
-        ports = ["mysql_server"]
+        image        = "docker://arm64v8/mysql:oracle"
+        ports        = ["mysql_server"]
         network_mode = "host"
       }
       env {
         MYSQL_ROOT_PASSWORD = "password" # pragma: allowlist secret
-        MYSQL_USER = "mysql"
-        MYSQL_PASSWORD = "password" # pragma: allowlist secret
-        MYSQL_DATABASE = "grafana"
+        MYSQL_USER          = "mysql"
+        MYSQL_PASSWORD      = "password" # pragma: allowlist secret
+        MYSQL_DATABASE      = "grafana"
       }
       resources {
         cpu    = 125
@@ -336,13 +339,21 @@ job "monitoring" {
       port "grafana_server" {}
     }
 
+    // volume "grafana_data" {
+    //   type = "csi"
+    //   source = "grafana"
+    //   read_only = false
+    //   attachment_mode = "file-system"
+    //   access_mode = "single-node-writer"
+    // }
+
     service {
       name = "grafana"
       tags = ["urlprefix-/grafana strip=/grafana"]
       port = "grafana_server"
 
       check {
-        port = "grafana_server"
+        port     = "grafana_server"
         name     = "grafana-api"
         path     = "/api/health"
         type     = "http"
@@ -354,8 +365,8 @@ job "monitoring" {
     restart {
       attempts = 1
       interval = "2m"
-      delay = "15s"
-      mode = "fail"
+      delay    = "15s"
+      mode     = "fail"
     }
 
     # Select ARMv7 machines
@@ -382,8 +393,8 @@ job "monitoring" {
     }
 
     migrate {
-      max_parallel = 1
-      health_check = "checks"
+      max_parallel     = 1
+      health_check     = "checks"
       min_healthy_time = "15s"
       healthy_deadline = "10m"
     }
@@ -393,8 +404,8 @@ job "monitoring" {
     }
 
     vault {
-      policies = ["read-only"]
-      change_mode = "restart"
+      policies      = ["read-only"]
+      change_mode   = "restart"
       change_signal = "SIGHUP"
     }
 
@@ -405,8 +416,12 @@ job "monitoring" {
       driver = "exec"
       config {
         command = "sh"
-        args = ["-c", "while ! nc -z mysql.service.consul 3306 ; do sleep 1 ; done"]
+        args    = ["-c", "while ! nc -z mysql.service.consul 3306 ; do sleep 1 ; done"]
       }
+      // volume_mount {
+      // volume = "grafana_data"
+      // destination = "${NOMAD_ALLOC_DIR}/data"
+      // }
     }
 
     task "grafana" {
@@ -417,7 +432,7 @@ job "monitoring" {
       }
       artifact {
         // source = local.grafana_url
-        source = "https://dl.grafana.com/oss/release/grafana-${var.grafana_version}.linux-arm64.tar.gz"
+        source      = "https://dl.grafana.com/oss/release/grafana-${var.grafana_version}.linux-arm64.tar.gz"
         destination = "${NOMAD_ALLOC_DIR}"
       }
       resources {
@@ -438,6 +453,6 @@ job "monitoring" {
 
         destination = "${NOMAD_ALLOC_DIR}/grafana-${var.grafana_version}/conf/conf.ini"
       } // Configuration template
-    } // Grafana server task
-  } // grafana server group
+    }   // Grafana server task
+  }     // grafana server group
 }
diff --git a/monitoring/promtail.nomad b/monitoring/promtail.nomad
index 965f3d9..ddfe8e2 100644
--- a/monitoring/promtail.nomad
+++ b/monitoring/promtail.nomad
@@ -1,25 +1,35 @@
 variable "promtail_version" {
   description = "Version of Promtail to deploy"
-  type = string
-  default = "2.9.1"
+  type        = string
+  default     = "2.9.1"
 }
 
 job "promtail" {
-
+  priority = "90"
   meta {
-    auto-backup = true
-    backup-schedule = "@daily"
+    auto-backup      = true
+    backup-schedule  = "@daily"
     backup-target-db = "postgres"
   }
   datacenters = ["dc1"]
-  type = "system"
-
+  type        = "system"
+  update {
+    max_parallel      = 2
+    health_check      = "checks"
+    min_healthy_time  = "10s"
+    healthy_deadline  = "5m"
+    progress_deadline = "10m"
+    auto_revert       = true
+    auto_promote      = true
+    canary            = 1
+    stagger           = "30s"
+  }
   group "promtail" {
     count = 1
     update {
-      max_parallel = 3
-      canary = 0
-      stagger = "30s"
+      max_parallel = 2
+      canary       = 1
+      stagger      = "30s"
     }
     network {
       port "http" {}
@@ -39,16 +49,16 @@ job "promtail" {
       }
 
       check {
-        name = "Promtail HTTP"
-        type = "http"
-        path = "/ready"
+        name     = "Promtail HTTP"
+        type     = "http"
+        path     = "/ready"
         interval = "10s"
-        timeout = "5s"
-        port = "http"
+        timeout  = "5s"
+        port     = "http"
 
         check_restart {
-          limit = 2
-          grace = "60s"
+          limit           = 2
+          grace           = "60s"
           ignore_warnings = false
         }
       }
@@ -58,31 +68,19 @@ job "promtail" {
       name = "promtail-grpc"
       tags = ["grpc"]
       port = "grpc"
-
-      // check {
-      //   name = "promtail-grpc"
-      //   grpc_service = "promtail-grpc"
-      //   type = "grpc"
-      //   interval = "15s"
-      //   timeout = "5s"
-      //   port = "grpc"
-      //   grpc_use_tls = false
-      //   tls_skip_verify = true
-      // }
-
     }
 
     restart {
       attempts = 1
       interval = "10m"
-      delay = "15s"
-      mode = "delay"
+      delay    = "15s"
+      mode     = "delay"
     }
 
     ephemeral_disk {
-      size = 11
+      size    = 11
       migrate = true
-      sticky = true
+      sticky  = true
     }
 
     task "promtail" {
@@ -92,30 +90,30 @@ job "promtail" {
 
       config {
         command = "promtail"
-        args = ["-config.file=local/promtail.yml"]
+        args    = ["-config.file=local/promtail.yml"]
       }
 
       artifact {
-        source = "https://github.com/grafana/loki/releases/download/v${var.promtail_version}/promtail-linux-${attr.cpu.arch}.zip"
+        source      = "https://github.com/grafana/loki/releases/download/v${var.promtail_version}/promtail-linux-${attr.cpu.arch}.zip"
         destination = "local/promtail"
-        mode = "file"
+        mode        = "file"
       }
       logs {
-        max_files = 1
+        max_files     = 1
         max_file_size = 10
       }
 
       resources {
-        cpu    = 60 # 500 MHz
-        memory = 125 # 256MB
+        cpu    = 250 # 500 MHz
+        memory = 150 # 256MB
       }
 
 
       template {
-         data          = file("templates/promtail.yml.tpl")
-         destination   = "local/promtail.yml"
-         change_mode   = "signal"
-         change_signal = "SIGHUP"
+        data          = file("templates/promtail.yml.tpl")
+        destination   = "local/promtail.yml"
+        change_mode   = "signal"
+        change_signal = "SIGHUP"
       }
     }
   }