GitHub runner (#70)

feat(github-runner): add terraform for github runner
chore: update promtail priority to 90
chore: add nomad format hook
fix(promtail): add an update stanza to promtail
fix(promtail): add an update stanza to promtail
fix(promtail): add an update stanza to promtail
fix(promtail): add an update stanza to promtail
fix(github-runner): redesign the nomad job template to use different orgs
fix(github-runner): define per-org resources
fix(github-runner): add service to task
fix(github-runner): add a service with script check
fix(github-runner): use github token for runner checks

---------

Signed-off-by: Bruce Becker <brucellino@protonmail.com>

github-runner/github-runner.nomad.tpl

@@ -1,11 +1,38 @@
 job "github-runner-${org}" {
+
   name = "github-runner-${org}"
+
   datacenters = ["dc1"]
+  migrate {
+    max_parallel     = 2
+    health_check     = "task_states"
+    min_healthy_time = "30s"
+    healthy_deadline = "5m"
+  }
+  update {
+    max_parallel      = 3
+    health_check      = "task_states"
+    min_healthy_time  = "30s"
+    healthy_deadline  = "5m"
+    progress_deadline = "10m"
+    auto_revert       = true
+    auto_promote      = true
+    canary            = 1
+    stagger           = "30s"
+  }
+  reschedule {
+    interval       = "1h"
+    delay          = "30s"
+    delay_function = "exponential"
+    max_delay      = "120s"
+    unlimited      = true
+  }
+
   group "${org}" {
 
     task "configure" {
       env {
-        RUNNER_CFG_PAT = "${token}"
+        RUNNER_CFG_PAT = "${registration_token}"
       }
       lifecycle {
         hook = "prestart"
@@ -19,10 +46,9 @@ job "github-runner-${org}" {
       config {
         command = "$${NOMAD_ALLOC_DIR}/actions-runner/config.sh"
         args = [
-          "config.sh",
           "--unattended",
           "--url", "https://github.com/${org}",
-          "--token", "${token}",
+          "--token", "${registration_token}",
           "--labels", "hah",
           "--ephemeral"
         ]
@@ -34,6 +60,20 @@ job "github-runner-${org}" {
       config {
         command = "$${NOMAD_ALLOC_DIR}/actions-runner/run.sh"
       }
+
+      service {
+        name = "github-runner-${org}"
+        tags = ["github-runner-${org}"]
+        provider = "consul"
+        check {
+          interval = "15s"
+          timeout = "10s"
+          type = "script"
+          command = "$${NOMAD_ALLOC_DIR}/actions-runner/run.sh"
+          args = ["--check", "--url", "https://github.com/${org}", "--pat", "${check_token}" ]
+        }
+      }
+
       scaling "cpu" {
         enabled = true
         min     = 100
@@ -60,7 +100,7 @@ job "github-runner-${org}" {
         args = [
           "remove",
           "--token",
-          "${token}"
+          "${registration_token}"
         ]
       }
     } // remove task

github-runner/main.tf

@@ -1,3 +1,8 @@
+# Define the required providers for this workload.
+# We store the state in a Consul cluster.check
+# We create resources in Github and Nomad, which need authentication tokens.
+# The tokens are stored in Vault, which implies the use of the Vault provider.
+#
 terraform {
   backend "consul" {
     scheme = "http"
@@ -23,16 +28,15 @@ terraform {
   }
 }
 
-variable "org_name" {
-  description = "Name of the Github organisation"
-  default     = "SouthAfricaDigitalScience"
+variable "orgs" {
+  description = "Names of the Github organisations"
+  default     = ["AAROC", "Hashi-at-Home", "SouthAfricaDigitalScience"]
   sensitive   = false
-  type        = string
+  type        = set(string)
 }
 
-provider "vault" {
-  address = "http://sense:8200"
-}
+#
+provider "vault" {}
 
 provider "nomad" {}
 
@@ -42,26 +46,28 @@ data "vault_kv_secret_v2" "name" {
 }
 
 provider "github" {
-  token = data.vault_kv_secret_v2.name.data.personal
+  token = data.vault_kv_secret_v2.name.data.org_scope
 }
 
-data "github_organization" "sads" {
-  name = var.org_name
+data "github_organization" "selected" {
+  for_each = var.orgs
+  name     = each.value
 }
 
 locals {
-  runners_api_url = "https://api.github.com/orgs/${var.org_name}/actions/runners"
+  # runners_api_url = "https://api.github.com/orgs/${var.org_name}/actions/runners"
   headers = {
     "Accept"               = "application/vnd.github+json"
-    "Authorization"        = "Bearer ${data.vault_kv_secret_v2.name.data.personal}"
+    "Authorization"        = "Bearer ${data.vault_kv_secret_v2.name.data.org_scope}"
     "X-GitHub-Api-Version" = "2022-11-28"
   }
 }
 
 provider "http" {}
 
 data "http" "runners" {
-  url             = local.runners_api_url
+  for_each        = data.github_organization.selected
+  url             = "https://api.github.com/orgs/${each.value.orgname}/actions/runners"
   request_headers = local.headers
   lifecycle {
     postcondition {
@@ -71,8 +77,15 @@ data "http" "runners" {
   }
 }
 
+
+output "runner_urls" {
+  value = [for e in data.github_organization.selected : "https://api.github.com/orgs/${e.orgname}/actions/runners"]
+}
+
 data "http" "runner_reg_token" {
-  url             = "${local.runners_api_url}/registration-token"
+  for_each = data.github_organization.selected
+  # url             = "${local.runners_api_url}/registration-token"
+  url             = "https://api.github.com/orgs/${each.value.orgname}/actions/runners/registration-token"
   request_headers = local.headers
   method          = "POST"
   lifecycle {
@@ -84,11 +97,13 @@ data "http" "runner_reg_token" {
 }
 
 resource "vault_kv_secret_v2" "runner_registration_token" {
+  for_each = data.http.runner_reg_token
+
   mount = "kv"
-  name  = "github_runner"
+  name  = "github_runner/${each.key}"
   # cas                 = 1
   # delete_all_versions = true
-  data_json = data.http.runner_reg_token.response_body
+  data_json = each.value.body
   custom_metadata {
     data = {
       created_by = "Terraform"
@@ -97,16 +112,11 @@ resource "vault_kv_secret_v2" "runner_registration_token" {
 }
 
 resource "nomad_job" "runner" {
+  for_each = data.github_organization.selected
   jobspec = templatefile("github-runner.nomad.tpl", {
-    token          = jsondecode(vault_kv_secret_v2.runner_registration_token.data_json).token,
-    runner_version = "2.310.2",
-    org_name       = var.org_name
+    registration_token = jsondecode(vault_kv_secret_v2.runner_registration_token[each.key].data_json).token,
+    check_token        = jsondecode(data.vault_kv_secret_v2.name.data_json).runner_check,
+    runner_version     = "2.310.2",
+    org                = each.key
   })
 }
-
-resource "github_actions_runner_group" "arm64" {
-  allows_public_repositories = false
-  name                       = "hashi-at-home"
-  visibility                 = "private"
-  # default                    = false
-}