From 92baf6fabd330a230480d526679e6f038b6e9cc9 Mon Sep 17 00:00:00 2001 From: Bruce Becker Date: Fri, 27 Oct 2023 14:47:12 +0200 Subject: [PATCH] GitHub runner (#70) feat(github-runner): add terraform for github runner chore: update promtail priority to 90 chore: add nomad format hook fix(promtail): add an update stanza to promtail fix(promtail): add an update stanza to promtail fix(promtail): add an update stanza to promtail fix(promtail): add an update stanza to promtail fix(github-runner): redesign the nomad job template to use different orgs fix(github-runner): define per-org resources fix(github-runner): add service to task fix(github-runner): add a service with script check fix(github-runner): use github token for runner checks --------- Signed-off-by: Bruce Becker --- github-runner/github-runner.nomad.tpl | 48 +++++++++++++++++++-- github-runner/main.tf | 62 ++++++++++++++++----------- 2 files changed, 80 insertions(+), 30 deletions(-) diff --git a/github-runner/github-runner.nomad.tpl b/github-runner/github-runner.nomad.tpl index bde043b..2ca9471 100644 --- a/github-runner/github-runner.nomad.tpl +++ b/github-runner/github-runner.nomad.tpl @@ -1,11 +1,38 @@ job "github-runner-${org}" { + name = "github-runner-${org}" + datacenters = ["dc1"] + migrate { + max_parallel = 2 + health_check = "task_states" + min_healthy_time = "30s" + healthy_deadline = "5m" + } + update { + max_parallel = 3 + health_check = "task_states" + min_healthy_time = "30s" + healthy_deadline = "5m" + progress_deadline = "10m" + auto_revert = true + auto_promote = true + canary = 1 + stagger = "30s" + } + reschedule { + interval = "1h" + delay = "30s" + delay_function = "exponential" + max_delay = "120s" + unlimited = true + } + group "${org}" { task "configure" { env { - RUNNER_CFG_PAT = "${token}" + RUNNER_CFG_PAT = "${registration_token}" } lifecycle { hook = "prestart" @@ -19,10 +46,9 @@ job "github-runner-${org}" { config { command = "$${NOMAD_ALLOC_DIR}/actions-runner/config.sh" args = [ - "config.sh", "--unattended", "--url", "https://github.com/${org}", - "--token", "${token}", + "--token", "${registration_token}", "--labels", "hah", "--ephemeral" ] @@ -34,6 +60,20 @@ job "github-runner-${org}" { config { command = "$${NOMAD_ALLOC_DIR}/actions-runner/run.sh" } + + service { + name = "github-runner-${org}" + tags = ["github-runner-${org}"] + provider = "consul" + check { + interval = "15s" + timeout = "10s" + type = "script" + command = "$${NOMAD_ALLOC_DIR}/actions-runner/run.sh" + args = ["--check", "--url", "https://github.com/${org}", "--pat", "${check_token}" ] + } + } + scaling "cpu" { enabled = true min = 100 @@ -60,7 +100,7 @@ job "github-runner-${org}" { args = [ "remove", "--token", - "${token}" + "${registration_token}" ] } } // remove task diff --git a/github-runner/main.tf b/github-runner/main.tf index c94ce6e..ef799f4 100644 --- a/github-runner/main.tf +++ b/github-runner/main.tf @@ -1,3 +1,8 @@ +# Define the required providers for this workload. +# We store the state in a Consul cluster.check +# We create resources in Github and Nomad, which need authentication tokens. +# The tokens are stored in Vault, which implies the use of the Vault provider. +# terraform { backend "consul" { scheme = "http" @@ -23,16 +28,15 @@ terraform { } } -variable "org_name" { - description = "Name of the Github organisation" - default = "SouthAfricaDigitalScience" +variable "orgs" { + description = "Names of the Github organisations" + default = ["AAROC", "Hashi-at-Home", "SouthAfricaDigitalScience"] sensitive = false - type = string + type = set(string) } -provider "vault" { - address = "http://sense:8200" -} +# +provider "vault" {} provider "nomad" {} @@ -42,18 +46,19 @@ data "vault_kv_secret_v2" "name" { } provider "github" { - token = data.vault_kv_secret_v2.name.data.personal + token = data.vault_kv_secret_v2.name.data.org_scope } -data "github_organization" "sads" { - name = var.org_name +data "github_organization" "selected" { + for_each = var.orgs + name = each.value } locals { - runners_api_url = "https://api.github.com/orgs/${var.org_name}/actions/runners" + # runners_api_url = "https://api.github.com/orgs/${var.org_name}/actions/runners" headers = { "Accept" = "application/vnd.github+json" - "Authorization" = "Bearer ${data.vault_kv_secret_v2.name.data.personal}" + "Authorization" = "Bearer ${data.vault_kv_secret_v2.name.data.org_scope}" "X-GitHub-Api-Version" = "2022-11-28" } } @@ -61,7 +66,8 @@ locals { provider "http" {} data "http" "runners" { - url = local.runners_api_url + for_each = data.github_organization.selected + url = "https://api.github.com/orgs/${each.value.orgname}/actions/runners" request_headers = local.headers lifecycle { postcondition { @@ -71,8 +77,15 @@ data "http" "runners" { } } + +output "runner_urls" { + value = [for e in data.github_organization.selected : "https://api.github.com/orgs/${e.orgname}/actions/runners"] +} + data "http" "runner_reg_token" { - url = "${local.runners_api_url}/registration-token" + for_each = data.github_organization.selected + # url = "${local.runners_api_url}/registration-token" + url = "https://api.github.com/orgs/${each.value.orgname}/actions/runners/registration-token" request_headers = local.headers method = "POST" lifecycle { @@ -84,11 +97,13 @@ data "http" "runner_reg_token" { } resource "vault_kv_secret_v2" "runner_registration_token" { + for_each = data.http.runner_reg_token + mount = "kv" - name = "github_runner" + name = "github_runner/${each.key}" # cas = 1 # delete_all_versions = true - data_json = data.http.runner_reg_token.response_body + data_json = each.value.body custom_metadata { data = { created_by = "Terraform" @@ -97,16 +112,11 @@ resource "vault_kv_secret_v2" "runner_registration_token" { } resource "nomad_job" "runner" { + for_each = data.github_organization.selected jobspec = templatefile("github-runner.nomad.tpl", { - token = jsondecode(vault_kv_secret_v2.runner_registration_token.data_json).token, - runner_version = "2.310.2", - org_name = var.org_name + registration_token = jsondecode(vault_kv_secret_v2.runner_registration_token[each.key].data_json).token, + check_token = jsondecode(data.vault_kv_secret_v2.name.data_json).runner_check, + runner_version = "2.310.2", + org = each.key }) } - -resource "github_actions_runner_group" "arm64" { - allows_public_repositories = false - name = "hashi-at-home" - visibility = "private" - # default = false -}