Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Ghaf infra new environments #276

Open
wants to merge 5 commits into
base: main
Choose a base branch
from
+445 −547
Open

Ghaf infra new environments #276

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
e8d895a
terraform: New environments storage setup
henrirosten Oct 9, 2024
27a693a
terraform: Initial release config
henrirosten Oct 17, 2024
b62b3b4
terraform: Move jenkins artifacts to persistent
henrirosten Oct 19, 2024
da630ec
terraform: Update documentation
henrirosten Oct 21, 2024
a0d1735
terraform: Remove terraform-playground.sh
henrirosten Oct 21, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
160 changes: 64 additions & 96 deletions terraform/README.md
View file
Open in desktop

Large diffs are not rendered by default.

4 changes: 2 additions & 2 deletions terraform/jenkins-controller.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ module "jenkins_controller_vm" {
"path" = "/var/lib/rclone-http/env"
},
{
content = "AZURE_STORAGE_ACCOUNT_NAME=${azurerm_storage_account.jenkins_artifacts.name}",
content = "AZURE_STORAGE_ACCOUNT_NAME=${data.azurerm_storage_account.jenkins_artifacts.name}",
"path" = "/var/lib/rclone-jenkins-artifacts/env"
},
# Render /etc/nix/machines with terraform. In the future, we might want to
Expand Down Expand Up @@ -190,7 +190,7 @@ resource "azurerm_role_assignment" "jenkins_controller_access_storage" {

# Allow the VM to *write* to (and read from) the jenkins artifacts bucket
resource "azurerm_role_assignment" "jenkins_controller_access_artifacts" {
scope = azurerm_storage_container.jenkins_artifacts_1.resource_manager_id
scope = data.azurerm_storage_container.jenkins_artifacts_1.resource_manager_id
role_definition_name = "Storage Blob Data Contributor"
principal_id = module.jenkins_controller_vm.virtual_machine_identity_principal_id
}
Expand Down
197 changes: 101 additions & 96 deletions terraform/main.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,11 +22,6 @@ terraform {
source = "numtide/secret"
}
}
}

################################################################################

terraform {
# Backend for storing terraform state (see ../state-storage)
backend "azurerm" {
# resource_group_name and storage_account_name are set by the callee
Expand All @@ -41,16 +36,6 @@ terraform {
# Current signed-in user
data "azurerm_client_config" "current" {}

variable "envtype" {
type = string
description = "Set the environment type; determines e.g. the Azure VM sizes"
default = "priv"
validation {
condition = contains(["priv", "dev", "prod"], var.envtype)
error_message = "Must be either \"priv\", \"dev\", or \"prod\""
}
}

variable "envfile" {
type = string
description = "Error out if .env-file is missing"
Expand All @@ -63,7 +48,7 @@ variable "envfile" {

variable "convince" {
type = bool
description = "Protect against accidental dev or prod environment deployment"
description = "Protect against accidental non-priv environment deployment"
default = false
}

Expand All @@ -87,23 +72,31 @@ locals {
# https://github.com/claranet/terraform-azurerm-regions/blob/master/REGIONS.md
shortloc = module.azure_region.location_short

assert_region_mismatch_1 = regex(
("${local.shortloc}" != "eun" && local.envs["storage_account_rg_name"] != "ghaf-infra-state-${local.shortloc}") ?
"((Force invalid regex pattern)\n\nERROR: region (non-eun) mismatch, re-run terraform-init.sh" : "", "")

assert_region_mismatch_2 = regex(
("${local.shortloc}" == "eun" && local.envs["storage_account_rg_name"] != "ghaf-infra-state") ?
"((Force invalid regex pattern)\n\nERROR: region (eun) mismatch, re-run terraform-init.sh" : "", "")

# Sanitize workspace name
ws = substr(replace(lower(terraform.workspace), "/[^a-z0-9]/", ""), 0, 16)

ext_builder_machines = [
"ssh://remote-build@builder.vedenemo.dev x86_64-linux /etc/secrets/remote-build-ssh-key 32 3 kvm,nixos-test,benchmark,big-parallel - -",
"ssh://remote-build@hetzarm.vedenemo.dev aarch64-linux /etc/secrets/remote-build-ssh-key 40 3 kvm,nixos-test,benchmark,big-parallel - -"
]
ext_builder_keyscan = ["builder.vedenemo.dev", "hetzarm.vedenemo.dev"]
binary_cache_url_common = "https://ghaf-binary-cache-${local.ws}.${azurerm_resource_group.infra.location}.cloudapp.azure.com"
# TODO: adding multiple urls as comma-and-whitespace separated
# string is more or less a hack. If we plan to have multiple domains
# per host permanently, we could make the below variable a list, and
# do the templating to a comma-and-whitespace separated list of urls
# before we pass it to caddy.
binary_cache_url_dev = "${local.binary_cache_url_common}, https://dev-cache.vedenemo.dev"
binary_cache_url_prod = "${local.binary_cache_url_common}, https://prod-cache.vedenemo.dev"

# Environment-specific configuration options.
# See Azure vm sizes and specs at:
# https://azure.microsoft.com/en-us/pricing/details/virtual-machines/linux
# E.g. 'Standard_D2_v3' means: 2 vCPU, 8 GiB RAM
opts = {
priv = {
builder_sshkey_id = "ext"
persistent_id = "priv"
vm_size_binarycache = "Standard_D2_v3"
osdisk_size_binarycache = "50"
vm_size_builder_x86 = "Standard_D2_v3"
Expand All @@ -113,16 +106,14 @@ locals {
osdisk_size_controller = "150"
num_builders_x86 = 0
num_builders_aarch64 = 0
# 'priv' and 'dev' environments use the same binary cache signing key
binary_cache_public_key = "ghaf-infra-dev:EdgcUJsErufZitluMOYmoJDMQE+HFyveI/D270Cr84I="
binary_cache_url = "https://ghaf-binary-cache-${local.ws}.${azurerm_resource_group.infra.location}.cloudapp.azure.com"
ext_builder_machines = [
"ssh://remote-build@builder.vedenemo.dev x86_64-linux /etc/secrets/remote-build-ssh-key 32 3 kvm,nixos-test,benchmark,big-parallel - -",
"ssh://remote-build@hetzarm.vedenemo.dev aarch64-linux /etc/secrets/remote-build-ssh-key 40 3 kvm,nixos-test,benchmark,big-parallel - -"
]
ext_builder_keyscan = ["builder.vedenemo.dev", "hetzarm.vedenemo.dev"]
binary_cache_public_key = "priv-cache.vedenemo.dev~1:FmJGfAkx+2fhqpzHGT/V3M35VcPm2pfkCuiTo8xQD0A="
binary_cache_url = local.binary_cache_url_common
ext_builder_machines = local.ext_builder_machines
ext_builder_keyscan = local.ext_builder_keyscan
}
dev = {
builder_sshkey_id = "ext"
persistent_id = "prod"
karim20230 marked this conversation as resolved.
Show resolved Hide resolved
vm_size_binarycache = "Standard_D4_v3"
osdisk_size_binarycache = "250"
vm_size_builder_x86 = "Standard_D16_v3"
Expand All @@ -132,21 +123,33 @@ locals {
osdisk_size_controller = "1000"
num_builders_x86 = 0
num_builders_aarch64 = 0
# 'priv' and 'dev' environments use the same binary cache signing key
binary_cache_public_key = "ghaf-infra-dev:EdgcUJsErufZitluMOYmoJDMQE+HFyveI/D270Cr84I="
# TODO: adding multiple urls as comma-and-whitespace separated
# string is more or less a hack. If we plan to have multiple domains
# per host permanently, we could make the below variable a list, and
# do the templating to a comma-and-whitespace separated list of urls
# before we pass it to caddy.
binary_cache_url = "https://ghaf-binary-cache-${local.ws}.${azurerm_resource_group.infra.location}.cloudapp.azure.com, https://dev-cache.vedenemo.dev"
ext_builder_machines = [
"ssh://remote-build@builder.vedenemo.dev x86_64-linux /etc/secrets/remote-build-ssh-key 32 3 kvm,nixos-test,benchmark,big-parallel - -",
"ssh://remote-build@hetzarm.vedenemo.dev aarch64-linux /etc/secrets/remote-build-ssh-key 40 3 kvm,nixos-test,benchmark,big-parallel - -"
]
ext_builder_keyscan = ["builder.vedenemo.dev", "hetzarm.vedenemo.dev"]
# 'dev' and 'prod' use the same binary cache storage and key
binary_cache_public_key = "prod-cache.vedenemo.dev~1:JcytRNMJJdYJVQCYwLNsrfVhct5dhCK2D3fa6O1WHOI="
binary_cache_url = local.binary_cache_url_dev
ext_builder_machines = local.ext_builder_machines
ext_builder_keyscan = local.ext_builder_keyscan
}
prod = {
builder_sshkey_id = "ext"
persistent_id = "prod"
vm_size_binarycache = "Standard_D4_v3"
osdisk_size_binarycache = "250"
vm_size_builder_x86 = "Standard_D16_v3"
vm_size_builder_aarch64 = "Standard_D8ps_v5"
osdisk_size_builder = "250"
vm_size_controller = "Standard_E4_v5"
osdisk_size_controller = "1000"
num_builders_x86 = 0
num_builders_aarch64 = 0
# 'dev' and 'prod' use the same binary cache storage and key
binary_cache_public_key = "prod-cache.vedenemo.dev~1:JcytRNMJJdYJVQCYwLNsrfVhct5dhCK2D3fa6O1WHOI="
binary_cache_url = local.binary_cache_url_prod
ext_builder_machines = local.ext_builder_machines
ext_builder_keyscan = local.ext_builder_keyscan
}
release = {
builder_sshkey_id = "release"
persistent_id = "release"
vm_size_binarycache = "Standard_D4_v3"
osdisk_size_binarycache = "250"
vm_size_builder_x86 = "Standard_D64_v3"
Expand All @@ -156,8 +159,8 @@ locals {
osdisk_size_controller = "1000"
num_builders_x86 = 1
num_builders_aarch64 = 1
binary_cache_public_key = "ghaf-infra-prod:DIrhJsqehIxjuUQ93Fqx6gmo4cDgn5srW5dedvMbqD0="
binary_cache_url = "https://ghaf-binary-cache-${local.ws}.${azurerm_resource_group.infra.location}.cloudapp.azure.com"
binary_cache_public_key = "release-cache.vedenemo.dev~1:kxSUdZvNF8ax7hpJMu+PexEBQGUkZDqeugu+pwz/ACk="
binary_cache_url = local.binary_cache_url_common
ext_builder_machines = []
ext_builder_keyscan = []
}
Expand All @@ -166,24 +169,23 @@ locals {
# Read ssh-keys.yaml into local.ssh_keys
ssh_keys = yamldecode(file("../ssh-keys.yaml"))

# This determines the configuration options used in the
# ghaf-infra instance (defines e.g. vm_sizes and number of builders).
# If workspace name is "dev" or "prod" use the workspace name as
# envtype, otherwise, use the value from var.envtype.
conf = local.ws == "dev" || local.ws == "prod" ? local.ws : var.envtype
# Determine the configuration options used in the ghaf-infra instance
# based on the workspace name
is_release = length(regexall("^release.*", local.ws)) > 0
is_prod = length(regexall("^prod.*", local.ws)) > 0
is_dev = length(regexall("^dev.*", local.ws)) > 0
conf = local.is_release ? "release" : local.is_prod ? "prod" : local.is_dev ? "dev" : "priv"

# Protect against accidental dev or prod environment deployment by requiring
# Protect against accidental non-priv environment deployment by requiring
# variable -var="convince=true".
assert_accidental_deployment = regex(
("${local.conf}" != "priv" && !(var.convince)) ?
"((Force invalid regex pattern\n\nERROR: Deployment to 'prod' or 'dev' requires variable 'convince'" : "", "")

# Selects the persistent data (see ./persistent) used in the ghaf-infra
# instance; currently either "dev" or "prod" based on the environment type:
# "priv" ==> "dev"
# "dev" ==> "dev"
# "prod" ==> "prod"
persistent_data = local.conf == "priv" ? "dev" : local.conf
"((Force invalid regex pattern\n\nERROR: Deployment to non-priv requires variable 'convince'" : "", "")

# Selects the persistent data for this ghaf-infra instance (see ./persistent)
persistent_rg = local.envs["persistent_rg_name"]
builder_sshkey_id = "id0${local.opts[local.conf].builder_sshkey_id}${local.shortloc}"
persistent_id = "id0${local.opts[local.conf].persistent_id}${local.shortloc}"
}

################################################################################
Expand All @@ -196,6 +198,8 @@ resource "azurerm_resource_group" "infra" {

################################################################################

# Environment specific resources

# Virtual network
resource "azurerm_virtual_network" "vnet" {
name = "ghaf-infra-vnet"
Expand All @@ -220,10 +224,7 @@ resource "azurerm_subnet" "builders" {
address_prefixes = ["10.0.4.0/28"]
}

################################################################################

# Storage account and storage container used to store VM images

resource "azurerm_storage_account" "vm_images" {
name = "img${local.ws}${local.shortloc}"
resource_group_name = azurerm_resource_group.infra.name
Expand All @@ -241,43 +242,22 @@ resource "azurerm_storage_container" "vm_images" {

################################################################################

# Data sources to access state data, see ./state-storage
# Data sources to access terraform state, see ./state-storage

data "azurerm_storage_account" "tfstate" {
name = local.envs["storage_account_name"]
resource_group_name = local.envs["storage_account_rg_name"]
}

# Storage account and storage container used to store Jenkins artifacts
resource "azurerm_storage_account" "jenkins_artifacts" {
name = "art${local.ws}${local.shortloc}"
resource_group_name = azurerm_resource_group.infra.name
location = azurerm_resource_group.infra.location
account_tier = "Standard"
account_replication_type = "LRS"
allow_nested_items_to_be_public = false
}

resource "azurerm_storage_container" "jenkins_artifacts_1" {
name = "jenkins-artifacts-v1"
storage_account_name = azurerm_storage_account.jenkins_artifacts.name
container_access_type = "private"
}

# Data sources to access 'persistent' data, see ./persistent
################################################################################

data "azurerm_storage_account" "binary_cache" {
name = "ghafbincache${local.persistent_data}${local.shortloc}"
resource_group_name = "ghaf-infra-persistent-${local.shortloc}"
}
data "azurerm_storage_container" "binary_cache_1" {
name = "binary-cache-v1"
storage_account_name = data.azurerm_storage_account.binary_cache.name
}
# Data sources to access 'persistent' data
# see ./persistent and ./persistent/resources

# Builder ssh key
data "azurerm_key_vault" "ssh_remote_build" {
name = "ssh-builder-${local.persistent_data}-${local.shortloc}"
resource_group_name = "ghaf-infra-persistent-${local.shortloc}"
name = "sshb-${local.builder_sshkey_id}"
resource_group_name = local.persistent_rg
provider = azurerm
}

Expand All @@ -293,9 +273,21 @@ data "azurerm_key_vault_secret" "ssh_remote_build_pub" {
provider = azurerm
}

# Binary cache storage
data "azurerm_storage_account" "binary_cache" {
name = "bches${local.persistent_id}"
resource_group_name = local.persistent_rg
}

data "azurerm_storage_container" "binary_cache_1" {
name = "binary-cache-v1"
storage_account_name = data.azurerm_storage_account.binary_cache.name
}

# Binary cache signing key
data "azurerm_key_vault" "binary_cache_signing_key" {
name = "bche-sigkey-${local.persistent_data}-${local.shortloc}"
resource_group_name = "ghaf-infra-persistent-${local.shortloc}"
name = "bchek-${local.persistent_id}"
resource_group_name = local.persistent_rg
provider = azurerm
}

Expand All @@ -314,14 +306,27 @@ data "azurerm_key_vault" "ghaf_devenv_ca" {
# Data sources to access 'workspace-specific persistent' data
# see: ./persistent/workspace-specific

# Caddy state disk: binary cache
data "azurerm_managed_disk" "binary_cache_caddy_state" {
name = "binary-cache-vm-caddy-state-${local.ws}"
resource_group_name = "ghaf-infra-persistent-${local.shortloc}"
resource_group_name = local.persistent_rg
}

# Caddy state disk: jenkins controller
data "azurerm_managed_disk" "jenkins_controller_caddy_state" {
name = "jenkins-controller-vm-caddy-state-${local.ws}"
resource_group_name = "ghaf-infra-persistent-${local.shortloc}"
resource_group_name = local.persistent_rg
}

# Jenkins artifacts storage
data "azurerm_storage_account" "jenkins_artifacts" {
name = "artifact${local.ws}"
resource_group_name = local.persistent_rg
}

data "azurerm_storage_container" "jenkins_artifacts_1" {
name = "jenkins-artifacts-v1"
storage_account_name = data.azurerm_storage_account.jenkins_artifacts.name
}

################################################################################
Loading