diff --git a/.github/workflows/test-ghaf-infra.yml b/.github/workflows/test-ghaf-infra.yml index 659562a5..4d115958 100644 --- a/.github/workflows/test-ghaf-infra.yml +++ b/.github/workflows/test-ghaf-infra.yml @@ -20,12 +20,3 @@ jobs: - uses: cachix/install-nix-action@v22 - name: Run ghaf-infra CI tests run: nix develop --command inv pre-push - - name: Run terraform fmt - run: | - TF_LOG="DEBUG" bash -c 'nix develop .#terraform --command terraform -chdir=terraform fmt' - TF_LOG="DEBUG" bash -c 'nix develop .#terraform --command terraform -chdir=terraform/azure-storage fmt' - # TODO: Enable the below check when 'terraform validate' passes: - # - name: Run terraform validate - # run: | - # TF_LOG="DEBUG" bash -c 'nix develop .#terraform --command terraform -chdir=terraform validate' - # TF_LOG="DEBUG" bash -c 'nix develop .#terraform --command terraform -chdir=terraform/azure-storage validate' diff --git a/shell.nix b/shell.nix index df5228f6..2fe1c976 100644 --- a/shell.nix +++ b/shell.nix @@ -26,8 +26,10 @@ pkgs.mkShell { python3.pkgs.invoke python3.pkgs.pycodestyle python3.pkgs.pylint + python3.pkgs.tabulate + reuse sops ssh-to-age - reuse + terraform ]; } diff --git a/tasks.py b/tasks.py index 952cd7be..688062c6 100644 --- a/tasks.py +++ b/tasks.py @@ -41,11 +41,15 @@ from pathlib import Path from tempfile import TemporaryDirectory from typing import Any, Union +from collections import OrderedDict +from dataclasses import dataclass +from tabulate import tabulate from colorlog import ColoredFormatter, default_log_colors -from deploykit import DeployHost, DeployGroup, HostKeyCheck +from deploykit import DeployHost, HostKeyCheck from invoke import task + ################################################################################ ROOT = Path(__file__).parent.resolve() @@ -55,6 +59,37 @@ ################################################################################ +@dataclass(eq=False) +class TargetHost: + """Represents target host""" + + hostname: str + nixosconfig: str + + +# Below dictionary defines the set of ghaf-infra hosts: +# - Name (e.g. 'build01-dev) defines the aliasname for each target. +# - TargetHost.hostname: host name or IP address of the target. +# - TargetHost.nixosconfig: name of the nixosConfiguration installed/deployed +# on the given host. +TARGETS = OrderedDict( + { + "build01-dev": TargetHost(hostname="51.12.57.124", nixosconfig="build01"), + "ghafhydra-dev": TargetHost(hostname="51.12.56.79", nixosconfig="ghafhydra"), + } +) + + +def _get_target(alias: str) -> TargetHost: + if alias not in TARGETS: + LOG.fatal("Unknown alias '%s'", alias) + sys.exit(1) + return TARGETS[alias] + + +################################################################################ + + def set_log_verbosity(verbosity: int = 1) -> None: """Set logging verbosity (0=NOTSET, 1=INFO, or 2=DEBUG)""" log_levels = [logging.NOTSET, logging.INFO, logging.DEBUG] @@ -96,11 +131,17 @@ def _init_logging(verbosity: int = 1) -> None: set_log_verbosity(1) -def exec_cmd(cmd, raise_on_error=True): +def exec_cmd(cmd, raise_on_error=True, capture_output=True): """Run shell command cmd""" - LOG.debug("Running: %s", cmd) + LOG.info("Running: %s", cmd) try: - return subprocess.run(cmd.split(), capture_output=True, text=True, check=True) + if capture_output: + return subprocess.run( + cmd.split(), capture_output=True, text=True, check=True + ) + return subprocess.run( + cmd.split(), text=True, check=True, stdout=subprocess.PIPE + ) except subprocess.CalledProcessError as error: warn = [f"'{cmd}':"] if error.stdout: @@ -116,6 +157,23 @@ def exec_cmd(cmd, raise_on_error=True): ################################################################################ +@task +def alias_list(_c: Any) -> None: + """ + List available targets (i.e. configurations and alias names) + + Example usage: + inv list-name + """ + table_rows = [] + table_rows.append(["alias", "nixosconfig", "hostname"]) + for alias, host in TARGETS.items(): + row = [alias, host.nixosconfig, host.hostname] + table_rows.append(row) + table = tabulate(table_rows, headers="firstrow", tablefmt="fancy_outline") + print(f"\nCurrent ghaf-infra targets:\n\n{table}") + + @task def update_sops_files(c: Any) -> None: """ @@ -135,15 +193,16 @@ def update_sops_files(c: Any) -> None: @task -def print_keys(_c: Any, target: str) -> None: +def print_keys(_c: Any, alias: str) -> None: """ - Decrypt host private key, print ssh and age public keys for `target`. + Decrypt host private key, print ssh and age public keys for `alias` config. Example usage: - inv print-keys --target ghafhydra + inv print-keys --target ghafhydra-dev """ with TemporaryDirectory() as tmpdir: - decrypt_host_key(target, tmpdir) + nixosconfig = _get_target(alias).nixosconfig + decrypt_host_key(nixosconfig, tmpdir) key = f"{tmpdir}/etc/ssh/ssh_host_ed25519_key" pubkey = subprocess.run( ["ssh-keygen", "-y", "-f", f"{key}"], @@ -162,13 +221,13 @@ def print_keys(_c: Any, target: str) -> None: ) -def get_deploy_host(target: str = "", hostname: str = "") -> DeployHost: +def get_deploy_host(alias: str = "") -> DeployHost: """ - Return DeployHost object, given `hostname` and `target` + Return DeployHost object, given `alias` """ + hostname = _get_target(alias).hostname deploy_host = DeployHost( host=hostname, - meta={"target": target}, host_key_check=HostKeyCheck.NONE, # verbose_ssh=True, ) @@ -176,14 +235,14 @@ def get_deploy_host(target: str = "", hostname: str = "") -> DeployHost: @task -def deploy(_c: Any, target: str, hostname: str) -> None: +def deploy(_c: Any, alias: str) -> None: """ - Deploy NixOS configuration `target` to host `hostname`. + Deploy the configuration for `alias`. Example usage: - inv deploy --target ghafhydra --hostname 192.168.1.107 + inv deploy --alias ghafhydra-dev """ - h = get_deploy_host(target, hostname) + h = get_deploy_host(alias) command = "sudo nixos-rebuild" res = h.run_local( ["nix", "flake", "archive", "--to", f"ssh://{h.host}", "--json"], @@ -193,12 +252,13 @@ def deploy(_c: Any, target: str, hostname: str) -> None: path = data["path"] LOG.debug("data['path']: %s", path) flags = "--option accept-flake-config true" - h.run(f"{command} switch {flags} --flake {path}#{h.meta['target']}") + nixosconfig = _get_target(alias).nixosconfig + h.run(f"{command} switch {flags} --flake {path}#{nixosconfig}") -def decrypt_host_key(target: str, tmpdir: str) -> None: +def decrypt_host_key(nixosconfig: str, tmpdir: str) -> None: """ - Run sops to extract `target` secret 'ssh_host_ed25519_key' + Run sops to extract `nixosconfig` secret 'ssh_host_ed25519_key' """ def opener(path: str, flags: int) -> Union[str, int]: @@ -217,13 +277,15 @@ def opener(path: str, flags: int) -> Union[str, int]: "--extract", '["ssh_host_ed25519_key"]', "--decrypt", - f"{ROOT}/hosts/{target}/secrets.yaml", + f"{ROOT}/hosts/{nixosconfig}/secrets.yaml", ], check=True, stdout=fh, ) except subprocess.CalledProcessError: - LOG.warning("Failed reading secret 'ssh_host_ed25519_key' for '%s'", target) + LOG.warning( + "Failed reading secret 'ssh_host_ed25519_key' for '%s'", nixosconfig + ) ask = input("Still continue? [y/N] ") if ask != "y": sys.exit(1) @@ -240,27 +302,28 @@ def opener(path: str, flags: int) -> Union[str, int]: @task -def install(c: Any, target: str, hostname: str) -> None: +def install(c: Any, alias) -> None: """ - Install `target` on `hostname` using nixos-anywhere, deploying host private key. - Note: this will automatically partition and re-format `hostname` hard drive, + Install `alias` configuration using nixos-anywhere, deploying host private key. + Note: this will automatically partition and re-format the target hard drive, meaning all data on the target will be completely overwritten with no option to rollback. Example usage: - inv install --target ghafscan --hostname 192.168.1.109 + inv install --alias ghafscan-dev """ - ask = input(f"Install configuration '{target}' on host '{hostname}'? [y/N] ") + h = get_deploy_host(alias) + + ask = input(f"Install configuration '{alias}'? [y/N] ") if ask != "y": return - h = get_deploy_host(target, hostname) # Check sudo nopasswd try: h.run("sudo -nv", become_root=True) except subprocess.CalledProcessError: LOG.warning( - "sudo on '%s' needs password: installation will likely fail", hostname + "sudo on '%s' needs password: installation will likely fail", h.host ) ask = input("Still continue? [y/N] ") if ask != "y": @@ -271,7 +334,7 @@ def install(c: Any, target: str, hostname: str) -> None: except subprocess.CalledProcessError: pass else: - LOG.warning("Above address(es) on '%s' use dynamic addressing.", hostname) + LOG.warning("Above address(es) on '%s' use dynamic addressing.", h.host) LOG.warning( "This might cause issues if you assume the target host is reachable " "from any such address also after kexec switch. " @@ -282,55 +345,40 @@ def install(c: Any, target: str, hostname: str) -> None: if ask != "y": sys.exit(1) + nixosconfig = _get_target(alias).nixosconfig with TemporaryDirectory() as tmpdir: - decrypt_host_key(target, tmpdir) + decrypt_host_key(nixosconfig, tmpdir) command = "nix run github:numtide/nixos-anywhere --" - command += f" {hostname} --extra-files {tmpdir} --flake .#{target}" + command += f" {h.host} --extra-files {tmpdir} --flake .#{nixosconfig}" command += " --option accept-flake-config true" + LOG.warning(command) c.run(command) + # Reboot - print(f"Wait for {hostname} to start", end="") - wait_for_port(hostname, 22) - reboot(c, hostname) + print(f"Wait for {h.host} to start", end="") + wait_for_port(h.host, 22) + reboot(c, alias) @task -def build_local(_c: Any, target: str = "") -> None: +def build_local(_c: Any, alias: str = "") -> None: """ - Build NixOS configuration `target` locally. - If `target` is not specificied, builds all nixosConfigurations in the flake. + Build NixOS configuration `alias` locally. + If `alias` is not specificied, builds all TARGETS. Example usage: - inv build-local --target ghafhydra + inv build-local --alias ghafhydra-dev """ - if target: - # For local builds, we pretend hostname is the target - g = DeployGroup([get_deploy_host(hostname=target)]) + if alias: + target_configs = [_get_target(alias).nixosconfig] else: - res = subprocess.run( - ["nix", "flake", "show", "--json"], - check=True, - text=True, - stdout=subprocess.PIPE, - ) - data = json.loads(res.stdout) - targets = data["nixosConfigurations"] - g = DeployGroup([get_deploy_host(hostname=t) for t in targets]) - - def _build_local(h: DeployHost) -> None: - h.run_local( - [ - "nixos-rebuild", - "build", - "--option", - "accept-flake-config", - "true", - "--flake", - f".#{h.host}", - ] + target_configs = [target.nixosconfig for _, target in TARGETS.items()] + for nixosconfig in target_configs: + cmd = ( + "nixos-rebuild build --option accept-flake-config true " + f" -v --flake .#{nixosconfig}" ) - - g.run_function(_build_local) + exec_cmd(cmd, capture_output=False) def wait_for_port(host: str, port: int, shutdown: bool = False) -> None: @@ -351,14 +399,14 @@ def wait_for_port(host: str, port: int, shutdown: bool = False) -> None: @task -def reboot(_c: Any, hostname: str) -> None: +def reboot(_c: Any, alias: str) -> None: """ - Reboot host `hostname`. + Reboot host identified as `alias`. Example usage: - inv reboot --hostname 192.168.1.112 + inv reboot --alias ghafhydra-dev """ - h = get_deploy_host(hostname=hostname) + h = get_deploy_host(alias) h.run("sudo reboot &") print(f"Wait for {h.host} to shutdown", end="") @@ -383,36 +431,34 @@ def pre_push(c: Any) -> None: cmd = "find . -type f -name *.py ! -path *result* ! -path *eggs*" ret = exec_cmd(cmd) pyfiles = ret.stdout.replace("\n", " ") - LOG.info("Running black") cmd = f"black -q {pyfiles}" ret = exec_cmd(cmd, raise_on_error=False) if not ret: sys.exit(1) - LOG.info("Running pylint") cmd = f"pylint --disable duplicate-code -rn {pyfiles}" ret = exec_cmd(cmd, raise_on_error=False) if not ret: sys.exit(1) - LOG.info("Running pycodestyle") cmd = f"pycodestyle --max-line-length=90 {pyfiles}" ret = exec_cmd(cmd, raise_on_error=False) if not ret: sys.exit(1) - LOG.info("Running reuse lint") cmd = "reuse lint" ret = exec_cmd(cmd, raise_on_error=False) if not ret: sys.exit(1) - LOG.info("Running nix fmt") + cmd = "terraform fmt -check -recursive" + ret = exec_cmd(cmd, raise_on_error=False) + if not ret: + LOG.warning("Run `terraform fmt -recursive` locally to fix formatting") + sys.exit(1) cmd = "nix fmt" ret = exec_cmd(cmd, raise_on_error=False) if not ret: sys.exit(1) - LOG.info("Running nix flake check") - cmd = "nix flake check" + cmd = "nix flake check -vv" ret = exec_cmd(cmd, raise_on_error=False) if not ret: sys.exit(1) - LOG.info("Building all nixosConfigurations") build_local(c) LOG.info("All pre-push checks passed") diff --git a/terraform/azure-ghaf-infra.tf b/terraform/azure-ghaf-infra.tf index 0724b253..580dab5e 100644 --- a/terraform/azure-ghaf-infra.tf +++ b/terraform/azure-ghaf-infra.tf @@ -1,36 +1,57 @@ # SPDX-FileCopyrightText: 2023 Technology Innovation Institute (TII) # # SPDX-License-Identifier: Apache-2.0 - # Resource group -resource "azurerm_resource_group" "rg" { - name = "ghaf-infra-terraform-dev" +resource "azurerm_resource_group" "ghaf_infra_tf_dev" { + name = "ghaf-infra-tf-dev" location = var.resource_group_location } -# Create VN -resource "azurerm_virtual_network" "ghaf-infra-vnet" { - name = "ghaf-infra-terraform-dev-vnet" - address_space = ["10.3.0.0/24"] - location = azurerm_resource_group.rg.location - resource_group_name = azurerm_resource_group.rg.name -} - - -# Create public IPs -resource "azurerm_public_ip" "ghafhydra_terraform_public_ip" { - name = "ghaf-infra-terraform-dev-ip" - location = azurerm_resource_group.rg.location - resource_group_name = azurerm_resource_group.rg.name +# Create VN +resource "azurerm_virtual_network" "ghaf_infra_tf_vnet" { + name = "ghaf-infra-tf-vnet" + address_space = ["10.0.0.0/16"] + location = var.resource_group_location + resource_group_name = azurerm_resource_group.ghaf_infra_tf_dev.name +} +# Create Subnet +resource "azurerm_subnet" "ghaf_infra_tf_subnet" { + name = "ghaf-infra-tf-subnet" + resource_group_name = azurerm_resource_group.ghaf_infra_tf_dev.name + virtual_network_name = azurerm_virtual_network.ghaf_infra_tf_vnet.name + address_prefixes = ["10.0.2.0/24"] +} +# Network interface +resource "azurerm_network_interface" "ghaf_infra_tf_network_interface" { + name = "ghaf-infratf286-z1" + location = var.resource_group_location + resource_group_name = azurerm_resource_group.ghaf_infra_tf_dev.name + ip_configuration { + name = "my_nic_configuration" + subnet_id = azurerm_subnet.ghaf_infra_tf_subnet.id + private_ip_address_allocation = "Dynamic" + public_ip_address_id = azurerm_public_ip.ghaf_infra_tf_public_ip.id + } +} +# Create Availability Set +resource "azurerm_availability_set" "ghaf_infra_tf_availability_set" { + name = "ghaf-infra-tf-availability-set" + location = var.resource_group_location + resource_group_name = azurerm_resource_group.ghaf_infra_tf_dev.name + platform_fault_domain_count = 2 + platform_update_domain_count = 2 +} +# Create Public IPs +resource "azurerm_public_ip" "ghaf_infra_tf_public_ip" { + name = "ghaf-infra-tf-public-ip" + location = var.resource_group_location + resource_group_name = azurerm_resource_group.ghaf_infra_tf_dev.name allocation_method = "Dynamic" } - - -# Create Network SG and rule -resource "azurerm_network_security_group" "ghafhydra_terraform_nsg" { - name = "ghaf-infra-terraform-dev-nsg" - location = azurerm_resource_group.rg.location - resource_group_name = azurerm_resource_group.rg.name - +# Create Network Security Group and rule +resource "azurerm_network_security_group" "ghaf_infra_tf_nsg" { + name = "ghaf-infra-tf-nsg" + location = var.resource_group_location + resource_group_name = azurerm_resource_group.ghaf_infra_tf_dev.name security_rule { name = "SSH" priority = 300 @@ -43,4 +64,62 @@ resource "azurerm_network_security_group" "ghafhydra_terraform_nsg" { destination_address_prefix = "*" } } - +# Create Storage Account +resource "azurerm_storage_account" "ghafinfra_tf_storage_account" { + name = "ghafinfrastorage" + location = var.resource_group_location + resource_group_name = azurerm_resource_group.ghaf_infra_tf_dev.name + account_tier = "Standard" + account_replication_type = "LRS" +} +# Create Linux Virtual Machine +resource "azurerm_linux_virtual_machine" "ghafinfra_tf" { + name = "ghafinfratf" + location = var.resource_group_location + resource_group_name = azurerm_resource_group.ghaf_infra_tf_dev.name + availability_set_id = azurerm_availability_set.ghaf_infra_tf_availability_set.id + network_interface_ids = [ + azurerm_network_interface.ghaf_infra_tf_network_interface.id + ] + size = "Standard_B8ms" + os_disk { + name = "ghafinfratfdisk1" + caching = "ReadWrite" + storage_account_type = "Premium_LRS" + disk_size_gb = 512 + } + source_image_reference { + publisher = "canonical" + offer = "0001-com-ubuntu-server-jammy" + sku = "22_04-lts-gen2" + version = "latest" + } + admin_username = "karim" + disable_password_authentication = true + admin_ssh_key { + username = "karim" + public_key = file("~/.ssh/id_rsa_nixos.pub") + } +} +# Create Custom Script Extension +resource "azurerm_virtual_machine_extension" "customScript" { + name = "customScript" + virtual_machine_id = azurerm_linux_virtual_machine.ghafinfra_tf.id + publisher = "Microsoft.Azure.Extensions" + type = "CustomScript" + type_handler_version = "2.1" + settings = jsonencode({ + commandToExecute = <<-SCRIPT + #!/bin/bash + sudo apt-get update + sudo apt-get install -y apache2 + mkdir -p /home/karim/.ssh + echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDe5L8iOqhNPsYz5eh9Bz/URYguG60JjMGmKG0wwLIb6Gf2M8Txzk24ESGbMR/F5RYsV1yWYOocL47ngDWQIbO6MGJ7ftUr7slWoUA/FSVwh/jsG681mRqIuJXjKM/YQhBkI9k6+eVxRfLDTs5XZfbwdm7T4aP8ZI2609VY0guXfa/F7DSE1BxN7IJMn0CWLQJanBpoYUxqyQXCUXgljMokdPjTrqAxlBluMsVTP+ZKDnjnpHcVE/hCKk5BxaU6K97OdeIOOEWXAd6uEHssomjtU7+7dhiZzjhzRPKDiSJDF9qtIw50kTHz6ZTdH8SAZmu0hsS6q8OmmDTAnt24dFJV karim@nixos' >> /home/karim/.ssh/authorized_keys + chown -R karim:karim /home/karim/.ssh + chmod 700 /home/karim/.ssh + chmod 600 /home/karim/.ssh/authorized_keys + sed -i 's/\s*PasswordAuthentication\s\+yes/PasswordAuthentication no/' /etc/ssh/sshd_config + systemctl restart sshd + SCRIPT + }) +} \ No newline at end of file diff --git a/terraform/outputs.tf b/terraform/outputs.tf index 5689a6ba..0720668f 100644 --- a/terraform/outputs.tf +++ b/terraform/outputs.tf @@ -3,9 +3,9 @@ # SPDX-License-Identifier: Apache-2.0 output "resource_group_name" { - value = azurerm_resource_group.rg.name + value = azurerm_resource_group.ghaf_infra_tf_dev.name } output "resource_group_location" { - value = azurerm_resource_group.rg.location + value = var.resource_group_location } diff --git a/terraform/providers.tf b/terraform/providers.tf index 6e9e9206..8f976809 100644 --- a/terraform/providers.tf +++ b/terraform/providers.tf @@ -9,10 +9,10 @@ provider "azurerm" { terraform { required_providers { azurerm = { - source = "hashicorp/azurerm" + source = "hashicorp/azurerm" } sops = { - source = "carlpett/sops" + source = "carlpett/sops" } } } diff --git a/terraform/variables.tf b/terraform/variables.tf index f60249f2..0f408c7b 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -11,7 +11,7 @@ variable "resource_group_location" { variable "resourcegroup" { description = "The Azure Resource Group Name within your Subscription in which this resource will be created." - default = "ghaf-infra-swe" + default = "ghaf-infra-swe" } variable "resource_group_name_prefix" {