Skip to content

Commit

Permalink
Start using terraform infra with nixos configs
Browse files Browse the repository at this point in the history 
Prepare ghaf-infra so that the infra created with terraform can be
used with flake nixos configurations. This includes the following
changes:
- Remove testhost terraform configuration since it's not used in
  ghaf-infra.
- Use static internal IP addresses in terraform VM configurations.
- Configure public cloudapp.azure.com domain names for terraform VMs
  by using the 'domain_name_label' configuration option.
- Use the cloudapp.azure.com domain names in task.py alias hostnames
  where appropriate.
- Use the azarm VM as a remote aarch64 builder for the ghafhydra (in
  addition to awsarm).

In addition, this commit changes some common garbage collection
options.

Signed-off-by: Henri Rosten <henri.rosten@unikie.com>
  • Loading branch information
@henrirosten
henrirosten committed Nov 15, 2023
1 parent ff38a56 commit dc58805
Show file tree
Hide file tree
Showing 6 changed files with 49 additions and 40 deletions.
6 changes: 3 additions & 3 deletions hosts/common.nix
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,9 +44,9 @@ in {
# perform a garbage-collection until max-free bytes are available or there
# is no more garbage.
min-free = asGB 20;
max-free = asGB 100;
# check the free disk space every 10 seconds
min-free-check-interval = 10;
max-free = asGB 200;
# check the free disk space every 5 seconds
min-free-check-interval = 5;
};
# Garbage collection
gc.automatic = true;
Expand Down
9 changes: 5 additions & 4 deletions hosts/default.nix
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,10 +23,11 @@
# make self and inputs available in nixos modules
specialArgs = {inherit self inputs;};
in {
build01 = lib.nixosSystem {
inherit specialArgs;
modules = [self.nixosModules.host-build01];
};
# Currently not used for anything:
# build01 = lib.nixosSystem {
# inherit specialArgs;
# modules = [self.nixosModules.host-build01];
# };
ghafhydra = lib.nixosSystem {
inherit specialArgs;
modules = [self.nixosModules.host-ghafhydra];
Expand Down
6 changes: 3 additions & 3 deletions hosts/ghafhydra/default.nix
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@
security.acme.acceptTerms = true;
services.nginx = {
virtualHosts = {
"ghafhydra.swedencentral.cloudapp.azure.com" = {
"ghafhydra.northeurope.cloudapp.azure.com" = {
forceSSL = true;
enableACME = true;
locations."/".proxyPass = "http://localhost:${toString config.services.hydra.port}";
Expand All @@ -58,10 +58,10 @@
# TODO: demo with static IP:
networking.useDHCP = false;
networking.nameservers = ["1.1.1.1" "8.8.8.8"];
networking.defaultGateway = "10.3.0.1";
networking.defaultGateway = "10.0.2.1";
networking.interfaces.eth0.ipv4.addresses = [
{
address = "10.3.0.4";
address = "10.0.2.4";
prefixLength = 24;
}
];
Expand Down
20 changes: 9 additions & 11 deletions services/hydra/default.nix
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,17 +23,16 @@
localhost x86_64-linux - 8 2 kvm,benchmark,big-parallel,nixos-test - -
'';
};
build01Machine = pkgs.writeTextFile {
name = "build-build01Machine";
# TODO: get rid of static IP config:
azarmMachine = pkgs.writeTextFile {
name = "build-azarmMachine";
text = ''
ssh://nix@10.3.0.5 x86_64-linux ${config.sops.secrets.id_buildfarm.path} 8 2 kvm,benchmark,big-parallel,nixos-test - -
ssh://nix@10.0.2.10 aarch64-linux ${config.sops.secrets.id_buildfarm.path} 8 2 kvm,benchmark,big-parallel,nixos-test - -
'';
};
awsarmMachine = pkgs.writeTextFile {
name = "build-awsarmMachine";
text = ''
ssh://nix@awsarm.vedenemo.dev aarch64-linux ${config.sops.secrets.id_buildfarm.path} 8 2 kvm,benchmark,big-parallel,nixos-test - -
ssh://nix@awsarm.vedenemo.dev aarch64-linux ${config.sops.secrets.id_buildfarm.path} 16 4 kvm,benchmark,big-parallel,nixos-test - -
'';
};
createJobsetsScript = pkgs.stdenv.mkDerivation {
Expand All @@ -49,12 +48,11 @@
in {
programs.ssh.knownHosts = {
# Add builder machines' public ids to ssh known_hosts
build01 = {
# TODO: get rid of static IP config:
hostNames = ["10.3.0.5"];
publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAID+hx/Ff8U123lI8wMYvmVYn5M3Cv4m+XQxxNYFgJGTo";
azarm = {
hostNames = ["10.0.2.10"];
publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILWojItdbPn0dNbGKfCSJv+duYsb+xzJ6hPWOu+TZ4rm";
};
armbuild01 = {
awsarm = {
hostNames = ["awsarm.vedenemo.dev"];
publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIL3f7tAAO3Fc+8BqemsBQc/Yl/NmRfyhzr5SFOSKqrv0";
};
Expand All @@ -73,7 +71,7 @@ in {

buildMachinesFiles = [
"${localMachine}"
"${build01Machine}"
"${azarmMachine}"
"${awsarmMachine}"
];

Expand Down
9 changes: 5 additions & 4 deletions tasks.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,13 +68,14 @@ class TargetHost:


# Below dictionary defines the set of ghaf-infra configuration aliases:
# - Name (e.g. 'build01-dev) defines the alias name for each target.
# - Name (e.g. 'ghafhydra-dev) defines the alias name for each target.
# - TargetHost.hostname: host name or IP address of the target.
# - TargetHost.nixosconfig: name of the target nixosConfiguration.
TARGETS = OrderedDict(
{
"build01-dev": TargetHost(hostname="51.12.57.124", nixosconfig="build01"),
"ghafhydra-dev": TargetHost(hostname="51.12.56.79", nixosconfig="ghafhydra"),
"ghafhydra-dev": TargetHost(
hostname="ghafhydra.northeurope.cloudapp.azure.com", nixosconfig="ghafhydra"
),
"binarycache-ficolo": TargetHost(
hostname="172.18.20.109", nixosconfig="binarycache"
),
Expand Down Expand Up @@ -245,7 +246,6 @@ def deploy(_c: Any, alias: str) -> None:
inv deploy --alias ghafhydra-dev
"""
h = get_deploy_host(alias)
command = "sudo nixos-rebuild"
res = h.run_local(
["nix", "flake", "archive", "--to", f"ssh://{h.host}", "--json"],
stdout=subprocess.PIPE,
Expand All @@ -255,6 +255,7 @@ def deploy(_c: Any, alias: str) -> None:
LOG.debug("data['path']: %s", path)
flags = "--option accept-flake-config true"
nixosconfig = _get_target(alias).nixosconfig
command = "sudo nixos-rebuild"
h.run(f"{command} switch {flags} --flake {path}#{nixosconfig}")


Expand Down
39 changes: 24 additions & 15 deletions terraform/azure-ghaf-infra.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,38 +65,40 @@ resource "azurerm_network_security_group" "ghaf_infra_tf_nsg" {

################################################################################

# testhost
# ghafhydra:

# Public IP
resource "azurerm_public_ip" "testhost_public_ip" {
name = "testhost-public-ip"
resource "azurerm_public_ip" "ghafhydra_public_ip" {
name = "ghafhydra-public-ip"
domain_name_label = "ghafhydra"
location = azurerm_resource_group.ghaf_infra_tf_dev.location
resource_group_name = azurerm_resource_group.ghaf_infra_tf_dev.name
allocation_method = "Static"
}
# Network interface
resource "azurerm_network_interface" "testhost_ni" {
name = "testhost-nic"
resource "azurerm_network_interface" "ghafhydra_ni" {
name = "ghafhydra-nic"
location = azurerm_resource_group.ghaf_infra_tf_dev.location
resource_group_name = azurerm_resource_group.ghaf_infra_tf_dev.name
ip_configuration {
name = "testhost_nic_configuration"
name = "ghafhydra_nic_configuration"
subnet_id = azurerm_subnet.ghaf_infra_tf_subnet.id
private_ip_address_allocation = "Dynamic"
public_ip_address_id = azurerm_public_ip.testhost_public_ip.id
private_ip_address_allocation = "Static"
private_ip_address = "10.0.2.4"
public_ip_address_id = azurerm_public_ip.ghafhydra_public_ip.id
}
}
# Example Linux Virtual Machine (testhost)
resource "azurerm_linux_virtual_machine" "testhost_vm" {
name = "testhost"
# Ghafhydra VM
resource "azurerm_linux_virtual_machine" "ghafhydra_vm" {
name = "ghafhydra"
location = azurerm_resource_group.ghaf_infra_tf_dev.location
resource_group_name = azurerm_resource_group.ghaf_infra_tf_dev.name
network_interface_ids = [
azurerm_network_interface.testhost_ni.id
azurerm_network_interface.ghafhydra_ni.id
]
size = "Standard_B8ms"
os_disk {
name = "testhost-disk"
name = "ghafhydra-disk"
caching = "ReadWrite"
storage_account_type = "Premium_LRS"
disk_size_gb = 512
Expand All @@ -119,11 +121,17 @@ resource "azurerm_linux_virtual_machine" "testhost_vm" {

################################################################################

# azarm
# azarm:
# aarch64-linux builder - Ubuntu host with nix package manager.
# Why not NixOS? The reason is: we have not managed to get nixos-anywhere
# working with azure arm VMs.
# Since the host is not NixOS, all the host configuration is done on
# terraform apply using the configuration script at scripts/ubuntu-builder.sh

# Public IP
resource "azurerm_public_ip" "azarm_public_ip" {
name = "azarm-public-ip"
domain_name_label = "azarm"
location = azurerm_resource_group.ghaf_infra_tf_dev.location
resource_group_name = azurerm_resource_group.ghaf_infra_tf_dev.name
allocation_method = "Static"
Expand All @@ -136,7 +144,8 @@ resource "azurerm_network_interface" "azarm_ni" {
ip_configuration {
name = "azarm_nic_configuration"
subnet_id = azurerm_subnet.ghaf_infra_tf_subnet.id
private_ip_address_allocation = "Dynamic"
private_ip_address_allocation = "Static"
private_ip_address = "10.0.2.10"
public_ip_address_id = azurerm_public_ip.azarm_public_ip.id
}
}
Expand Down

0 comments on commit dc58805

Please sign in to comment.