jenkins-controller: Initial Jenkins configuration

hosts/azure/jenkins-controller/configuration.nix

@@ -3,6 +3,7 @@
 # SPDX-License-Identifier: Apache-2.0
 {
   pkgs,
+  config,
   self,
   lib,
   ...
@@ -21,6 +22,28 @@
     exec nix --extra-experimental-features nix-command copy --to 'http://localhost:8080?secret-key=/etc/secrets/nix-signing-key&compression=zstd' $OUT_PATHS
   '';
 
+  # TODO: sort out jenkins authentication e.g.:
+  # https://plugins.jenkins.io/github-oauth/
+  # Below config requires admin to trigger builds or manage jenkins
+  # allowing read access for anonymous users:
+  jenkins-groovy = pkgs.writeText "groovy" ''
+    #!groovy
+
+    import jenkins.model.*
+    import jenkins.install.*
+    import hudson.security.*
+
+    def instance = Jenkins.getInstance()
+    // Disable Setup Wizard
+    instance.setInstallState(InstallState.INITIAL_SETUP_COMPLETED)
+
+    // Allow anonymous read access
+    def strategy = new FullControlOnceLoggedInAuthorizationStrategy()
+    strategy.setAllowAnonymousRead(true)
+    instance.setAuthorizationStrategy(strategy)
+    instance.save()
+  '';
+
   get-secret =
     pkgs.writers.writePython3 "get-secret" {
       libraries = with pkgs.python3.pkgs; [azure-keyvault-secrets azure-identity];
@@ -68,24 +91,105 @@ in {
     ];
   };
 
+  # Configure /var/lib/caddy in /etc/fstab.
+  # Due to an implicit RequiresMountsFor=$state-dir, systemd
+  # will block starting the service until this mounted.
+  fileSystems."/var/lib/caddy" = {
+    device = "/dev/disk/by-lun/11";
+    fsType = "ext4";
+    options = [
+      "x-systemd.makefs"
+      "x-systemd.growfs"
+    ];
+  };
+
   services.jenkins = {
     enable = true;
     listenAddress = "localhost";
     port = 8081;
     withCLI = true;
+    packages = with pkgs; [
+      bashInteractive # 'sh' step in jenkins pipeline requires this
+      coreutils
+      nix
+      git
+      zstd
+    ];
+    extraJavaOptions = [
+      # Useful when the 'sh' step fails:
+      "-Dorg.jenkinsci.plugins.durabletask.BourneShellScript.LAUNCH_DIAGNOSTICS=true"
+    ];
+    # Configure jenkins job(s):
+    # https://jenkins-job-builder.readthedocs.io/en/latest/project_pipeline.html
+    # https://github.com/NixOS/nixpkgs/blob/master/nixos/modules/services/continuous-integration/jenkins/job-builder.nix
+    jobBuilder = {
+      enable = true;
+      nixJobs = [
+        {
+          job = {
+            name = "ghaf-pipeline";
+            project-type = "pipeline";
+            pipeline-scm = {
+              scm = [
+                {
+                  git = {
+                    # TODO: eventually the Jenkins pipeline script should probably
+                    # be part of Ghaf repo at: https://github.com/tiiuae/ghaf,
+                    # but we are not ready for that yet. For now, we read the
+                    # Jenkinsfile from the following repo:
+                    url = "https://github.com/tiiuae/ghaf-jenkins-pipeline.git";
+                    clean = true;
+                    branches = ["*/main"];
+                  };
+                }
+              ];
+              script-path = "ghaf-build-pipeline.groovy";
+              lightweight-checkout = true;
+            };
+          };
+        }
+      ];
+    };
   };
-  systemd.services.jenkins.after = ["multi-user.target"];
-  systemd.services.jenkins.requires = ["multi-user.target"];
-  systemd.services.jenkins.serviceConfig = {
+  systemd.services.jenkins.serviceConfig = {Restart = "on-failure";};
+  systemd.services.jenkins-job-builder.serviceConfig = {
     Restart = "on-failure";
-    RestartSec = 1;
+    RestartSec = 5;
   };
 
   # set StateDirectory=jenkins, so state volume has the right permissions
   # and we wait on the mountpoint to appear.
   # https://github.com/NixOS/nixpkgs/pull/272679
   systemd.services.jenkins.serviceConfig.StateDirectory = "jenkins";
 
+  # Install jenkins plugins, apply initial jenkins config
+  systemd.services.jenkins-config = {
+    after = ["jenkins-job-builder.service"];
+    wantedBy = ["multi-user.target"];
+    # Make `jenkins-cli` available
+    path = with pkgs; [jenkins];
+    # Implicit URL parameter for `jenkins-cli`
+    environment = {
+      JENKINS_URL = "http://localhost:8081";
+    };
+    serviceConfig = {
+      Restart = "on-failure";
+      RestartSec = 5;
+    };
+    script = let
+      jenkins-auth = "-auth admin:\"$(cat /var/lib/jenkins/secrets/initialAdminPassword)\"";
+    in ''
+      # Install plugins
+      jenkins-cli ${jenkins-auth} install-plugin "workflow-aggregator" "github" -deploy
+
+      # Jenkins groovy config
+      jenkins-cli ${jenkins-auth} groovy = < ${jenkins-groovy}
+
+      # Restart jenkins
+      jenkins-cli ${jenkins-auth} safe-restart
+    '';
+  };
+
   # Define a fetch-remote-build-ssh-key unit populating
   # /etc/secrets/remote-build-ssh-key from Azure Key Vault.
   # Make it before and requiredBy nix-daemon.service.
@@ -203,7 +307,37 @@ in {
     post-build-hook = ${post-build-hook}
   '';
 
-  # TODO: deploy reverse proxy, sort out authentication (SSO?)
+  # TODO: use https://caddyserver.com/docs/caddyfile-tutorial#environment-variables for domain
+  services.caddy = {
+    enable = true;
+    configFile = pkgs.writeTextDir "Caddyfile" ''
+      # Disable the admin API, we don't want to reconfigure Caddy at runtime.
+      {
+        admin off
+      }
+
+      # Proxy all requests to jenkins.
+      https://{$SITE_ADDRESS} {
+        reverse_proxy localhost:8081
+      }
+    '';
+  };
+
+  # workaround for https://github.com/NixOS/nixpkgs/issues/272532
+  # FUTUREWORK: rebase once https://github.com/NixOS/nixpkgs/pull/272617 landed
+  services.caddy.enableReload = false;
+  systemd.services.caddy.serviceConfig.ExecStart = lib.mkForce [
+    ""
+    "${pkgs.caddy}/bin/caddy run --environ --config ${config.services.caddy.configFile}/Caddyfile"
+  ];
+  systemd.services.caddy.serviceConfig.EnvironmentFile = "/run/caddy.env";
+
+  # Wait for cloud-init mounting before we start caddy.
+  systemd.services.caddy.after = ["cloud-init.service"];
+  systemd.services.caddy.requires = ["cloud-init.service"];
+
+  # Expose the HTTPS port. No need for HTTP, as caddy can use TLS-ALPN-01.
+  networking.firewall.allowedTCPPorts = [443];
 
   nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
 

terraform/jenkins-controller.tf

@@ -62,6 +62,10 @@ module "jenkins_controller_vm" {
       {
         content = join("\n", toset(module.builder_vm[*].virtual_machine_private_ip_address))
         "path"  = "/var/lib/builder-keyscan/scanlist"
+      },
+      {
+        content = "SITE_ADDRESS=ghaf-jenkins-controller-${local.env}.northeurope.cloudapp.azure.com",
+        "path"  = "/run/caddy.env"
       }
     ]
   })])
@@ -70,14 +74,24 @@ module "jenkins_controller_vm" {
   subnet_id          = azurerm_subnet.jenkins.id
 
   # Attach disk to the VM
-  data_disks = [{
-    name            = azurerm_managed_disk.jenkins_controller_jenkins_state.name
-    managed_disk_id = azurerm_managed_disk.jenkins_controller_jenkins_state.id
-    lun             = "10"
-    # create_option = "Attach"
-    caching      = "None"
-    disk_size_gb = azurerm_managed_disk.jenkins_controller_jenkins_state.disk_size_gb
-  }]
+  data_disks = [
+    {
+      name            = azurerm_managed_disk.jenkins_controller_jenkins_state.name
+      managed_disk_id = azurerm_managed_disk.jenkins_controller_jenkins_state.id
+      lun             = "10"
+      # create_option = "Attach"
+      caching      = "None"
+      disk_size_gb = azurerm_managed_disk.jenkins_controller_jenkins_state.disk_size_gb
+    },
+    {
+      name            = data.azurerm_managed_disk.jenkins_controller_caddy_state.name
+      managed_disk_id = data.azurerm_managed_disk.jenkins_controller_caddy_state.id
+      lun             = "11"
+      create_option   = "Attach"
+      caching         = "None"
+      disk_size_gb    = data.azurerm_managed_disk.jenkins_controller_caddy_state.disk_size_gb
+    }
+  ]
 }
 
 resource "azurerm_network_interface_security_group_association" "jenkins_controller_vm" {
@@ -91,13 +105,13 @@ resource "azurerm_network_security_group" "jenkins_controller_vm" {
   location            = azurerm_resource_group.infra.location
 
   security_rule {
-    name                       = "AllowSSHInbound"
+    name                       = "AllowSSHHTTPSInbound"
     priority                   = 400
     direction                  = "Inbound"
     access                     = "Allow"
     protocol                   = "Tcp"
     source_port_range          = "*"
-    destination_port_ranges    = [22]
+    destination_port_ranges    = [22, 443]
     source_address_prefix      = "*"
     destination_address_prefix = "*"
   }

terraform/main.tf

@@ -42,7 +42,7 @@ variable "location" {
 }
 
 # Use azure_region module to get the short name of the Azure region,
-# see: https://registry.terraform.io/modules/claranet/regions/azurerm/latest 
+# see: https://registry.terraform.io/modules/claranet/regions/azurerm/latest
 module "azure_region" {
   source       = "claranet/regions/azurerm"
   azure_region = var.location
@@ -204,10 +204,17 @@ data "azurerm_key_vault_secret" "binary_cache_signing_key" {
   provider     = azurerm
 }
 
+# Data sources to access 'workspace-specific persistent' data
+# see: ./persistent/workspace-specific
+
 data "azurerm_managed_disk" "binary_cache_caddy_state" {
   name                = "binary-cache-vm-caddy-state-${local.ws}"
   resource_group_name = "ghaf-infra-persistent-${local.shortloc}"
 }
 
+data "azurerm_managed_disk" "jenkins_controller_caddy_state" {
+  name                = "jenkins-controller-vm-caddy-state-${local.ws}"
+  resource_group_name = "ghaf-infra-persistent-${local.shortloc}"
+}
 
 ################################################################################

terraform/persistent/workspace-specific/main.tf

@@ -73,4 +73,13 @@ resource "azurerm_managed_disk" "binary_cache_caddy_state" {
   disk_size_gb         = 1
 }
 
+resource "azurerm_managed_disk" "jenkins_controller_caddy_state" {
+  name                 = "jenkins-controller-vm-caddy-state-${local.ws}"
+  resource_group_name  = data.azurerm_resource_group.persistent.name
+  location             = data.azurerm_resource_group.persistent.location
+  storage_account_type = "Standard_LRS"
+  create_option        = "Empty"
+  disk_size_gb         = 1
+}
+
 ################################################################################