Automatically fetch agent secrets from controller

Signed-off-by: Joonas Rautiola <joonas.rautiola@unikie.com>
tiiuae · Oct 22, 2024 · 5edb0c5 · 5edb0c5
1 parent 1bc76f8
commit 5edb0c5
Showing 1 changed file with 111 additions and 82 deletions.
diff --git a/hosts/testagent-dev/configuration.nix b/hosts/testagent-dev/configuration.nix
@@ -11,80 +11,6 @@ let
   # Vendored in, as brainstem isn't suitable for nixpkgs packaging upstream:
   # https://github.com/NixOS/nixpkgs/pull/313643
   brainstem = pkgs.callPackage ../../pkgs/brainstem { };
-
-  mkAgent =
-    device:
-    let
-      # Temporary url for development
-      controllerUrl = "https://ghaf-jenkins-controller-villepekkajuntun.northeurope.cloudapp.azure.com";
-      workDir = "/var/lib/jenkins/agents/${device}";
-
-      jenkins-connection-script =
-        pkgs.writeScript "jenkins-connect.sh" # sh
-          ''
-            #!/usr/bin/env bash
-            set -eu
-
-            mkdir -p "${workDir}"
-
-            # get agent.jar
-            if [ ! -f agent.jar ]; then
-              wget "${controllerUrl}/jnlpJars/agent.jar"
-            fi
-
-            # TODO: get secret from jenkins api here, right now it's manual process
-            if [ ! -f "secret_${device}" ]; then echo "Error: /var/lib/jenkins/secret_${device} not found"; exit 1; fi;
-
-            ${pkgs.jdk}/bin/java \
-              -jar agent.jar \
-              -url "${controllerUrl}" \
-              -name "${device}" \
-              -secret "@secret_${device}" \
-              -workDir "${workDir}" \
-              -webSocket
-          '';
-    in
-    {
-      after = [ "network.target" ];
-      wantedBy = [ "multi-user.target" ];
-
-      # Give up if it fails more than 5 times in 60 second interval
-      startLimitBurst = 5;
-      startLimitIntervalSec = 60;
-
-      path =
-        [
-          brainstem
-          inputs.robot-framework.packages.${pkgs.system}.ghaf-robot
-        ]
-        ++ (with pkgs; [
-          wget
-          jdk
-          git
-          bashInteractive
-          coreutils
-          util-linux
-          nix
-          zstd
-          jq
-          csvkit
-          sudo
-          openssh
-          iputils
-          netcat
-          python3
-          usbsdmux
-        ]);
-
-      serviceConfig = {
-        Type = "simple";
-        User = "jenkins";
-        WorkingDirectory = "/var/lib/jenkins";
-        ExecStart = "${jenkins-connection-script}";
-        Restart = "on-failure";
-        RestartSec = 5;
-      };
-    };
 in
 {
   imports =
@@ -176,14 +102,117 @@ in
     "tty"
   ];
 
-  # Agent services per hardware test device
-  systemd.services = {
-    agent-orin-agx = mkAgent "orin-agx";
-    agent-orin-nx = mkAgent "orin-nx";
-    agent-riscv = mkAgent "riscv";
-    agent-nuc = mkAgent "nuc";
-    agent-lenovo-x1 = mkAgent "lenovo-x1";
-  };
+  systemd.services =
+    let
+      # verifies that the environment file is properly configured
+      # and downloads agent.jar from the controller
+      jenkins-setup-script = pkgs.writeShellScript "jenkins-setup.sh" ''
+        if [[ ! -f jenkins.env ]]; then
+          echo "Please add jenkins controller details to $(pwd)/jenkins.env"
+          echo "CONTROLLER=" > jenkins.env
+          echo "ADMIN_PASSWORD=" >> jenkins.env
+          exit 1
+        fi
+
+        source jenkins.env
+
+        if [[ -z "$CONTROLLER" ]]; then
+          echo "Variable CONTROLLER not set in $(pwd)/jenkins.env"
+          exit 1
+        fi
+
+        if [[ -z "$ADMIN_PASSWORD" ]]; then
+          echo "Variable ADMIN_PASSWORD not set in $(pwd)/jenkins.env"
+          exit 1
+        fi
+
+        wget "$CONTROLLER/jnlpJars/agent.jar"
+      '';
+
+      # Helper function to create agent services for each hardware device
+      mkAgent =
+        device:
+        let
+          # opens a websocket connection to the jenkins controller from this agent
+          jenkins-connect-script = pkgs.writeShellScript "jenkins-connect.sh" ''
+            JENKINS_SECRET="$(
+              curl -L -s -u admin:$ADMIN_PASSWORD \
+              $CONTROLLER/computer/${device}/jenkins-agent.jnlp | 
+              sed "s/.*<application-desc><argument>\([a-z0-9]*\).*/\1\n/"
+            )"
+
+            mkdir -p "/var/lib/jenkins/agents/${device}"
+
+            ${pkgs.jdk}/bin/java \
+              -jar agent.jar \
+              -url "$CONTROLLER" \
+              -name "${device}" \
+              -secret "$JENKINS_SECRET" \
+              -workDir "/var/lib/jenkins/agents/${device}" \
+              -webSocket
+          '';
+        in
+        {
+          # agents require the setup service to run without errors before starting
+          requires = [ "setup-agents.service" ];
+          wantedBy = [ "setup-agents.service" ];
+          after = [ "setup-agents.service" ];
+
+          path =
+            [
+              brainstem
+              inputs.robot-framework.packages.${pkgs.system}.ghaf-robot
+            ]
+            ++ (with pkgs; [
+              curl
+              wget
+              jdk
+              git
+              bashInteractive
+              coreutils
+              util-linux
+              nix
+              zstd
+              jq
+              csvkit
+              sudo
+              openssh
+              iputils
+              netcat
+              python3
+              usbsdmux
+            ]);
+
+          serviceConfig = {
+            Type = "simple";
+            User = "jenkins";
+            EnvironmentFile = "/var/lib/jenkins/jenkins.env";
+            WorkingDirectory = "/var/lib/jenkins";
+            ExecStart = "${jenkins-connect-script}";
+          };
+        };
+    in
+    {
+      # the setup service does not start automatically or it would fail activation
+      # of the system since jenkins.env is empty before manually set up
+      setup-agents = {
+        path = with pkgs; [ wget ];
+        serviceConfig = {
+          Type = "oneshot";
+          User = "jenkins";
+          RemainAfterExit = "yes";
+          WorkingDirectory = "/var/lib/jenkins";
+          ExecStart = "${jenkins-setup-script}";
+        };
+      };
+
+      # one agent per unique hardware device to act as a lock
+      agent-orin-agx = mkAgent "orin-agx";
+      agent-orin-nx = mkAgent "orin-nx";
+      agent-riscv = mkAgent "riscv";
+      agent-nuc = mkAgent "nuc";
+      agent-lenovo-x1 = mkAgent "lenovo-x1";
+    };
 
   # Details of the hardware devices connected to this host
   environment.etc."jenkins/test_config.json".text =