feat: generalize backup workflows

wittdennis · Dec 23, 2024 · 21e4e26 · 21e4e26
1 parent 95c45df
commit 21e4e26
Show file tree

Hide file tree

Showing 6 changed files with 224 additions and 18 deletions.
diff --git a/cloud/custom-resources/argo-workflows/backup-restore.yaml b/cloud/custom-resources/argo-workflows/backup-restore.yaml
@@ -39,13 +39,15 @@ spec:
         - - name: scale-down
             arguments:
               parameters:
-                - name: deployment
+                - name: name
                   value: foundry
                 - name: replicaCount
                   value: "0"
+                - name: type
+                  value: statefulset
             templateRef:
-              template: scale-deployment
-              name: scale-deployment
+              template: scale-workload
+              name: scale-workload
         - - name: restore-data
             template: restore-data
             arguments:
@@ -195,10 +197,12 @@ spec:
         - - name: scale-up
             arguments:
               parameters:
-                - name: deployment
+                - name: name
                   value: foundry
                 - name: replicaCount
                   value: "1"
+                - name: type
+                  value: statefulset
             templateRef:
-              template: scale-deployment
-              name: scale-deployment
+              template: scale-workload
+              name: scale-workload
diff --git a/cloud/custom-resources/argo-workflows/foundry-backup-template.yaml b/cloud/custom-resources/argo-workflows/foundry-backup-template.yaml
@@ -0,0 +1,155 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/argoproj/argo-workflows/main/api/jsonschema/schema.json
+---
+apiVersion: argoproj.io/v1alpha1
+kind: WorkflowTemplate
+metadata:
+  name: foundry-backup-template
+spec:
+  entrypoint: main
+  onExit: exit-handler
+  artifactGC:
+    strategy: OnWorkflowDeletion
+    serviceAccountName: foundry-workflows
+  securityContext:
+    allowPrivilegeEscalation: false
+    readOnlyRootFilesystem: true
+    runAsNonRoot: true
+    seccompProfile:
+      type: RuntimeDefault
+  arguments:
+    parameters:
+      - name: instance
+      - name: restic-hostname
+      - name: pvc-name
+  volumes:
+    - name: foundry-data
+      persistentVolumeClaim:
+        claimName: "{{inputs.parameters.pvc-name}}"
+    - name: tmp
+      emptyDir: {}
+  templates:
+    - name: main
+      inputs:
+        parameters:
+          - name: instance
+      steps:
+        - - name: scale-down
+            arguments:
+              parameters:
+                - name: name
+                  value: "{{inputs.parameters.instance}}"
+                - name: replicaCount
+                  value: "0"
+                - name: type
+                  value: deployment
+            templateRef:
+              template: scale-workload
+              name: scale-workload
+        - - name: create-backup
+            template: archive-data
+        - - name: upload-backup
+            template: upload-backup
+            arguments:
+              parameters:
+                - name: restic-hostname
+              artifacts:
+                - name: backup-archive
+                  from: "{{steps.create-backup.outputs.artifacts.backup-archive}}"
+
+    - name: exit-handler
+      inputs:
+        parameters:
+          - name: instance
+      steps:
+        - - name: scale-up
+            arguments:
+              parameters:
+                - name: name
+                  value: "{{inputs.parameters.instance}}"
+                - name: replicaCount
+                  value: "1"
+                - name: type
+                  value: deployment
+            templateRef:
+              template: scale-workload
+              name: scale-workload
+
+    - name: archive-data
+      serviceAccountName: foundry-workflows
+      outputs:
+        artifacts:
+          - name: backup-archive
+            path: /tmp/foundry-data.tar.xz
+            s3:
+              key: "{{workflow.name}}/foundry-data.tar.xz"
+      container:
+        image: denniswitt/tar:0.1.2 # renovate
+        workingDir: /tmp
+        command:
+          - sh
+          - -c
+        args:
+          - |
+            set -eux
+
+            tar cfJv foundry-data.tar.xz /foundry-data/Data
+            du -sch foundry-data.tar.xz
+        volumeMounts:
+          - mountPath: /tmp
+            name: tmp
+          - mountPath: /foundry-data
+            name: foundry-data
+        securityContext:
+          runAsUser: 1000
+          runAsGroup: 1000
+          allowPrivilegeEscalation: false
+          readOnlyRootFilesystem: true
+          runAsNonRoot: true
+          seccompProfile:
+            type: RuntimeDefault
+
+    - name: upload-backup
+      serviceAccountName: foundry-workflows
+      inputs:
+        parameters:
+          - name: restic-hostname
+        artifacts:
+          - name: backup-archive
+            path: /tmp/foundry-data.tar.xz
+      container:
+        image: restic/restic:0.17.3 # renovate
+        command: [sh, -c]
+        args:
+          - |
+            restic init --no-cache || true
+            restic backup --no-cache --host {{inputs.parameters.restic-hostname}} /tmp/foundry-data.tar.xz
+            restic forget --no-cache --keep-last 90 --prune
+        env:
+          - name: AZURE_ACCOUNT_NAME
+            valueFrom:
+              secretKeyRef:
+                name: backup-settings
+                key: azure-account-name
+          - name: AZURE_ACCOUNT_KEY
+            valueFrom:
+              secretKeyRef:
+                name: backup-settings
+                key: azure-account-key
+          - name: RESTIC_PASSWORD
+            valueFrom:
+              secretKeyRef:
+                name: backup-settings
+                key: restic-password
+          - name: RESTIC_REPOSITORY
+            value: azure:foundry:/
+        volumeMounts:
+          - mountPath: /tmp
+            name: tmp
+        securityContext:
+          runAsUser: 1000
+          runAsGroup: 1000
+          allowPrivilegeEscalation: false
+          readOnlyRootFilesystem: true
+          runAsNonRoot: true
+          seccompProfile:
+            type: RuntimeDefault
diff --git a/cloud/custom-resources/argo-workflows/offsite-backup.yaml b/cloud/custom-resources/argo-workflows/offsite-backup.yaml
@@ -33,13 +33,15 @@ spec:
           - - name: scale-down
               arguments:
                 parameters:
-                  - name: deployment
+                  - name: name
                     value: foundry
                   - name: replicaCount
                     value: "0"
+                  - name: type
+                    value: deployment
               templateRef:
-                template: scale-deployment
-                name: scale-deployment
+                template: scale-workload
+                name: scale-workload
           - - name: create-backup
               template: archive-data
           - - name: upload-backup
@@ -54,13 +56,15 @@ spec:
           - - name: scale-up
               arguments:
                 parameters:
-                  - name: deployment
+                  - name: name
                     value: foundry
                   - name: replicaCount
                     value: "1"
+                  - name: type
+                    value: deployment
               templateRef:
-                template: scale-deployment
-                name: scale-deployment
+                template: scale-workload
+                name: scale-workload
 
       - name: archive-data
         serviceAccountName: foundry-workflows

diff --git a/cloud/custom-resources/argo-workflows/saltysausage-offsite-backup.yaml b/cloud/custom-resources/argo-workflows/saltysausage-offsite-backup.yaml
@@ -0,0 +1,38 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/argoproj/argo-workflows/main/api/jsonschema/schema.json
+---
+apiVersion: argoproj.io/v1alpha1
+kind: CronWorkflow
+metadata:
+  name: saltysausage-offsite-backup
+spec:
+  schedule: "30 3 * * *"
+  concurrencyPolicy: "Replace"
+  startingDeadlineSeconds: 0
+  successfulJobsHistoryLimit: 7
+  workflowSpec:
+    entrypoint: main
+    onExit: exit-handler
+    artifactGC:
+      strategy: OnWorkflowDeletion
+      serviceAccountName: foundry-workflows
+    securityContext:
+      allowPrivilegeEscalation: false
+      readOnlyRootFilesystem: true
+      runAsNonRoot: true
+      seccompProfile:
+        type: RuntimeDefault
+    templates:
+      - name: main
+        steps:
+          - - name: scale-down
+              arguments:
+                parameters:
+                  - name: instance
+                    value: foundry-saltysausage
+                  - name: restic-hostname
+                    value: foundry-saltysausage
+                  - name: pvc-name
+                    value: foundry-saltysausage-data-pvc
+              templateRef:
+                template: foundry-backup-template
+                name: backup
diff --git a/...rces/argo-workflows/scale-deployment.yaml → ...ources/argo-workflows/scale-workload.yaml b/...rces/argo-workflows/scale-deployment.yaml → ...ources/argo-workflows/scale-workload.yaml
@@ -3,9 +3,9 @@
 apiVersion: argoproj.io/v1alpha1
 kind: WorkflowTemplate
 metadata:
-  name: scale-deployment
+  name: scale-workload
 spec:
-  entrypoint: scale-deployment
+  entrypoint: scale-workload
   artifactGC:
     strategy: OnWorkflowDeletion
     serviceAccountName: foundry-workflows
@@ -20,16 +20,19 @@ spec:
   arguments:
     parameters:
       - name: replicaCount
-      - name: deployment
+      - name: name
         default: foundry
+      - name: type
+        default: statefulset
   templates:
-    - name: scale-deployment
+    - name: scale-workload
       serviceAccountName: foundry-workflows
       automountServiceAccountToken: true
       inputs:
         parameters:
           - name: replicaCount
-          - name: deployment
+          - name: name
+          - name: type
       container:
         image: bitnami/kubectl:1.32.0 # renovate
         command:
@@ -39,7 +42,7 @@ spec:
           - |
             set -eu
 
-            kubectl scale --replicas={{inputs.parameters.replicaCount}} -n foundry deployment/{{inputs.parameters.deployment}}
+            kubectl scale --replicas={{inputs.parameters.replicaCount}} -n foundry {{inputs.parameters.type}}/{{inputs.parameters.name}}
       securityContext:
         runAsUser: 1000
         runAsGroup: 1000

diff --git a/cloud/rbac/deployment-scaler-role.yaml b/cloud/rbac/deployment-scaler-role.yaml
@@ -8,12 +8,14 @@ rules:
       - get
     resources:
       - deployments
+      - statefulsets
     apiGroups:
       - apps
   - verbs:
       - patch
       - update
     resources:
       - deployments/scale
+      - statefulsets/scale
     apiGroups:
       - apps