Merge pull request #801 from k8up-io/pvc-rwo

feat: add support for RWO PVCs backups
k8up-io · Feb 9, 2023 · 5a86c1d · 5a86c1d
2 parents f843a8b + dd9e544
commit 5a86c1d
Show file tree

Hide file tree

Showing 27 changed files with 1,123 additions and 172 deletions.
diff --git a/api/v1/common_types.go b/api/v1/common_types.go
@@ -74,12 +74,17 @@ const (
 
 	// LabelK8upType is the label key that identifies the job type
 	LabelK8upType = "k8up.io/type"
+	// LabelK8upOwnedBy is a label used to indicated which resource owns this resource to make it easy to fetch owned resources.
+	LabelK8upOwnedBy = "k8up.io/owned-by"
 	// Deprecated: LegacyLabelK8upType is the former label key that identified the job type
 	LegacyLabelK8upType = "k8up.syn.tools/type"
 	// LabelManagedBy identifies the tool being used to manage the operation of a resource
 	LabelManagedBy = "app.kubernetes.io/managed-by"
 	// LabelRepositoryHash is the label key that identifies the Restic repository
 	LabelRepositoryHash = "k8up.io/repository-hash"
+
+	// AnnotationK8upHostname is an annotation one can set on RWO PVCs to try to back up them on the specified node.
+	AnnotationK8upHostname = "k8up.io/hostname"
 )
 
 // String casts the value to string.

diff --git a/charts/k8up/templates/operator-clusterrole.yaml b/charts/k8up/templates/operator-clusterrole.yaml
@@ -31,15 +31,6 @@ rules:
       - patch
       - update
       - watch
-  - apiGroups:
-      - batch
-    resources:
-      - jobs/finalizers
-      - jobs/status
-    verbs:
-      - get
-      - patch
-      - update
   - apiGroups:
       - coordination.k8s.io
     resources:
@@ -64,6 +55,14 @@ rules:
       - get
       - list
       - watch
+  - apiGroups:
+      - ""
+    resources:
+      - persistentvolumes
+    verbs:
+      - get
+      - list
+      - watch
   - apiGroups:
       - ""
     resources:

diff --git a/cmd/restic/main.go b/cmd/restic/main.go
@@ -6,6 +6,7 @@ import (
 	"os"
 	"os/signal"
 	"syscall"
+	"time"
 
 	"github.com/go-logr/logr"
 	"github.com/urfave/cli/v2"
@@ -81,6 +82,9 @@ var (
 			&cli.StringFlag{Destination: &cfg.Config.PruneKeepWithinMonthly, Name: "keepWithinMonthly", EnvVars: []string{"KEEP_WITHIN_MONTHLY"}, Usage: "While pruning, keep monthly snapshots within the given duration, e.g. '2y5m7d3h'"},
 			&cli.StringFlag{Destination: &cfg.Config.PruneKeepWithinYearly, Name: "keepWithinYearly", EnvVars: []string{"KEEP_WITHIN_YEARLY"}, Usage: "While pruning, keep yearly snapshots within the given duration, e.g. '2y5m7d3h'"},
 			&cli.StringFlag{Destination: &cfg.Config.PruneKeepWithin, Name: "keepWithin", EnvVars: []string{"KEEP_WITHIN"}, Usage: "While pruning, keep tagged snapshots within the given duration, e.g. '2y5m7d3h'"},
+
+			&cli.StringSliceFlag{Name: "targetPods", EnvVars: []string{"TARGET_PODS"}, Usage: "Filter list of pods by TARGET_PODS names"},
+			&cli.DurationFlag{Destination: &cfg.Config.SleepDuration, Name: "sleepDuration", EnvVars: []string{"SLEEP_DURATION"}, Usage: "Sleep for specified amount until init starts"},
 		},
 	}
 )
@@ -90,6 +94,8 @@ func resticMain(c *cli.Context) error {
 	resticLog.Info("initializing")
 
 	cfg.Config.Tags = c.StringSlice("tag")
+	cfg.Config.TargetPods = c.StringSlice("targetPods")
+
 	err := cfg.Config.Validate()
 	if err != nil {
 		return err
@@ -122,6 +128,10 @@ func run(ctx context.Context, resticCLI *resticCli.Restic, mainLogger logr.Logge
 }
 
 func resticInitialization(resticCLI *resticCli.Restic, mainLogger logr.Logger) error {
+	if cfg.Config.SleepDuration > 0 {
+		mainLogger.Info("sleeping until init", "duration", cfg.Config.SleepDuration)
+		time.Sleep(cfg.Config.SleepDuration)
+	}
 	if err := resticCLI.Init(); err != nil {
 		return fmt.Errorf("failed to initialise the restic repository: %w", err)
 	}
@@ -235,7 +245,7 @@ func backupAnnotatedPods(ctx context.Context, resticCLI *resticCli.Restic, mainL
 		return nil
 	}
 
-	podLister := kubernetes.NewPodLister(ctx, cfg.Config.BackupCommandAnnotation, cfg.Config.BackupFileExtensionAnnotation, cfg.Config.Hostname, mainLogger)
+	podLister := kubernetes.NewPodLister(ctx, cfg.Config.BackupCommandAnnotation, cfg.Config.BackupFileExtensionAnnotation, cfg.Config.Hostname, cfg.Config.TargetPods, mainLogger)
 	podList, err := podLister.ListPods()
 	if err != nil {
 		mainLogger.Error(err, "could not list pods", "namespace", cfg.Config.Hostname)

diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml
@@ -53,6 +53,14 @@ rules:
   - get
   - list
   - watch
+- apiGroups:
+  - ""
+  resources:
+  - persistentvolumes
+  verbs:
+  - get
+  - list
+  - watch
 - apiGroups:
   - ""
   resources:

diff --git a/config/samples/deployments/mongodb.yaml b/config/samples/deployments/mongodb.yaml
@@ -1,12 +1,10 @@
 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  annotations:
-    k8up.io/backupcommand: mongodump --archive -uroot -pexample
   labels:
     app: mongodb
   name: mongodb
-  namespace: myproject
+  namespace: default
 spec:
   replicas: 1
   revisionHistoryLimit: 2
@@ -22,6 +20,8 @@ spec:
     metadata:
       labels:
         app: mongodb
+      annotations:
+        k8up.io/backupcommand: mongodump --archive -uroot -pexample
     spec:
       containers:
         - env:
@@ -50,11 +50,20 @@ spec:
         - name: mongodb
           persistentVolumeClaim:
             claimName: mongodb
+      tolerations:
+        - effect: NoSchedule
+          key: node-role.kubernetes.io/master
+        - effect: NoSchedule
+          key: node-role.kubernetes.io/control-plane
+      nodeSelector:
+        kubernetes.io/os: linux
+        node-role.kubernetes.io/control-plane: ""
 ---
 kind: PersistentVolumeClaim
 apiVersion: v1
 metadata:
   name: mongodb
+  namespace: default
 spec:
   accessModes:
     - ReadWriteOnce

diff --git a/docs/modules/ROOT/examples/usage/restic.txt b/docs/modules/ROOT/examples/usage/restic.txt
@@ -11,39 +11,41 @@ DESCRIPTION:
    Start k8up in restic mode
 
 OPTIONS:
-   --check                               Set, if the container should do a check (default: false)
-   --prune                               Set, if the container should do a prune (default: false)
-   --restore                             Set, if the container should attempt a restore (default: false)
-   --archive                             Set, if the container should do an archive (default: false)
-   --tag value [ --tag value ]           List of tags to consider for given operation
-   --backupCommandAnnotation value       Defines the command to invoke when doing a backup via STDOUT. [$BACKUPCOMMAND_ANNOTATION]
-   --fileExtensionAnnotation value       Defines the file extension to use for STDOUT backups. [$FILEEXTENSION_ANNOTATION]
-   --promURL value                       Sets the URL of a prometheus push gateway to report metrics. [$PROM_URL]
-   --webhookURL value, --statsURL value  Sets the URL of a server which will retrieve a webhook after the action completes. [$STATS_URL]
-   --backupDir value                     Set from which directory the backup should be performed. (default: "/data") [$BACKUP_DIR]
-   --restoreDir value                    Set to which directory the restore should be performed. (default: "/data") [$RESTORE_DIR]
-   --restoreFilter value                 Simple filter to define what should get restored. For example the PVC name
-   --restoreSnap value                   Snapshot ID, if empty takes the latest snapshot
-   --restoreType value                   Type of this restore, 'folder' or 's3'
-   --restoreS3AccessKey value            S3 access key used to connect to the S3 endpoint when restoring [$RESTORE_ACCESSKEYID]
-   --restoreS3SecretKey value            S3 secret key used to connect to the S3 endpoint when restoring [$RESTORE_SECRETACCESSKEY]
-   --restoreS3Endpoint value             S3 endpoint to connect to when restoring, e.g. 'https://minio.svc:9000/backup [$RESTORE_S3ENDPOINT]
-   --verifyRestore                       If the restore should get verified, only for PVCs restore (default: false)
-   --trimRestorePath                     If set, strips the value of --restoreDir from the lefts side of the remote restore path value (default: enabled) [$TRIM_RESTOREPATH]
-   --resticBin value                     The path to the restic binary. (default: "/usr/local/bin/restic") [$RESTIC_BINARY]
-   --resticRepository value              The restic repository to perform the action with [$RESTIC_REPOSITORY]
-   --resticOptions value                 Additional options to pass to restic in the format 'key=value,key2=value2' [$RESTIC_OPTIONS]
-   --keepLatest value                    While pruning, keep at the latest snapshot (default: 0) [$KEEP_LAST, $KEEP_LATEST]
-   --keepHourly value                    While pruning, keep hourly snapshots (default: 0) [$KEEP_HOURLY]
-   --keepDaily value                     While pruning, keep daily snapshots (default: 0) [$KEEP_DAILY]
-   --keepWeekly value                    While pruning, keep weekly snapshots (default: 0) [$KEEP_WEEKLY]
-   --keepMonthly value                   While pruning, keep monthly snapshots (default: 0) [$KEEP_MONTHLY]
-   --keepYearly value                    While pruning, keep yearly snapshots (default: 0) [$KEEP_YEARLY]
-   --keepTags                            While pruning, keep tagged snapshots (default: false) [$KEEP_TAG, $KEEP_TAGS]
-   --keepWithinHourly value              While pruning, keep hourly snapshots within the given duration, e.g. '2y5m7d3h' [$KEEP_WITHIN_HOURLY]
-   --keepWithinDaily value               While pruning, keep daily snapshots within the given duration, e.g. '2y5m7d3h' [$KEEP_WITHIN_DAILY]
-   --keepWithinWeekly value              While pruning, keep weekly snapshots within the given duration, e.g. '2y5m7d3h' [$KEEP_WITHIN_WEEKLY]
-   --keepWithinMonthly value             While pruning, keep monthly snapshots within the given duration, e.g. '2y5m7d3h' [$KEEP_WITHIN_MONTHLY]
-   --keepWithinYearly value              While pruning, keep yearly snapshots within the given duration, e.g. '2y5m7d3h' [$KEEP_WITHIN_YEARLY]
-   --keepWithin value                    While pruning, keep tagged snapshots within the given duration, e.g. '2y5m7d3h' [$KEEP_WITHIN]
-   --help, -h                            show help (default: false)
+   --check                                    Set, if the container should do a check (default: false)
+   --prune                                    Set, if the container should do a prune (default: false)
+   --restore                                  Set, if the container should attempt a restore (default: false)
+   --archive                                  Set, if the container should do an archive (default: false)
+   --tag value [ --tag value ]                List of tags to consider for given operation
+   --backupCommandAnnotation value            Defines the command to invoke when doing a backup via STDOUT. [$BACKUPCOMMAND_ANNOTATION]
+   --fileExtensionAnnotation value            Defines the file extension to use for STDOUT backups. [$FILEEXTENSION_ANNOTATION]
+   --promURL value                            Sets the URL of a prometheus push gateway to report metrics. [$PROM_URL]
+   --webhookURL value, --statsURL value       Sets the URL of a server which will retrieve a webhook after the action completes. [$STATS_URL]
+   --backupDir value                          Set from which directory the backup should be performed. (default: "/data") [$BACKUP_DIR]
+   --restoreDir value                         Set to which directory the restore should be performed. (default: "/data") [$RESTORE_DIR]
+   --restoreFilter value                      Simple filter to define what should get restored. For example the PVC name
+   --restoreSnap value                        Snapshot ID, if empty takes the latest snapshot
+   --restoreType value                        Type of this restore, 'folder' or 's3'
+   --restoreS3AccessKey value                 S3 access key used to connect to the S3 endpoint when restoring [$RESTORE_ACCESSKEYID]
+   --restoreS3SecretKey value                 S3 secret key used to connect to the S3 endpoint when restoring [$RESTORE_SECRETACCESSKEY]
+   --restoreS3Endpoint value                  S3 endpoint to connect to when restoring, e.g. 'https://minio.svc:9000/backup [$RESTORE_S3ENDPOINT]
+   --verifyRestore                            If the restore should get verified, only for PVCs restore (default: false)
+   --trimRestorePath                          If set, strips the value of --restoreDir from the lefts side of the remote restore path value (default: enabled) [$TRIM_RESTOREPATH]
+   --resticBin value                          The path to the restic binary. (default: "/usr/local/bin/restic") [$RESTIC_BINARY]
+   --resticRepository value                   The restic repository to perform the action with [$RESTIC_REPOSITORY]
+   --resticOptions value                      Additional options to pass to restic in the format 'key=value,key2=value2' [$RESTIC_OPTIONS]
+   --keepLatest value                         While pruning, keep at the latest snapshot (default: 0) [$KEEP_LAST, $KEEP_LATEST]
+   --keepHourly value                         While pruning, keep hourly snapshots (default: 0) [$KEEP_HOURLY]
+   --keepDaily value                          While pruning, keep daily snapshots (default: 0) [$KEEP_DAILY]
+   --keepWeekly value                         While pruning, keep weekly snapshots (default: 0) [$KEEP_WEEKLY]
+   --keepMonthly value                        While pruning, keep monthly snapshots (default: 0) [$KEEP_MONTHLY]
+   --keepYearly value                         While pruning, keep yearly snapshots (default: 0) [$KEEP_YEARLY]
+   --keepTags                                 While pruning, keep tagged snapshots (default: false) [$KEEP_TAG, $KEEP_TAGS]
+   --keepWithinHourly value                   While pruning, keep hourly snapshots within the given duration, e.g. '2y5m7d3h' [$KEEP_WITHIN_HOURLY]
+   --keepWithinDaily value                    While pruning, keep daily snapshots within the given duration, e.g. '2y5m7d3h' [$KEEP_WITHIN_DAILY]
+   --keepWithinWeekly value                   While pruning, keep weekly snapshots within the given duration, e.g. '2y5m7d3h' [$KEEP_WITHIN_WEEKLY]
+   --keepWithinMonthly value                  While pruning, keep monthly snapshots within the given duration, e.g. '2y5m7d3h' [$KEEP_WITHIN_MONTHLY]
+   --keepWithinYearly value                   While pruning, keep yearly snapshots within the given duration, e.g. '2y5m7d3h' [$KEEP_WITHIN_YEARLY]
+   --keepWithin value                         While pruning, keep tagged snapshots within the given duration, e.g. '2y5m7d3h' [$KEEP_WITHIN]
+   --targetPods value [ --targetPods value ]  Filter list of pods by TARGET_PODS names [$TARGET_PODS]
+   --sleepDuration value                      Sleep for specified amount until init starts (default: 0s) [$SLEEP_DURATION]
+   --help, -h                                 show help (default: false)
diff --git a/e2e/definitions/kind/config.yaml b/e2e/definitions/kind/config.yaml
@@ -2,6 +2,6 @@ apiVersion: kind.x-k8s.io/v1alpha4
 kind: Cluster
 nodes:
 - role: control-plane
-  extraMounts:
-  - hostPath: ./e2e/debug/data
-    containerPath: /tmp/e2e
+- role: worker
+  labels:
+    worker: true
diff --git a/e2e/definitions/pv/pv.yaml b/e2e/definitions/pv/pv.yaml
diff --git a/e2e/definitions/pv/pvc.yaml b/e2e/definitions/pv/pvc.yaml
@@ -9,8 +9,5 @@ spec:
   resources:
     requests:
       storage: 1Gi
-  storageClassName: hostpath
+  storageClassName: standard
   volumeMode: Filesystem
-  selector:
-    matchLabels:
-      pv.kubernetes.io/type: e2e-test
diff --git a/e2e/definitions/pvc-rwo-subject/controlplane.yaml b/e2e/definitions/pvc-rwo-subject/controlplane.yaml
@@ -0,0 +1,63 @@
+kind: PersistentVolumeClaim
+apiVersion: v1
+metadata:
+  name: pvc-rwo-subject-pvc-controlplane
+  namespace: k8up-e2e-subject
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 10Mi
+  storageClassName: standard
+  volumeMode: Filesystem
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: pvc-rwo-subject-controlplane
+  namespace: k8up-e2e-subject
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: subject-controlplane
+  template:
+    metadata:
+      labels:
+        app: subject-controlplane
+    spec:
+      containers:
+      - name: pvc-rwo-subject-container
+        image: quay.io/prometheus/busybox:latest
+        imagePullPolicy: IfNotPresent
+        args:
+        - sh
+        - -c
+        - |
+          printf "$BACKUP_FILE_CONTENT" | tee "/data/$BACKUP_FILE_NAME" && \
+          echo && \
+          ls -la /data && \
+          echo "test file /data/$BACKUP_FILE_NAME written, sleeping now" && \
+          sleep infinity
+        securityContext:
+          runAsUser: $ID
+        volumeMounts:
+        - name: volume
+          mountPath: /data
+        env:
+        - name: BACKUP_FILE_CONTENT
+          value: ""
+        - name: BACKUP_FILE_NAME
+          value: ""
+      volumes:
+      - name: volume
+        persistentVolumeClaim:
+          claimName: pvc-rwo-subject-pvc-controlplane
+      tolerations:
+        - effect: NoSchedule
+          key: node-role.kubernetes.io/master
+        - effect: NoSchedule
+          key: node-role.kubernetes.io/control-plane
+      nodeSelector:
+        node-role.kubernetes.io/control-plane: ""
diff --git a/e2e/definitions/pvc-rwo-subject/worker.yaml b/e2e/definitions/pvc-rwo-subject/worker.yaml
@@ -0,0 +1,58 @@
+kind: PersistentVolumeClaim
+apiVersion: v1
+metadata:
+  name: pvc-rwo-subject-pvc-worker
+  namespace: k8up-e2e-subject
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 10Mi
+  storageClassName: standard
+  volumeMode: Filesystem
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: pvc-rwo-subject-worker
+  namespace: k8up-e2e-subject
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: subject-worker
+  template:
+    metadata:
+      labels:
+        app: subject-worker
+    spec:
+      containers:
+      - name: pvc-rwo-subject-container
+        image: quay.io/prometheus/busybox:latest
+        imagePullPolicy: IfNotPresent
+        args:
+        - sh
+        - -c
+        - |
+          printf "$BACKUP_FILE_CONTENT" | tee "/data/$BACKUP_FILE_NAME" && \
+          echo && \
+          ls -la /data && \
+          echo "test file /data/$BACKUP_FILE_NAME written, sleeping now" && \
+          sleep infinity
+        securityContext:
+          runAsUser: $ID
+        volumeMounts:
+        - name: volume
+          mountPath: /data
+        env:
+        - name: BACKUP_FILE_CONTENT
+          value: ""
+        - name: BACKUP_FILE_NAME
+          value: ""
+      volumes:
+      - name: volume
+        persistentVolumeClaim:
+          claimName: pvc-rwo-subject-pvc-worker
+      nodeSelector:
+        worker: "true"