chore: support wal-g datafile backup for pgsql and improve log ttl

(cherry picked from commit bb20ad2)

deploy/apecloud-mysql/templates/backuptool-pitr.yaml

@@ -17,8 +17,6 @@ spec:
       value: "$(VOLUME_DATA_DIR)/data"
     - name: CONF_DIR
       value: "$(VOLUME_DATA_DIR)/conf"
-    - name: RECOVERY_TIME
-      value: $KB_RECOVERY_TIME
     - name: TIME_FORMAT
       value: 2006-01-02T15:04:05Z
     - name: LOG_DIR
@@ -55,8 +53,8 @@ spec:
           BASE_BACKUP_TIME=$(cat $DATA_DIR/xtrabackup_info | grep start_time | awk -F ' = ' '{print $2}');
           BASE_BACKUP_TIME=$(date -d"${BASE_BACKUP_TIME}" -u '+%Y-%m-%dT%H:%M:%SZ')
         fi
-        BINLOG_LATEST_TIME=$(date -d"${RECOVERY_TIME} +1 day" -u '+%Y-%m-%dT%H:%M:%SZ')
-        wal-g binlog-replay --since-time "${BASE_BACKUP_TIME}" --until "${RECOVERY_TIME}" --until-binlog-last-modified-time "${BINLOG_LATEST_TIME}";
+        BINLOG_LATEST_TIME=$(date -d"${KB_RECOVERY_TIME} +1 day" -u '+%Y-%m-%dT%H:%M:%SZ')
+        wal-g binlog-replay --since-time "${BASE_BACKUP_TIME}" --until "${KB_RECOVERY_TIME}" --until-binlog-last-modified-time "${BINLOG_LATEST_TIME}";
         echo "done.";
         sync;
   backupCommands:

deploy/mongodb/templates/backuptool_pitr.yaml

@@ -35,7 +35,7 @@ spec:
       #!/bin/bash
       mkdir -p ${BACKUP_DIR} && cd ${BACKUP_DIR}
       # retention 8 days by default
-      retention_minute=11520
+      retention_minute=""
       if [ ! -z ${LOGFILE_TTL_SECOND} ];then
         retention_minute=$((${LOGFILE_TTL_SECOND}/60))
       fi
@@ -92,14 +92,16 @@ spec:
       }
       # purge the expired files
       purge_expired_files() {
-        purgeCounter=$((purgeCounter+3))
-        if [ $purgeCounter -ge 60 ]; then
-           purgeCounter=0
-           fileCount=$(find ${BACKUP_DIR}/oplog_005 -mmin +${retention_minute} -name "*.lz4" | wc -l)
-           find ${BACKUP_DIR}/oplog_005 -mmin +${retention_minute} -name "*.lz4" -exec rm -rf {} \;
-           if [ ${fileCount} -gt 0 ]; then
-              echo "clean up expired oplog file successfully, file count: ${fileCount}"
-           fi
+        if [ ! -z ${LOGFILE_TTL_SECOND} ];then
+          purgeCounter=$((purgeCounter+3))
+          if [ $purgeCounter -ge 60 ]; then
+             purgeCounter=0
+             fileCount=$(find ${BACKUP_DIR}/oplog_005 -mmin +${retention_minute} -name "*.lz4" | wc -l)
+             find ${BACKUP_DIR}/oplog_005 -mmin +${retention_minute} -name "*.lz4" -exec rm -rf {} \;
+             if [ ${fileCount} -gt 0 ]; then
+                echo "clean up expired oplog file successfully, file count: ${fileCount}"
+             fi
+          fi
         fi
       }
       # create oplog push process

deploy/postgresql/templates/backuppolicytemplate.yaml

@@ -33,7 +33,7 @@ spec:
         cronExpression: "0 18 * * *"
       logfile:
         enable: false
-        cronExpression: "*/5 * * * *"
+        cronExpression: "*/2 * * * *"
     snapshot:
       target:
         connectionCredentialKey:

deploy/postgresql/templates/backuptool.yaml → deploy/postgresql/templates/backuptool-pgbasebackup.yaml

@@ -9,12 +9,6 @@ spec:
   image: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.repository }}:{{ .Values.image.tag }}
   deployKind: job
   env:
-    - name: RESTORE_DATA_DIR
-      value: /home/postgres/pgdata/kb_restore
-    - name: TMP_DATA_DIR
-      value: /home/postgres/pgdata/kb_restore/tmp_data
-    - name: TMP_ARCH_DATA_DIR
-      value: /home/postgres/pgdata/kb_restore/arch
     - name: DATA_DIR
       value: /home/postgres/pgdata/pgroot/data
   physical:
@@ -26,8 +20,8 @@ spec:
         set -e;
         cd ${BACKUP_DIR};
         mkdir -p ${DATA_DIR};
-        tar -xvf base.tar.gz -C ${DATA_DIR}/;
-        tar -xvf pg_wal.tar.gz -C ${DATA_DIR}/pg_wal/;
+        tar -xvf base.tar -C ${DATA_DIR}/;
+        tar -xvf pg_wal.tar -C ${DATA_DIR}/pg_wal/;
         echo "done!";
     incrementalRestoreCommands: []
   logical:
@@ -36,9 +30,12 @@ spec:
   backupCommands:
     - sh
     - -c
-    - >
+    - |
       set -e;
+      if [ -d ${BACKUP_DIR} ]; then
+        rm -rf ${BACKUP_DIR}
+      fi
       mkdir -p ${BACKUP_DIR};
-      echo ${DB_PASSWORD} | pg_basebackup -Ft -Pv -Xs -z -D ${BACKUP_DIR} -Z5 -h ${DB_HOST} -U standby -W;
+      echo ${DB_PASSWORD} | pg_basebackup -Ft -Pv -c fast -Xs -D ${BACKUP_DIR} -h ${DB_HOST} -U standby -W;
       echo "TOTAL SIZE: $(du -shx ${BACKUP_DIR}|awk '{print $1}')" > ${DATA_DIR}/basebackup.info;
   incrementalBackupCommands: []

deploy/postgresql/templates/backuptool-pitr.yaml

@@ -19,8 +19,6 @@ spec:
       value: "$(VOLUME_DATA_DIR)/pgroot/data"
     - name: CONF_DIR
       value: "$(VOLUME_DATA_DIR)/conf"
-    - name: RECOVERY_TIME
-      value: $KB_RECOVERY_TIME
     - name: TIME_FORMAT
       value: 2006-01-02 15:04:05 MST
     - name: LOG_DIR
@@ -43,6 +41,7 @@ spec:
         if [ -d ${DATA_DIR}.old ]; then echo "${DATA_DIR}.old directory already exists, skip restore."; exit 0; fi
         mkdir -p ${PITR_DIR};
         cd ${PITR_DIR}
+        # TODO Obtain logs directly from WAL archive。
         for i in $(find ${BACKUP_DIR} -name "*.gz"); do
           echo "copying ${i}";
           cp ${i} $(basename $i);
@@ -58,7 +57,7 @@ spec:
         echo "sync;" >> ${RESTORE_SCRIPT_DIR}/kb_restore.sh;
         chmod +x ${RESTORE_SCRIPT_DIR}/kb_restore.sh;
         echo "restore_command='case "%f" in *history) cp ${PITR_DIR}/%f %p ;; *) mv ${PITR_DIR}/%f %p ;; esac'" > ${CONF_DIR}/recovery.conf;
-        echo "recovery_target_time='${RECOVERY_TIME}'" >> ${CONF_DIR}/recovery.conf;
+        echo "recovery_target_time='${KB_RECOVERY_TIME}'" >> ${CONF_DIR}/recovery.conf;
         echo "recovery_target_action='promote'" >> ${CONF_DIR}/recovery.conf;
         echo "recovery_target_timeline='latest'" >> ${CONF_DIR}/recovery.conf;
         mv ${DATA_DIR} ${DATA_DIR}.old;
@@ -69,8 +68,11 @@ spec:
     - -c
     - |
       set -e;
-      EXPIRED_INCR_LOG=${BACKUP_DIR}/$(date -d"7 day ago" +%Y%m%d);
-      if [ -d ${EXPIRED_INCR_LOG} ]; then rm -rf ${EXPIRED_INCR_LOG}; fi
+      if [ ! -z ${LOGFILE_TTL_SECOND} ];then
+        retention_day=$((${LOGFILE_TTL_SECOND}/86400))
+        EXPIRED_INCR_LOG=${BACKUP_DIR}/$(date -d"${retention_day} day ago" +%Y%m%d);
+        if [ -d ${EXPIRED_INCR_LOG} ]; then rm -rf ${EXPIRED_INCR_LOG}; fi
+      fi
       export PGPASSWORD=${DB_PASSWORD}
       PSQL="psql -h ${DB_HOST} -U ${DB_USER}"
       LAST_TRANS=$(pg_waldump $(${PSQL} -Atc "select pg_walfile_name(pg_current_wal_lsn())") --rmgr=Transaction |tail -n 1)

deploy/postgresql/templates/backuptool-wal-g.yaml

@@ -0,0 +1,49 @@
+apiVersion: dataprotection.kubeblocks.io/v1alpha1
+kind: BackupTool
+metadata:
+  name: postgres-wal-g
+  labels:
+    clusterdefinition.kubeblocks.io/name: postgresql
+spec:
+  image: registry.cn-hangzhou.aliyuncs.com/apecloud/spilo:14.7.2
+  deployKind: job
+  env:
+    - name: DATA_DIR
+      value: /home/postgres/pgdata/pgroot/data
+    - name: WAL_DIR
+      value: $(DATA_DIR)/pg_wal
+    - name: WALG_PG_WAL_SIZE
+      value: "16"
+    - name: WALG_TAR_SIZE_THRESHOLD
+      value: "4294967296"
+    - name: WALG_UPLOAD_DISK_CONCURRENCY
+      value: "8"
+  physical:
+    restoreCommands:
+      - bash
+      - -c
+      - |
+        #!/bin/bash
+        # NOTE: this basebackup only support for pitr.
+        set -e;
+        mkdir -p ${DATA_DIR};
+        # TODO check KB_RECOVERY_TIME, if not exits, do wal-log restore.
+        WALG_FILE_PREFIX=${BACKUP_DIR} wal-g backup-fetch ${DATA_DIR} LATEST
+        echo "done!";
+    incrementalRestoreCommands: []
+  logical:
+    restoreCommands: []
+    incrementalRestoreCommands: []
+  backupCommands:
+    - bash
+    - -c
+    - |-
+      set -e;
+      # TODO check if pitr is enabled
+      if [ -d ${BACKUP_DIR} ]; then
+        rm -rf ${BACKUP_DIR}
+      fi
+      mkdir -p ${BACKUP_DIR}
+      WALG_FILE_PREFIX=${BACKUP_DIR} PGHOST=${DB_HOST} PGUSER=postgres PGPASSWORD=${DB_PASSWORD} PGPORT=5432 wal-g backup-push ${DATA_DIR}
+      echo "TOTAL SIZE: $(du -shx ${BACKUP_DIR}|awk '{print $1}')" > ${DATA_DIR}/basebackup.info;
+  incrementalBackupCommands: []

internal/controller/plan/restore.go

@@ -331,20 +331,8 @@ func (p *RestoreManager) getRecoveryInfo(baseBackup, logfileBackup *dpv1alpha1.B
 	}
 	// build env of recovery time
 	spec := &backupTool.Spec
-	timeFormat := time.RFC3339
-	envTimeEnvIdx := -1
-	for i, env := range spec.Env {
-		if env.Value == "$KB_RECOVERY_TIME" {
-			envTimeEnvIdx = i
-		} else if env.Name == constant.DPTimeFormat {
-			timeFormat = env.Value
-		}
-	}
-	if envTimeEnvIdx != -1 {
-		spec.Env[envTimeEnvIdx].Value = p.restoreTime.UTC().Format(timeFormat)
-	} else {
-		headEnv = append(headEnv, corev1.EnvVar{Name: constant.DPKBRecoveryTime, Value: p.restoreTime.UTC().Format(timeFormat)})
-	}
+	timeFormat := p.getTimeFormat(spec.Env)
+	headEnv = append(headEnv, corev1.EnvVar{Name: constant.DPKBRecoveryTime, Value: p.restoreTime.UTC().Format(timeFormat)})
 	headEnv = append(headEnv, corev1.EnvVar{Name: constant.DPKBRecoveryTimestamp, Value: strconv.FormatInt(p.restoreTime.Unix(), 10)})
 	// build env of backup startTime and user contexts
 	if baseBackup.Status.Manifests != nil {
@@ -576,19 +564,16 @@ func (p *RestoreManager) BuildDatafileRestoreJobByPVCS(synthesizedComponent *com
 	}
 
 	// builds env
-	backupDataPath := fmt.Sprintf("/%s/%s", backup.Name, backup.Namespace)
-	manifests := backup.Status.Manifests
-	if manifests != nil && manifests.BackupTool != nil {
-		backupDataPath = fmt.Sprintf("/%s%s", backup.Name, manifests.BackupTool.FilePath)
-	}
 	env := []corev1.EnvVar{
 		{
-			Name:  "BACKUP_NAME",
+			Name:  constant.DPBackupName,
 			Value: backup.Name,
-		}, {
-			Name:  "BACKUP_DIR",
-			Value: backupDataPath,
-		}}
+		},
+	}
+	manifests := backup.Status.Manifests
+	if manifests != nil && manifests.BackupTool != nil {
+		env = append(env, corev1.EnvVar{Name: constant.DPBackupDIR, Value: fmt.Sprintf("/%s%s", backup.Name, manifests.BackupTool.FilePath)})
+	}
 	// merges env from backup tool.
 	env = append(env, backupTool.Spec.Env...)
 	objs = make([]client.Object, 0)
@@ -826,3 +811,12 @@ func (p *RestoreManager) BuildCommonLabels(synthesizedComponent *component.Synth
 		constant.KBAppComponentLabelKey: synthesizedComponent.Name,
 	}
 }
+
+func (p *RestoreManager) getTimeFormat(envs []corev1.EnvVar) string {
+	for _, env := range envs {
+		if env.Name == constant.DPTimeFormat {
+			return env.Value
+		}
+	}
+	return time.RFC3339
+}