diff --git a/deploy/apecloud-mysql/templates/backuptool-pitr.yaml b/deploy/apecloud-mysql/templates/backuptool-pitr.yaml index c12335934d4..20dc56d25c9 100644 --- a/deploy/apecloud-mysql/templates/backuptool-pitr.yaml +++ b/deploy/apecloud-mysql/templates/backuptool-pitr.yaml @@ -17,8 +17,6 @@ spec: value: "$(VOLUME_DATA_DIR)/data" - name: CONF_DIR value: "$(VOLUME_DATA_DIR)/conf" - - name: RECOVERY_TIME - value: $KB_RECOVERY_TIME - name: TIME_FORMAT value: 2006-01-02T15:04:05Z - name: LOG_DIR @@ -55,8 +53,8 @@ spec: BASE_BACKUP_TIME=$(cat $DATA_DIR/xtrabackup_info | grep start_time | awk -F ' = ' '{print $2}'); BASE_BACKUP_TIME=$(date -d"${BASE_BACKUP_TIME}" -u '+%Y-%m-%dT%H:%M:%SZ') fi - BINLOG_LATEST_TIME=$(date -d"${RECOVERY_TIME} +1 day" -u '+%Y-%m-%dT%H:%M:%SZ') - wal-g binlog-replay --since-time "${BASE_BACKUP_TIME}" --until "${RECOVERY_TIME}" --until-binlog-last-modified-time "${BINLOG_LATEST_TIME}"; + BINLOG_LATEST_TIME=$(date -d"${KB_RECOVERY_TIME} +1 day" -u '+%Y-%m-%dT%H:%M:%SZ') + wal-g binlog-replay --since-time "${BASE_BACKUP_TIME}" --until "${KB_RECOVERY_TIME}" --until-binlog-last-modified-time "${BINLOG_LATEST_TIME}"; echo "done."; sync; backupCommands: diff --git a/deploy/mongodb/templates/backuptool_pitr.yaml b/deploy/mongodb/templates/backuptool_pitr.yaml index 85c4ce2119f..df0abe01df1 100644 --- a/deploy/mongodb/templates/backuptool_pitr.yaml +++ b/deploy/mongodb/templates/backuptool_pitr.yaml @@ -35,7 +35,7 @@ spec: #!/bin/bash mkdir -p ${BACKUP_DIR} && cd ${BACKUP_DIR} # retention 8 days by default - retention_minute=11520 + retention_minute="" if [ ! -z ${LOGFILE_TTL_SECOND} ];then retention_minute=$((${LOGFILE_TTL_SECOND}/60)) fi @@ -92,14 +92,16 @@ spec: } # purge the expired files purge_expired_files() { - purgeCounter=$((purgeCounter+3)) - if [ $purgeCounter -ge 60 ]; then - purgeCounter=0 - fileCount=$(find ${BACKUP_DIR}/oplog_005 -mmin +${retention_minute} -name "*.lz4" | wc -l) - find ${BACKUP_DIR}/oplog_005 -mmin +${retention_minute} -name "*.lz4" -exec rm -rf {} \; - if [ ${fileCount} -gt 0 ]; then - echo "clean up expired oplog file successfully, file count: ${fileCount}" - fi + if [ ! -z ${LOGFILE_TTL_SECOND} ];then + purgeCounter=$((purgeCounter+3)) + if [ $purgeCounter -ge 60 ]; then + purgeCounter=0 + fileCount=$(find ${BACKUP_DIR}/oplog_005 -mmin +${retention_minute} -name "*.lz4" | wc -l) + find ${BACKUP_DIR}/oplog_005 -mmin +${retention_minute} -name "*.lz4" -exec rm -rf {} \; + if [ ${fileCount} -gt 0 ]; then + echo "clean up expired oplog file successfully, file count: ${fileCount}" + fi + fi fi } # create oplog push process diff --git a/deploy/postgresql/templates/backuppolicytemplate.yaml b/deploy/postgresql/templates/backuppolicytemplate.yaml index 10554403b97..4584393d91f 100644 --- a/deploy/postgresql/templates/backuppolicytemplate.yaml +++ b/deploy/postgresql/templates/backuppolicytemplate.yaml @@ -33,7 +33,7 @@ spec: cronExpression: "0 18 * * *" logfile: enable: false - cronExpression: "*/5 * * * *" + cronExpression: "*/2 * * * *" snapshot: target: connectionCredentialKey: diff --git a/deploy/postgresql/templates/backuptool.yaml b/deploy/postgresql/templates/backuptool-pgbasebackup.yaml similarity index 67% rename from deploy/postgresql/templates/backuptool.yaml rename to deploy/postgresql/templates/backuptool-pgbasebackup.yaml index c8f91394b65..1029efa4ca4 100644 --- a/deploy/postgresql/templates/backuptool.yaml +++ b/deploy/postgresql/templates/backuptool-pgbasebackup.yaml @@ -9,12 +9,6 @@ spec: image: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.repository }}:{{ .Values.image.tag }} deployKind: job env: - - name: RESTORE_DATA_DIR - value: /home/postgres/pgdata/kb_restore - - name: TMP_DATA_DIR - value: /home/postgres/pgdata/kb_restore/tmp_data - - name: TMP_ARCH_DATA_DIR - value: /home/postgres/pgdata/kb_restore/arch - name: DATA_DIR value: /home/postgres/pgdata/pgroot/data physical: @@ -26,8 +20,8 @@ spec: set -e; cd ${BACKUP_DIR}; mkdir -p ${DATA_DIR}; - tar -xvf base.tar.gz -C ${DATA_DIR}/; - tar -xvf pg_wal.tar.gz -C ${DATA_DIR}/pg_wal/; + tar -xvf base.tar -C ${DATA_DIR}/; + tar -xvf pg_wal.tar -C ${DATA_DIR}/pg_wal/; echo "done!"; incrementalRestoreCommands: [] logical: @@ -36,9 +30,12 @@ spec: backupCommands: - sh - -c - - > + - | set -e; + if [ -d ${BACKUP_DIR} ]; then + rm -rf ${BACKUP_DIR} + fi mkdir -p ${BACKUP_DIR}; - echo ${DB_PASSWORD} | pg_basebackup -Ft -Pv -Xs -z -D ${BACKUP_DIR} -Z5 -h ${DB_HOST} -U standby -W; + echo ${DB_PASSWORD} | pg_basebackup -Ft -Pv -c fast -Xs -D ${BACKUP_DIR} -h ${DB_HOST} -U standby -W; echo "TOTAL SIZE: $(du -shx ${BACKUP_DIR}|awk '{print $1}')" > ${DATA_DIR}/basebackup.info; incrementalBackupCommands: [] diff --git a/deploy/postgresql/templates/backuptool-pitr.yaml b/deploy/postgresql/templates/backuptool-pitr.yaml index 134c5128119..c6114e73189 100644 --- a/deploy/postgresql/templates/backuptool-pitr.yaml +++ b/deploy/postgresql/templates/backuptool-pitr.yaml @@ -19,8 +19,6 @@ spec: value: "$(VOLUME_DATA_DIR)/pgroot/data" - name: CONF_DIR value: "$(VOLUME_DATA_DIR)/conf" - - name: RECOVERY_TIME - value: $KB_RECOVERY_TIME - name: TIME_FORMAT value: 2006-01-02 15:04:05 MST - name: LOG_DIR @@ -43,6 +41,7 @@ spec: if [ -d ${DATA_DIR}.old ]; then echo "${DATA_DIR}.old directory already exists, skip restore."; exit 0; fi mkdir -p ${PITR_DIR}; cd ${PITR_DIR} + # TODO Obtain logs directly from WAL archive。 for i in $(find ${BACKUP_DIR} -name "*.gz"); do echo "copying ${i}"; cp ${i} $(basename $i); @@ -58,7 +57,7 @@ spec: echo "sync;" >> ${RESTORE_SCRIPT_DIR}/kb_restore.sh; chmod +x ${RESTORE_SCRIPT_DIR}/kb_restore.sh; echo "restore_command='case "%f" in *history) cp ${PITR_DIR}/%f %p ;; *) mv ${PITR_DIR}/%f %p ;; esac'" > ${CONF_DIR}/recovery.conf; - echo "recovery_target_time='${RECOVERY_TIME}'" >> ${CONF_DIR}/recovery.conf; + echo "recovery_target_time='${KB_RECOVERY_TIME}'" >> ${CONF_DIR}/recovery.conf; echo "recovery_target_action='promote'" >> ${CONF_DIR}/recovery.conf; echo "recovery_target_timeline='latest'" >> ${CONF_DIR}/recovery.conf; mv ${DATA_DIR} ${DATA_DIR}.old; @@ -69,8 +68,11 @@ spec: - -c - | set -e; - EXPIRED_INCR_LOG=${BACKUP_DIR}/$(date -d"7 day ago" +%Y%m%d); - if [ -d ${EXPIRED_INCR_LOG} ]; then rm -rf ${EXPIRED_INCR_LOG}; fi + if [ ! -z ${LOGFILE_TTL_SECOND} ];then + retention_day=$((${LOGFILE_TTL_SECOND}/86400)) + EXPIRED_INCR_LOG=${BACKUP_DIR}/$(date -d"${retention_day} day ago" +%Y%m%d); + if [ -d ${EXPIRED_INCR_LOG} ]; then rm -rf ${EXPIRED_INCR_LOG}; fi + fi export PGPASSWORD=${DB_PASSWORD} PSQL="psql -h ${DB_HOST} -U ${DB_USER}" LAST_TRANS=$(pg_waldump $(${PSQL} -Atc "select pg_walfile_name(pg_current_wal_lsn())") --rmgr=Transaction |tail -n 1) diff --git a/deploy/postgresql/templates/backuptool-wal-g.yaml b/deploy/postgresql/templates/backuptool-wal-g.yaml new file mode 100644 index 00000000000..75e2d057dc5 --- /dev/null +++ b/deploy/postgresql/templates/backuptool-wal-g.yaml @@ -0,0 +1,49 @@ +apiVersion: dataprotection.kubeblocks.io/v1alpha1 +kind: BackupTool +metadata: + name: postgres-wal-g + labels: + clusterdefinition.kubeblocks.io/name: postgresql +spec: + image: registry.cn-hangzhou.aliyuncs.com/apecloud/spilo:14.7.2 + deployKind: job + env: + - name: DATA_DIR + value: /home/postgres/pgdata/pgroot/data + - name: WAL_DIR + value: $(DATA_DIR)/pg_wal + - name: WALG_PG_WAL_SIZE + value: "16" + - name: WALG_TAR_SIZE_THRESHOLD + value: "4294967296" + - name: WALG_UPLOAD_DISK_CONCURRENCY + value: "8" + physical: + restoreCommands: + - bash + - -c + - | + #!/bin/bash + # NOTE: this basebackup only support for pitr. + set -e; + mkdir -p ${DATA_DIR}; + # TODO check KB_RECOVERY_TIME, if not exits, do wal-log restore. + WALG_FILE_PREFIX=${BACKUP_DIR} wal-g backup-fetch ${DATA_DIR} LATEST + echo "done!"; + incrementalRestoreCommands: [] + logical: + restoreCommands: [] + incrementalRestoreCommands: [] + backupCommands: + - bash + - -c + - |- + set -e; + # TODO check if pitr is enabled + if [ -d ${BACKUP_DIR} ]; then + rm -rf ${BACKUP_DIR} + fi + mkdir -p ${BACKUP_DIR} + WALG_FILE_PREFIX=${BACKUP_DIR} PGHOST=${DB_HOST} PGUSER=postgres PGPASSWORD=${DB_PASSWORD} PGPORT=5432 wal-g backup-push ${DATA_DIR} + echo "TOTAL SIZE: $(du -shx ${BACKUP_DIR}|awk '{print $1}')" > ${DATA_DIR}/basebackup.info; + incrementalBackupCommands: [] diff --git a/internal/controller/plan/restore.go b/internal/controller/plan/restore.go index 8b3674593cc..7228a7012a1 100644 --- a/internal/controller/plan/restore.go +++ b/internal/controller/plan/restore.go @@ -331,20 +331,8 @@ func (p *RestoreManager) getRecoveryInfo(baseBackup, logfileBackup *dpv1alpha1.B } // build env of recovery time spec := &backupTool.Spec - timeFormat := time.RFC3339 - envTimeEnvIdx := -1 - for i, env := range spec.Env { - if env.Value == "$KB_RECOVERY_TIME" { - envTimeEnvIdx = i - } else if env.Name == constant.DPTimeFormat { - timeFormat = env.Value - } - } - if envTimeEnvIdx != -1 { - spec.Env[envTimeEnvIdx].Value = p.restoreTime.UTC().Format(timeFormat) - } else { - headEnv = append(headEnv, corev1.EnvVar{Name: constant.DPKBRecoveryTime, Value: p.restoreTime.UTC().Format(timeFormat)}) - } + timeFormat := p.getTimeFormat(spec.Env) + headEnv = append(headEnv, corev1.EnvVar{Name: constant.DPKBRecoveryTime, Value: p.restoreTime.UTC().Format(timeFormat)}) headEnv = append(headEnv, corev1.EnvVar{Name: constant.DPKBRecoveryTimestamp, Value: strconv.FormatInt(p.restoreTime.Unix(), 10)}) // build env of backup startTime and user contexts if baseBackup.Status.Manifests != nil { @@ -576,19 +564,16 @@ func (p *RestoreManager) BuildDatafileRestoreJobByPVCS(synthesizedComponent *com } // builds env - backupDataPath := fmt.Sprintf("/%s/%s", backup.Name, backup.Namespace) - manifests := backup.Status.Manifests - if manifests != nil && manifests.BackupTool != nil { - backupDataPath = fmt.Sprintf("/%s%s", backup.Name, manifests.BackupTool.FilePath) - } env := []corev1.EnvVar{ { - Name: "BACKUP_NAME", + Name: constant.DPBackupName, Value: backup.Name, - }, { - Name: "BACKUP_DIR", - Value: backupDataPath, - }} + }, + } + manifests := backup.Status.Manifests + if manifests != nil && manifests.BackupTool != nil { + env = append(env, corev1.EnvVar{Name: constant.DPBackupDIR, Value: fmt.Sprintf("/%s%s", backup.Name, manifests.BackupTool.FilePath)}) + } // merges env from backup tool. env = append(env, backupTool.Spec.Env...) objs = make([]client.Object, 0) @@ -826,3 +811,12 @@ func (p *RestoreManager) BuildCommonLabels(synthesizedComponent *component.Synth constant.KBAppComponentLabelKey: synthesizedComponent.Name, } } + +func (p *RestoreManager) getTimeFormat(envs []corev1.EnvVar) string { + for _, env := range envs { + if env.Name == constant.DPTimeFormat { + return env.Value + } + } + return time.RFC3339 +}