diff --git a/e2e-tests/functions b/e2e-tests/functions index 5e9c9e777c..5ea96e0d31 100755 --- a/e2e-tests/functions +++ b/e2e-tests/functions @@ -1449,29 +1449,54 @@ start_cluster() { get_latest_restorable_time() { local cluster=$1 - local timestamp - local timeout + local first_timestamp + local second_timestamp + local retry=0 - until [[ $timestamp != "" && $timestamp != "null" ]]; do + # "pbm-agent status" can return different timestamp in first few seconds + # we need to get it twice to be sure that timestamp was not changed + until [[ $first_timestamp != "" && $first_timestamp != "null" && $first_timestamp == $second_timestamp ]]; do sleep 5 - timestamp=$(kubectl exec "$cluster-0" -c backup-agent -- pbm status -o json | jq '.backups.pitrChunks.pitrChunks | last | .range.end') - timeout=$((timeout + 5)) - if [[ $timeout -gt 30 ]]; then + if [[ $first_timestamp != "" && $first_timestamp != "null" ]]; then + second_timestamp=$(kubectl_bin exec "$cluster-0" -c backup-agent -- pbm status -o json | jq '.backups.pitrChunks.pitrChunks | last | .range.end') + else + first_timestamp=$(kubectl_bin exec "$cluster-0" -c backup-agent -- pbm status -o json | jq '.backups.pitrChunks.pitrChunks | last | .range.end') + fi + let retry+=1 + if [[ $retry -gt 30 ]]; then echo "Error: timeout waiting for timestamp" exit 1 fi done - $date -u -d @"$timestamp" "+%Y-%m-%dT%H:%M:%SZ" + $date -u -d @"$first_timestamp" "+%Y-%m-%dT%H:%M:%SZ" } +get_latest_restorable_time_from_backup_object() { + local backup_name=$1 + local latestRestorableTime + local retry=0 + + until [[ $latestRestorableTime != "" && $latestRestorableTime != "null" ]]; do + sleep 5 + latestRestorableTime=$(kubectl_bin get psmdb-backup "$backup_name" -o jsonpath='{.status.latestRestorableTime}') + let retry+=1 + if [[ $retry -gt 30 ]]; then + echo "Error: timeout waiting for latestRestorableTime" + exit 1 + fi + done + + echo "$latestRestorableTime" +} compare_latest_restorable_time() { local cluster=$1 local backup_name=$2 local latest_restorable_time - latest_restorable_time=$(get_latest_restorable_time "$cluster") local backup_time - backup_time=$(kubectl get psmdb-backup "$backup_name" -o jsonpath='{.status.latestRestorableTime}') + + latest_restorable_time=$(get_latest_restorable_time "$cluster") + backup_time=$(get_latest_restorable_time_from_backup_object "$backup_name") if [[ $latest_restorable_time != "$backup_time" ]]; then echo "Error: latestRestorableTime is not equal to the latest timestamp of the backup $backup_name: $latest_restorable_time != $backup_time" diff --git a/e2e-tests/pitr-sharded/run b/e2e-tests/pitr-sharded/run index bdf68166fc..94638aed9a 100755 --- a/e2e-tests/pitr-sharded/run +++ b/e2e-tests/pitr-sharded/run @@ -64,7 +64,7 @@ check_recovery() { | if [ -z "$restore_date" ]; then $sed -e "/date:/d"; else $sed -e "s/date:/date: $restore_date/"; fi \ | kubectl_bin apply -f - - wait_restore "$backup_name" "$cluster_name" + wait_restore "$backup_name" "$cluster_name" "ready" 1 echo set -o xtrace @@ -104,7 +104,7 @@ main() { desc 'check if all 3 Pods started' wait_for_running $cluster-rs0 3 wait_for_running $cluster-cfg 3 "false" - sleep 10 + wait_cluster_consistency $cluster desc 'check if statefulset created with expected config' compare_kubectl statefulset/$cluster-rs0