Merge branch 'main' into K8SPSMDB-985-correct-pbm-nodes

e2e-tests/functions

@@ -1449,29 +1449,54 @@ start_cluster() {
 
 get_latest_restorable_time() {
 	local cluster=$1
-	local timestamp
-	local timeout
+	local first_timestamp
+	local second_timestamp
+	local retry=0
 
-	until [[ $timestamp != "" && $timestamp != "null" ]]; do
+	# "pbm-agent status" can return different timestamp in first few seconds
+	# we need to get it twice to be sure that timestamp was not changed
+	until [[ $first_timestamp != "" && $first_timestamp != "null" && $first_timestamp == $second_timestamp ]]; do
 		sleep 5
-		timestamp=$(kubectl exec "$cluster-0" -c backup-agent -- pbm status -o json | jq '.backups.pitrChunks.pitrChunks | last | .range.end')
-		timeout=$((timeout + 5))
-		if [[ $timeout -gt 30 ]]; then
+		if [[ $first_timestamp != "" && $first_timestamp != "null" ]]; then
+			second_timestamp=$(kubectl_bin exec "$cluster-0" -c backup-agent -- pbm status -o json | jq '.backups.pitrChunks.pitrChunks | last | .range.end')
+		else
+			first_timestamp=$(kubectl_bin exec "$cluster-0" -c backup-agent -- pbm status -o json | jq '.backups.pitrChunks.pitrChunks | last | .range.end')
+		fi
+		let retry+=1
+		if [[ $retry -gt 30 ]]; then
 			echo "Error: timeout waiting for timestamp"
 			exit 1
 		fi
 	done
 
-	$date -u -d @"$timestamp" "+%Y-%m-%dT%H:%M:%SZ"
+	$date -u -d @"$first_timestamp" "+%Y-%m-%dT%H:%M:%SZ"
 }
 
+get_latest_restorable_time_from_backup_object() {
+	local backup_name=$1
+	local latestRestorableTime
+	local retry=0
+
+	until [[ $latestRestorableTime != "" && $latestRestorableTime != "null" ]]; do
+		sleep 5
+		latestRestorableTime=$(kubectl_bin get psmdb-backup "$backup_name" -o jsonpath='{.status.latestRestorableTime}')
+		let retry+=1
+		if [[ $retry -gt 30 ]]; then
+			echo "Error: timeout waiting for latestRestorableTime"
+			exit 1
+		fi
+	done
+
+	echo "$latestRestorableTime"
+}
 compare_latest_restorable_time() {
 	local cluster=$1
 	local backup_name=$2
 	local latest_restorable_time
-	latest_restorable_time=$(get_latest_restorable_time "$cluster")
 	local backup_time
-	backup_time=$(kubectl get psmdb-backup "$backup_name" -o jsonpath='{.status.latestRestorableTime}')
+
+	latest_restorable_time=$(get_latest_restorable_time "$cluster")
+	backup_time=$(get_latest_restorable_time_from_backup_object "$backup_name")
 
 	if [[ $latest_restorable_time != "$backup_time" ]]; then
 		echo "Error: latestRestorableTime is not equal to the latest timestamp of the backup $backup_name: $latest_restorable_time != $backup_time"

e2e-tests/pitr-sharded/run

@@ -64,7 +64,7 @@ check_recovery() {
 		| if [ -z "$restore_date" ]; then $sed -e "/date:/d"; else $sed -e "s/date:/date: $restore_date/"; fi \
 		| kubectl_bin apply -f -
 
-	wait_restore "$backup_name" "$cluster_name"
+	wait_restore "$backup_name" "$cluster_name" "ready" 1
 	echo
 	set -o xtrace
 
@@ -104,7 +104,7 @@ main() {
 	desc 'check if all 3 Pods started'
 	wait_for_running $cluster-rs0 3
 	wait_for_running $cluster-cfg 3 "false"
-	sleep 10
+	wait_cluster_consistency $cluster
 
 	desc 'check if statefulset created with expected config'
 	compare_kubectl statefulset/$cluster-rs0