diff --git a/test/coordination_SUITE.erl b/test/coordination_SUITE.erl index 557233ec..aa88d262 100644 --- a/test/coordination_SUITE.erl +++ b/test/coordination_SUITE.erl @@ -319,7 +319,7 @@ grow_cluster(Config) -> {ok, _, L1} = ra:members(A), [A, B, C] = rpc:call(ANode, ra_leaderboard, lookup_members, [ClusterName]), L1 = rpc:call(ANode, ra_leaderboard, lookup_leader, [ClusterName]), - %% TODO: handle race conditions + await_condition( fun () -> [A, B, C] == rpc:call(BNode, ra_leaderboard, lookup_members, [ClusterName]) andalso @@ -332,8 +332,12 @@ grow_cluster(Config) -> end, 20), ok = ra:leave_and_delete_server(?SYS, A, A), - {ok, _, _} = ra:process_command(B, banana), - {ok, _, L2} = ra:members(B), + %% wait for B to process the cluster change + await_condition( + fun () -> + [B, C] == rpc:call(CNode, ra_leaderboard, lookup_members, [ClusterName]) + end, 20), + {ok, _, L2} = ra:process_command(B, banana), %% check members [B, C] = rpc:call(BNode, ra_leaderboard, lookup_members, [ClusterName]), @@ -623,12 +627,6 @@ key_metrics(Config) -> CI > 0, StoppedMetrics), ok = ra:restart_server(?SYS, TestId), - await_condition( - fun () -> - Metrics = ra:key_metrics(TestId), - ct:pal("RecoverMetrics ~p", [Metrics]), - recover == maps:get(state, Metrics) - end, 200), {ok, _, _} = ra:process_command(Leader, {data, Data}), await_condition( fun () -> diff --git a/test/ra_2_SUITE.erl b/test/ra_2_SUITE.erl index 3ad77d01..817dd292 100644 --- a/test/ra_2_SUITE.erl +++ b/test/ra_2_SUITE.erl @@ -299,9 +299,9 @@ cluster_is_deleted_with_server_down(Config) -> % start node again ra:restart_server(?SYS, ServerId3), % validate all nodes have been shut down and terminated - ok = validate_process_down(element(1, ServerId1), 10), - ok = validate_process_down(element(1, ServerId2), 10), - ok = validate_process_down(element(1, ServerId3), 10), + ok = validate_process_down(element(1, ServerId1), 50), + ok = validate_process_down(element(1, ServerId2), 50), + ok = validate_process_down(element(1, ServerId3), 50), % validate there are no data dirs anymore [ begin @@ -814,10 +814,11 @@ init(_) -> {State0, ok} end. -state_enter(eol, State) -> +state_enter(eol = S, State) -> + ct:pal("state_enter ~w ~w", [self(), S]), [{send_msg, P, eol, ra_event} || {P, _} <- queue:to_list(State), is_pid(P)]; state_enter(S, _) -> - ct:pal("state_enter ~w", [S]), + ct:pal("state_enter ~w ~w", [self(), S]), []. flush() -> diff --git a/test/ra_SUITE.erl b/test/ra_SUITE.erl index b3a71584..d6704050 100644 --- a/test/ra_SUITE.erl +++ b/test/ra_SUITE.erl @@ -551,17 +551,24 @@ members_info(Config) -> {ok, _, _} = ra:add_member(Leader, CSpec), {ok, 9, Leader} = ra:consistent_query(C, fun(S) -> S end), ?assertMatch({ok, - #{Follower := #{status := normal, query_index := QI, - next_index := NI, match_index := MI, + #{Follower := #{status := normal, + query_index := QI, + next_index := NI, + match_index := MI, commit_index_sent := MI, voter_status := #{membership := voter}}, - Leader := #{status := normal, query_index := QI, - next_index := NI, match_index := MI, + Leader := #{status := normal, + query_index := QI, + next_index := NI, + match_index := MI, voter_status := #{membership := voter}}, - C := #{status := normal, query_index := QI, - next_index := NI, match_index := 0, + C := #{status := normal, + query_index := _, + next_index := NI, + match_index := 0, commit_index_sent := MI, - voter_status := #{membership := promotable, target := MI}}}, + voter_status := #{membership := promotable, + target := MI}}}, Leader}, ra:members_info(Follower)), ?assertMatch({ok, #{A := #{}, diff --git a/test/ra_log_wal_SUITE.erl b/test/ra_log_wal_SUITE.erl index fc369976..0101537d 100644 --- a/test/ra_log_wal_SUITE.erl +++ b/test/ra_log_wal_SUITE.erl @@ -863,6 +863,7 @@ checksum_failure_in_middle_of_file_should_fail(Config) -> {error, wal_checksum_validation_failure} = ra_log_wal:start_link(Conf), empty_mailbox(), + meck:unload(), ok. empty_mailbox() ->