From f38dba841528cab1bd36a955447bb5733ca1ed9b Mon Sep 17 00:00:00 2001 From: zane-neo Date: Sun, 29 Sep 2024 14:01:45 +0800 Subject: [PATCH 1/9] fix cluster not able to spin up issue when disk usage exceeds threshold Signed-off-by: zane-neo --- CHANGELOG.md | 2 ++ server/src/main/java/org/opensearch/bootstrap/Bootstrap.java | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 73db9bb9ed7af..c581f9ae8811e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -83,6 +83,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Fix protobuf-java leak through client library dependencies ([#16254](https://github.com/opensearch-project/OpenSearch/pull/16254)) - Fix multi-search with template doesn't return status code ([#16265](https://github.com/opensearch-project/OpenSearch/pull/16265)) - Fix wrong default value when setting `index.number_of_routing_shards` to null on index creation ([#16331](https://github.com/opensearch-project/OpenSearch/pull/16331)) +- Fix disk usage exceeds threshold cluster can't spin up issue ([#15258](https://github.com/opensearch-project/OpenSearch/pull/15258))) + ### Security diff --git a/server/src/main/java/org/opensearch/bootstrap/Bootstrap.java b/server/src/main/java/org/opensearch/bootstrap/Bootstrap.java index 4e167d10b99fa..3ff8a2439cde4 100644 --- a/server/src/main/java/org/opensearch/bootstrap/Bootstrap.java +++ b/server/src/main/java/org/opensearch/bootstrap/Bootstrap.java @@ -336,8 +336,8 @@ private static Environment createEnvironment( } private void start() throws NodeValidationException { - node.start(); keepAliveThread.start(); + node.start(); } static void stop() throws IOException { From 5dbdab907365b6f9745ff1eaf1681ee0e89f9d8c Mon Sep 17 00:00:00 2001 From: zane-neo Date: Tue, 8 Oct 2024 09:46:03 +0800 Subject: [PATCH 2/9] Add comment to changes Signed-off-by: zane-neo --- server/src/main/java/org/opensearch/bootstrap/Bootstrap.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/src/main/java/org/opensearch/bootstrap/Bootstrap.java b/server/src/main/java/org/opensearch/bootstrap/Bootstrap.java index 3ff8a2439cde4..0cf71ca51890f 100644 --- a/server/src/main/java/org/opensearch/bootstrap/Bootstrap.java +++ b/server/src/main/java/org/opensearch/bootstrap/Bootstrap.java @@ -336,6 +336,8 @@ private static Environment createEnvironment( } private void start() throws NodeValidationException { + // keepAliveThread should start first than node to ensure the cluster can spin up successfully in edge cases: + // https://github.com/opensearch-project/OpenSearch/issues/14791 keepAliveThread.start(); node.start(); } From 4fcd310597e4d95ff5f113375bff8d3402937c4a Mon Sep 17 00:00:00 2001 From: zane-neo Date: Thu, 10 Oct 2024 11:15:18 +0800 Subject: [PATCH 3/9] Add UT to ensure the keepAliveThread starts before node starts Signed-off-by: zane-neo --- .../opensearch/bootstrap/BootstrapTests.java | 57 +++++++++++++++++++ .../org/opensearch/bootstrap/Bootstrap.java | 6 +- .../org/opensearch/bootstrap/security.policy | 3 + 3 files changed, 65 insertions(+), 1 deletion(-) diff --git a/distribution/tools/keystore-cli/src/test/java/org/opensearch/bootstrap/BootstrapTests.java b/distribution/tools/keystore-cli/src/test/java/org/opensearch/bootstrap/BootstrapTests.java index e9219de218aef..1796143a35210 100644 --- a/distribution/tools/keystore-cli/src/test/java/org/opensearch/bootstrap/BootstrapTests.java +++ b/distribution/tools/keystore-cli/src/test/java/org/opensearch/bootstrap/BootstrapTests.java @@ -31,6 +31,11 @@ package org.opensearch.bootstrap; +import org.mockito.InOrder; +import org.opensearch.Version; +import org.opensearch.cluster.node.DiscoveryNode; +import org.opensearch.cluster.node.DiscoveryNodeRole; +import org.opensearch.common.UUIDs; import org.opensearch.common.settings.KeyStoreCommandTestCase; import org.opensearch.common.settings.KeyStoreWrapper; import org.opensearch.common.settings.SecureSettings; @@ -38,6 +43,10 @@ import org.opensearch.common.util.io.IOUtils; import org.opensearch.core.common.settings.SecureString; import org.opensearch.env.Environment; +import org.opensearch.node.MockNode; +import org.opensearch.node.Node; +import org.opensearch.plugins.Plugin; +import org.opensearch.test.MockHttpTransport; import org.opensearch.test.OpenSearchTestCase; import org.junit.After; import org.junit.Before; @@ -45,14 +54,22 @@ import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; +import java.lang.reflect.Field; import java.nio.charset.StandardCharsets; import java.nio.file.FileSystem; import java.nio.file.Files; import java.nio.file.Path; +import java.security.AccessController; +import java.security.PrivilegedAction; import java.util.ArrayList; import java.util.List; +import java.util.concurrent.CountDownLatch; import static org.hamcrest.Matchers.equalTo; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.inOrder; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; public class BootstrapTests extends OpenSearchTestCase { Environment env; @@ -131,4 +148,44 @@ private void assertPassphraseRead(String source, String expected) { } } + + public void testInitExecutionOrder() { + AccessController.doPrivileged((PrivilegedAction) () -> { + try { + // Create mock objects for Thread and Node + Thread mockThread = mock(Thread.class); + Node mockNode = mock(Node.class); + + // Use reflection to set the INSTANCE to null before the test + Field instanceField = Bootstrap.class.getDeclaredField("INSTANCE"); + instanceField.setAccessible(true); + instanceField.set(null, null); + + // Use reflection to replace the private fields with our mocks + Bootstrap bootstrap = new Bootstrap(); + Field threadField = Bootstrap.class.getDeclaredField("keepAliveThread"); + threadField.setAccessible(true); + threadField.set(bootstrap, mockThread); + + Field nodeField = Bootstrap.class.getDeclaredField("node"); + nodeField.setAccessible(true); + nodeField.set(bootstrap, mockNode); + + // Set the INSTANCE to our modified bootstrap + instanceField.set(null, bootstrap); + + // Call the startInstance method + Bootstrap.startInstance(bootstrap); + + // Verify the order of execution + InOrder inOrder = inOrder(mockThread, mockNode); + inOrder.verify(mockThread).start(); + inOrder.verify(mockNode).start(); + } catch (Exception e) { + fail(e.getMessage()); + } + return null; + }); + + } } diff --git a/server/src/main/java/org/opensearch/bootstrap/Bootstrap.java b/server/src/main/java/org/opensearch/bootstrap/Bootstrap.java index 0cf71ca51890f..093b699679455 100644 --- a/server/src/main/java/org/opensearch/bootstrap/Bootstrap.java +++ b/server/src/main/java/org/opensearch/bootstrap/Bootstrap.java @@ -412,7 +412,7 @@ static void init(final boolean foreground, final Path pidFile, final boolean qui throw new BootstrapException(e); } - INSTANCE.start(); + startInstance(INSTANCE); // We don't close stderr if `--quiet` is passed, because that // hides fatal startup errors. For example, if OpenSearch is @@ -464,6 +464,10 @@ static void init(final boolean foreground, final Path pidFile, final boolean qui } } + static void startInstance(Bootstrap instance) throws NodeValidationException { + instance.start(); + } + @SuppressForbidden(reason = "System#out") private static void closeSystOut() { System.out.close(); diff --git a/server/src/main/resources/org/opensearch/bootstrap/security.policy b/server/src/main/resources/org/opensearch/bootstrap/security.policy index 22e445f7d9022..78494061bebad 100644 --- a/server/src/main/resources/org/opensearch/bootstrap/security.policy +++ b/server/src/main/resources/org/opensearch/bootstrap/security.policy @@ -165,6 +165,9 @@ grant { // otherwise can be provided only to test libraries permission java.lang.RuntimePermission "fileSystemProvider"; + // needed for UT test in BootstrapTests + permission java.lang.RuntimePermission "shutdownHooks"; + // needed by jvminfo for monitoring the jvm permission java.lang.management.ManagementPermission "monitor"; From 2ad0126785ec3e8a742ef670c2a4f164ead690b1 Mon Sep 17 00:00:00 2001 From: zane-neo Date: Thu, 10 Oct 2024 11:16:43 +0800 Subject: [PATCH 4/9] remove unused imports Signed-off-by: zane-neo --- .../java/org/opensearch/bootstrap/BootstrapTests.java | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/distribution/tools/keystore-cli/src/test/java/org/opensearch/bootstrap/BootstrapTests.java b/distribution/tools/keystore-cli/src/test/java/org/opensearch/bootstrap/BootstrapTests.java index 1796143a35210..ba8bba1180b83 100644 --- a/distribution/tools/keystore-cli/src/test/java/org/opensearch/bootstrap/BootstrapTests.java +++ b/distribution/tools/keystore-cli/src/test/java/org/opensearch/bootstrap/BootstrapTests.java @@ -32,10 +32,6 @@ package org.opensearch.bootstrap; import org.mockito.InOrder; -import org.opensearch.Version; -import org.opensearch.cluster.node.DiscoveryNode; -import org.opensearch.cluster.node.DiscoveryNodeRole; -import org.opensearch.common.UUIDs; import org.opensearch.common.settings.KeyStoreCommandTestCase; import org.opensearch.common.settings.KeyStoreWrapper; import org.opensearch.common.settings.SecureSettings; @@ -43,10 +39,7 @@ import org.opensearch.common.util.io.IOUtils; import org.opensearch.core.common.settings.SecureString; import org.opensearch.env.Environment; -import org.opensearch.node.MockNode; import org.opensearch.node.Node; -import org.opensearch.plugins.Plugin; -import org.opensearch.test.MockHttpTransport; import org.opensearch.test.OpenSearchTestCase; import org.junit.After; import org.junit.Before; @@ -63,13 +56,10 @@ import java.security.PrivilegedAction; import java.util.ArrayList; import java.util.List; -import java.util.concurrent.CountDownLatch; import static org.hamcrest.Matchers.equalTo; -import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.inOrder; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.spy; public class BootstrapTests extends OpenSearchTestCase { Environment env; From acebaf34b86d40f24905a78cf1fb0eea5fc42b44 Mon Sep 17 00:00:00 2001 From: zane-neo Date: Thu, 10 Oct 2024 15:56:01 +0800 Subject: [PATCH 5/9] Fix forbidden API calls check failed issue Signed-off-by: zane-neo --- .../opensearch/bootstrap/BootstrapTests.java | 72 ++++++++----------- .../org/opensearch/bootstrap/Bootstrap.java | 11 +++ .../org/opensearch/bootstrap/security.policy | 3 - 3 files changed, 42 insertions(+), 44 deletions(-) diff --git a/distribution/tools/keystore-cli/src/test/java/org/opensearch/bootstrap/BootstrapTests.java b/distribution/tools/keystore-cli/src/test/java/org/opensearch/bootstrap/BootstrapTests.java index ba8bba1180b83..69ee30d26733a 100644 --- a/distribution/tools/keystore-cli/src/test/java/org/opensearch/bootstrap/BootstrapTests.java +++ b/distribution/tools/keystore-cli/src/test/java/org/opensearch/bootstrap/BootstrapTests.java @@ -31,7 +31,10 @@ package org.opensearch.bootstrap; -import org.mockito.InOrder; +import org.mockito.MockedStatic; +import org.mockito.Mockito; +import org.opensearch.cli.UserException; +import org.opensearch.common.logging.LogConfigurator; import org.opensearch.common.settings.KeyStoreCommandTestCase; import org.opensearch.common.settings.KeyStoreWrapper; import org.opensearch.common.settings.SecureSettings; @@ -40,6 +43,7 @@ import org.opensearch.core.common.settings.SecureString; import org.opensearch.env.Environment; import org.opensearch.node.Node; +import org.opensearch.node.NodeValidationException; import org.opensearch.test.OpenSearchTestCase; import org.junit.After; import org.junit.Before; @@ -47,19 +51,21 @@ import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; -import java.lang.reflect.Field; import java.nio.charset.StandardCharsets; import java.nio.file.FileSystem; import java.nio.file.Files; import java.nio.file.Path; -import java.security.AccessController; -import java.security.PrivilegedAction; import java.util.ArrayList; import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; import static org.hamcrest.Matchers.equalTo; -import static org.mockito.Mockito.inOrder; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; public class BootstrapTests extends OpenSearchTestCase { Environment env; @@ -138,44 +144,28 @@ private void assertPassphraseRead(String source, String expected) { } } + public void testInitExecutionOrder() throws Exception { + AtomicInteger order = new AtomicInteger(0); - public void testInitExecutionOrder() { - AccessController.doPrivileged((PrivilegedAction) () -> { - try { - // Create mock objects for Thread and Node - Thread mockThread = mock(Thread.class); - Node mockNode = mock(Node.class); - - // Use reflection to set the INSTANCE to null before the test - Field instanceField = Bootstrap.class.getDeclaredField("INSTANCE"); - instanceField.setAccessible(true); - instanceField.set(null, null); - - // Use reflection to replace the private fields with our mocks - Bootstrap bootstrap = new Bootstrap(); - Field threadField = Bootstrap.class.getDeclaredField("keepAliveThread"); - threadField.setAccessible(true); - threadField.set(bootstrap, mockThread); - - Field nodeField = Bootstrap.class.getDeclaredField("node"); - nodeField.setAccessible(true); - nodeField.set(bootstrap, mockNode); - - // Set the INSTANCE to our modified bootstrap - instanceField.set(null, bootstrap); - - // Call the startInstance method - Bootstrap.startInstance(bootstrap); - - // Verify the order of execution - InOrder inOrder = inOrder(mockThread, mockNode); - inOrder.verify(mockThread).start(); - inOrder.verify(mockNode).start(); - } catch (Exception e) { - fail(e.getMessage()); - } - return null; + Thread mockThread = new Thread(() -> { + assertEquals(0, order.getAndIncrement()); }); + Node mockNode = mock(Node.class); + doAnswer(invocation -> { + assertEquals(1, order.getAndIncrement()); + return null; + }).when(mockNode).start(); + + LogConfigurator.registerErrorListener(); + Bootstrap testBootstrap = new Bootstrap(mockThread, mockNode); + testBootstrap.setInstance(testBootstrap); + + Bootstrap.startInstance(testBootstrap); + + verify(mockNode).start(); + assertEquals(2, order.get()); } + + } diff --git a/server/src/main/java/org/opensearch/bootstrap/Bootstrap.java b/server/src/main/java/org/opensearch/bootstrap/Bootstrap.java index 093b699679455..e085adf2a01da 100644 --- a/server/src/main/java/org/opensearch/bootstrap/Bootstrap.java +++ b/server/src/main/java/org/opensearch/bootstrap/Bootstrap.java @@ -93,6 +93,17 @@ final class Bootstrap { private final Thread keepAliveThread; private final Spawner spawner = new Spawner(); + // For testing purpose + void setInstance(Bootstrap bootstrap) { + INSTANCE = bootstrap; + } + + // For testing purpose + Bootstrap(Thread keepAliveThread, Node node) { + this.keepAliveThread = keepAliveThread; + this.node = node; + } + /** creates a new instance */ Bootstrap() { keepAliveThread = new Thread(new Runnable() { diff --git a/server/src/main/resources/org/opensearch/bootstrap/security.policy b/server/src/main/resources/org/opensearch/bootstrap/security.policy index 78494061bebad..22e445f7d9022 100644 --- a/server/src/main/resources/org/opensearch/bootstrap/security.policy +++ b/server/src/main/resources/org/opensearch/bootstrap/security.policy @@ -165,9 +165,6 @@ grant { // otherwise can be provided only to test libraries permission java.lang.RuntimePermission "fileSystemProvider"; - // needed for UT test in BootstrapTests - permission java.lang.RuntimePermission "shutdownHooks"; - // needed by jvminfo for monitoring the jvm permission java.lang.management.ManagementPermission "monitor"; From c29a02befc83fa6636db30ab32659bbb7e25acb9 Mon Sep 17 00:00:00 2001 From: zane-neo Date: Thu, 10 Oct 2024 16:17:56 +0800 Subject: [PATCH 6/9] format code Signed-off-by: zane-neo --- .../test/java/org/opensearch/bootstrap/BootstrapTests.java | 7 ------- 1 file changed, 7 deletions(-) diff --git a/distribution/tools/keystore-cli/src/test/java/org/opensearch/bootstrap/BootstrapTests.java b/distribution/tools/keystore-cli/src/test/java/org/opensearch/bootstrap/BootstrapTests.java index 69ee30d26733a..9ad1d04ecc0c1 100644 --- a/distribution/tools/keystore-cli/src/test/java/org/opensearch/bootstrap/BootstrapTests.java +++ b/distribution/tools/keystore-cli/src/test/java/org/opensearch/bootstrap/BootstrapTests.java @@ -31,9 +31,6 @@ package org.opensearch.bootstrap; -import org.mockito.MockedStatic; -import org.mockito.Mockito; -import org.opensearch.cli.UserException; import org.opensearch.common.logging.LogConfigurator; import org.opensearch.common.settings.KeyStoreCommandTestCase; import org.opensearch.common.settings.KeyStoreWrapper; @@ -43,7 +40,6 @@ import org.opensearch.core.common.settings.SecureString; import org.opensearch.env.Environment; import org.opensearch.node.Node; -import org.opensearch.node.NodeValidationException; import org.opensearch.test.OpenSearchTestCase; import org.junit.After; import org.junit.Before; @@ -60,12 +56,9 @@ import java.util.concurrent.atomic.AtomicInteger; import static org.hamcrest.Matchers.equalTo; -import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doAnswer; -import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; public class BootstrapTests extends OpenSearchTestCase { Environment env; From 2b3cd47b0f1ab32e4ef3175c4b83cd119a478dd3 Mon Sep 17 00:00:00 2001 From: zane-neo Date: Thu, 10 Oct 2024 16:49:51 +0800 Subject: [PATCH 7/9] format code Signed-off-by: zane-neo --- .../test/java/org/opensearch/bootstrap/BootstrapTests.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/distribution/tools/keystore-cli/src/test/java/org/opensearch/bootstrap/BootstrapTests.java b/distribution/tools/keystore-cli/src/test/java/org/opensearch/bootstrap/BootstrapTests.java index 9ad1d04ecc0c1..848a727f5954f 100644 --- a/distribution/tools/keystore-cli/src/test/java/org/opensearch/bootstrap/BootstrapTests.java +++ b/distribution/tools/keystore-cli/src/test/java/org/opensearch/bootstrap/BootstrapTests.java @@ -140,9 +140,7 @@ private void assertPassphraseRead(String source, String expected) { public void testInitExecutionOrder() throws Exception { AtomicInteger order = new AtomicInteger(0); - Thread mockThread = new Thread(() -> { - assertEquals(0, order.getAndIncrement()); - }); + Thread mockThread = new Thread(() -> { assertEquals(0, order.getAndIncrement()); }); Node mockNode = mock(Node.class); doAnswer(invocation -> { @@ -160,5 +158,4 @@ public void testInitExecutionOrder() throws Exception { assertEquals(2, order.get()); } - } From af9d70d59f368605d76213fb27c25c44a710385e Mon Sep 17 00:00:00 2001 From: zane-neo Date: Fri, 11 Oct 2024 09:13:48 +0800 Subject: [PATCH 8/9] change setInstance method to static Signed-off-by: zane-neo --- .../src/test/java/org/opensearch/bootstrap/BootstrapTests.java | 2 +- server/src/main/java/org/opensearch/bootstrap/Bootstrap.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/distribution/tools/keystore-cli/src/test/java/org/opensearch/bootstrap/BootstrapTests.java b/distribution/tools/keystore-cli/src/test/java/org/opensearch/bootstrap/BootstrapTests.java index 848a727f5954f..fb2bc7038f865 100644 --- a/distribution/tools/keystore-cli/src/test/java/org/opensearch/bootstrap/BootstrapTests.java +++ b/distribution/tools/keystore-cli/src/test/java/org/opensearch/bootstrap/BootstrapTests.java @@ -150,7 +150,7 @@ public void testInitExecutionOrder() throws Exception { LogConfigurator.registerErrorListener(); Bootstrap testBootstrap = new Bootstrap(mockThread, mockNode); - testBootstrap.setInstance(testBootstrap); + Bootstrap.setInstance(testBootstrap); Bootstrap.startInstance(testBootstrap); diff --git a/server/src/main/java/org/opensearch/bootstrap/Bootstrap.java b/server/src/main/java/org/opensearch/bootstrap/Bootstrap.java index e085adf2a01da..757e2c9da6e49 100644 --- a/server/src/main/java/org/opensearch/bootstrap/Bootstrap.java +++ b/server/src/main/java/org/opensearch/bootstrap/Bootstrap.java @@ -94,7 +94,7 @@ final class Bootstrap { private final Spawner spawner = new Spawner(); // For testing purpose - void setInstance(Bootstrap bootstrap) { + static void setInstance(Bootstrap bootstrap) { INSTANCE = bootstrap; } From 7f244523dbd3c2468a1e136327570b4a88db125e Mon Sep 17 00:00:00 2001 From: zane-neo Date: Wed, 16 Oct 2024 15:34:26 +0800 Subject: [PATCH 9/9] Add countdownlatch in test to coordinate the thread to avoid concureency issue caused test failure Signed-off-by: zane-neo --- .../opensearch/bootstrap/BootstrapTests.java | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/distribution/tools/keystore-cli/src/test/java/org/opensearch/bootstrap/BootstrapTests.java b/distribution/tools/keystore-cli/src/test/java/org/opensearch/bootstrap/BootstrapTests.java index fb2bc7038f865..7aa63a2736a8c 100644 --- a/distribution/tools/keystore-cli/src/test/java/org/opensearch/bootstrap/BootstrapTests.java +++ b/distribution/tools/keystore-cli/src/test/java/org/opensearch/bootstrap/BootstrapTests.java @@ -53,6 +53,8 @@ import java.nio.file.Path; import java.util.ArrayList; import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import static org.hamcrest.Matchers.equalTo; @@ -139,11 +141,24 @@ private void assertPassphraseRead(String source, String expected) { public void testInitExecutionOrder() throws Exception { AtomicInteger order = new AtomicInteger(0); - - Thread mockThread = new Thread(() -> { assertEquals(0, order.getAndIncrement()); }); + CountDownLatch countDownLatch = new CountDownLatch(1); + Thread mockThread = new Thread(() -> { + assertEquals(0, order.getAndIncrement()); + countDownLatch.countDown(); + }); Node mockNode = mock(Node.class); doAnswer(invocation -> { + try { + boolean threadStarted = countDownLatch.await(1000, TimeUnit.MILLISECONDS); + assertTrue( + "Waited for one second but the keepAliveThread isn't started, please check the execution order of" + + "keepAliveThread.start and node.start", + threadStarted + ); + } catch (InterruptedException e) { + fail("Thread interrupted"); + } assertEquals(1, order.getAndIncrement()); return null; }).when(mockNode).start();