diff --git a/.github/actions/scenarios/spring/graceful/action.yml b/.github/actions/scenarios/spring/graceful/action.yml index 0a41808527..89458bb603 100644 --- a/.github/actions/scenarios/spring/graceful/action.yml +++ b/.github/actions/scenarios/spring/graceful/action.yml @@ -37,7 +37,6 @@ runs: grace.rule.enableGraceShutdown: true grace.rule.enableOfflineNotify: true grace.rule.warmUpTime: 600 - grace.rule.upstreamAddressExpiredTime: 600 servicecomb.service.enableSpringRegister: true servicecomb.service.preferIpAddress: true # graceful-rest-provider service port 8443 do not change, it special for springCloud Edgware.SR2 test ssl feature. @@ -67,15 +66,24 @@ runs: bash ./sermant-integration-tests/scripts/checkService.sh http://127.0.0.1:8014/testGraceful 60 - name: test graceful up shell: bash - run: mvn test -Dsermant.integration.test.type=GRACEFUL -Dgraceful.test.type=up --file sermant-integration-tests/spring-test/pom.xml - - name: test graceful down + run: | + mvn test -Dsermant.integration.test.type=GRACEFUL_FEIGN -Dgraceful.test.type=up --file sermant-integration-tests/spring-test/pom.xml + mvn test -Dsermant.integration.test.type=GRACEFUL_REST -Dgraceful.test.type=up --file sermant-integration-tests/spring-test/pom.xml + - name: test rest graceful down shell: bash run: | - jps -l | grep graceful-feign-provider | awk '{print $1}' | xargs -n 1 kill jps -l | grep graceful-rest-provider | awk '{print $1}' | xargs -n 1 kill - echo "========start test graceful=======" + echo "========start test rest graceful=======" + jps -l + mvn test -Dsermant.integration.test.type=GRACEFUL_REST -Dgraceful.test.type=down --file sermant-integration-tests/spring-test/pom.xml + - name: test feign graceful down + shell: bash + run: | + mvn test -Dsermant.integration.test.type=GRACEFUL_FEIGN -Ddown.request.count=100 -Dgraceful.test.type=down --file sermant-integration-tests/spring-test/pom.xml + jps -l | grep graceful-feign-provider | awk '{print $1}' | xargs -n 1 kill + echo "========start test feign graceful=======" jps -l - mvn test -Dsermant.integration.test.type=GRACEFUL -Dgraceful.test.type=down --file sermant-integration-tests/spring-test/pom.xml + mvn test -Dsermant.integration.test.type=GRACEFUL_FEIGN -Dgraceful.test.type=down --file sermant-integration-tests/spring-test/pom.xml - name: exit if: always() uses: ./.github/actions/common/exit diff --git a/sermant-integration-tests/spring-test/spring-integration-test/src/test/java/io/sermant/integration/graceful/FeignGracefulTest.java b/sermant-integration-tests/spring-test/spring-integration-test/src/test/java/io/sermant/integration/graceful/FeignGracefulTest.java index baefc0239a..7e38ffdfb4 100644 --- a/sermant-integration-tests/spring-test/spring-integration-test/src/test/java/io/sermant/integration/graceful/FeignGracefulTest.java +++ b/sermant-integration-tests/spring-test/spring-integration-test/src/test/java/io/sermant/integration/graceful/FeignGracefulTest.java @@ -24,7 +24,7 @@ * @author zhouss * @since 2022-11-15 */ -@EnabledIfSystemProperty(named = "sermant.integration.test.type", matches = "GRACEFUL") +@EnabledIfSystemProperty(named = "sermant.integration.test.type", matches = "GRACEFUL_FEIGN") public class FeignGracefulTest extends GracefulTest { @Override protected String getBaseUrl() { diff --git a/sermant-integration-tests/spring-test/spring-integration-test/src/test/java/io/sermant/integration/graceful/GracefulTest.java b/sermant-integration-tests/spring-test/spring-integration-test/src/test/java/io/sermant/integration/graceful/GracefulTest.java index f1a00eb39c..76b4796854 100644 --- a/sermant-integration-tests/spring-test/spring-integration-test/src/test/java/io/sermant/integration/graceful/GracefulTest.java +++ b/sermant-integration-tests/spring-test/spring-integration-test/src/test/java/io/sermant/integration/graceful/GracefulTest.java @@ -51,7 +51,7 @@ public abstract class GracefulTest { private static final int UP_REQUEST_COUNT = 500; - private static final int DOWN_REQUEST_COUNT = 1000; + private static final String DOWN_REQUEST_COUNT = "1000"; private final String url = getBaseUrl(); @@ -104,7 +104,7 @@ public void testGracefulDown() { return; } try { - for (int i = 0; i < DOWN_REQUEST_COUNT; i++) { + for (int i = 0; i < getDownRequestCount(); i++) { RequestUtils.get(buildUrl("testGraceful"), Collections.emptyMap(), String.class); } } catch (Exception exception) { @@ -113,6 +113,10 @@ public void testGracefulDown() { } } + private int getDownRequestCount() { + return Integer.parseInt(EnvUtils.getEnv("down.request.count", DOWN_REQUEST_COUNT)); + } + private void statistic(Map statisticMap) { final String port = RequestUtils.get(buildUrl("testGraceful"), Collections.emptyMap(), String.class); diff --git a/sermant-integration-tests/spring-test/spring-integration-test/src/test/java/io/sermant/integration/graceful/RestGracefulTest.java b/sermant-integration-tests/spring-test/spring-integration-test/src/test/java/io/sermant/integration/graceful/RestGracefulTest.java index a388596eb7..0291b6e916 100644 --- a/sermant-integration-tests/spring-test/spring-integration-test/src/test/java/io/sermant/integration/graceful/RestGracefulTest.java +++ b/sermant-integration-tests/spring-test/spring-integration-test/src/test/java/io/sermant/integration/graceful/RestGracefulTest.java @@ -24,7 +24,7 @@ * @author zhouss * @since 2022-11-15 */ -@EnabledIfSystemProperty(named = "sermant.integration.test.type", matches = "GRACEFUL") +@EnabledIfSystemProperty(named = "sermant.integration.test.type", matches = "GRACEFUL_REST") public class RestGracefulTest extends GracefulTest { @Override protected String getBaseUrl() { diff --git a/sermant-plugins/sermant-service-registry/registry-common/src/main/java/io/sermant/registry/config/GraceConfig.java b/sermant-plugins/sermant-service-registry/registry-common/src/main/java/io/sermant/registry/config/GraceConfig.java index 4bf7038286..dc4698c0a9 100644 --- a/sermant-plugins/sermant-service-registry/registry-common/src/main/java/io/sermant/registry/config/GraceConfig.java +++ b/sermant-plugins/sermant-service-registry/registry-common/src/main/java/io/sermant/registry/config/GraceConfig.java @@ -114,11 +114,6 @@ public class GraceConfig implements PluginConfig, Cloneable { */ private long upstreamAddressExpiredTime = GraceConstants.UPSTREAM_ADDRESS_DEFAULT_EXPIRED_TIME; - /** - * Cache the wait time of notifying upstream addresses - */ - private long waitNotifyTime = GraceConstants.WAIT_NOTIFY_TIME; - /** * Correct the relevant switch attributes according to the aggregation switch, * and turn on all functions of elegant online and offline with one click @@ -267,14 +262,6 @@ public void setUpstreamAddressExpiredTime(long upstreamAddressExpiredTime) { this.upstreamAddressExpiredTime = upstreamAddressExpiredTime; } - public long getWaitNotifyTime() { - return waitNotifyTime; - } - - public void setWaitNotifyTime(long waitNotifyTime) { - this.waitNotifyTime = waitNotifyTime; - } - /** * Check whether the preheating parameter is valid * diff --git a/sermant-plugins/sermant-service-registry/registry-common/src/main/java/io/sermant/registry/config/grace/GraceConstants.java b/sermant-plugins/sermant-service-registry/registry-common/src/main/java/io/sermant/registry/config/grace/GraceConstants.java index a733630a59..413e1b1ebc 100644 --- a/sermant-plugins/sermant-service-registry/registry-common/src/main/java/io/sermant/registry/config/grace/GraceConstants.java +++ b/sermant-plugins/sermant-service-registry/registry-common/src/main/java/io/sermant/registry/config/grace/GraceConstants.java @@ -156,11 +156,6 @@ public class GraceConstants { */ public static final long UPSTREAM_ADDRESS_DEFAULT_EXPIRED_TIME = 60L; - /** - * The default wait time of notifying upstream addresses - */ - public static final long WAIT_NOTIFY_TIME = 20L; - /** * Maximum port */ diff --git a/sermant-plugins/sermant-service-registry/spring-cloud-registry-plugin/src/main/java/io/sermant/registry/config/RegistryConfigSubscribeServiceImpl.java b/sermant-plugins/sermant-service-registry/spring-cloud-registry-plugin/src/main/java/io/sermant/registry/config/RegistryConfigSubscribeServiceImpl.java index e828708efe..35186ea1fa 100644 --- a/sermant-plugins/sermant-service-registry/spring-cloud-registry-plugin/src/main/java/io/sermant/registry/config/RegistryConfigSubscribeServiceImpl.java +++ b/sermant-plugins/sermant-service-registry/spring-cloud-registry-plugin/src/main/java/io/sermant/registry/config/RegistryConfigSubscribeServiceImpl.java @@ -25,7 +25,6 @@ import io.sermant.core.plugin.subscribe.DefaultGroupConfigSubscriber; import io.sermant.core.service.dynamicconfig.common.DynamicConfigEvent; import io.sermant.core.service.dynamicconfig.common.DynamicConfigListener; -import io.sermant.registry.entity.GraceShutdownHook; import java.util.Locale; import java.util.logging.Logger; @@ -66,9 +65,6 @@ private void fixGrace() { // and this configuration has the highest priority final GraceConfig graceConfig = PluginConfigManager.getPluginConfig(GraceConfig.class); graceConfig.fixGraceSwitch(); - if (graceConfig.isEnableSpring() && graceConfig.isEnableGraceShutdown()) { - Runtime.getRuntime().addShutdownHook(new GraceShutdownHook()); - } } /** diff --git a/sermant-plugins/sermant-service-registry/spring-cloud-registry-plugin/src/main/java/io/sermant/registry/entity/GraceShutdownBehavior.java b/sermant-plugins/sermant-service-registry/spring-cloud-registry-plugin/src/main/java/io/sermant/registry/entity/GraceShutdownBehavior.java deleted file mode 100644 index cc6e89e7ea..0000000000 --- a/sermant-plugins/sermant-service-registry/spring-cloud-registry-plugin/src/main/java/io/sermant/registry/entity/GraceShutdownBehavior.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * Based on org/apache/dubbo/config/DubboShutdownHook.java - * from the Apache Dubbo project. - */ - -package io.sermant.registry.entity; - -import io.sermant.core.common.LoggerFactory; -import io.sermant.core.plugin.config.PluginConfigManager; -import io.sermant.registry.config.ConfigConstants; -import io.sermant.registry.config.GraceConfig; -import io.sermant.registry.config.grace.GraceContext; -import io.sermant.registry.utils.CommonUtils; - -import java.util.Locale; -import java.util.logging.Logger; - -/** - * Graceful closing behavior - * - * @author zhouss - * @since 2022-06-24 - */ -public class GraceShutdownBehavior implements Runnable { - private static final Logger LOGGER = LoggerFactory.getLogger(); - - private final GraceConfig graceConfig; - - GraceShutdownBehavior() { - graceConfig = PluginConfigManager.getPluginConfig(GraceConfig.class); - } - - @Override - public void run() { - if (graceConfig.isEnableSpring() && graceConfig.isEnableGraceShutdown()) { - GraceContext.INSTANCE.getGraceShutDownManager().setShutDown(true); - graceShutDown(); - } - } - - private void graceShutDown() { - // wait notify consumer complete - CommonUtils.sleep(graceConfig.getWaitNotifyTime() * ConfigConstants.SEC_DELTA); - long shutdownWaitTime = graceConfig.getShutdownWaitTime() * ConfigConstants.SEC_DELTA; - final long shutdownCheckTimeUnit = graceConfig.getShutdownCheckTimeUnit() * ConfigConstants.SEC_DELTA; - while (GraceContext.INSTANCE.getGraceShutDownManager().getRequestCount() > 0 && shutdownWaitTime > 0) { - LOGGER.info(String.format(Locale.ENGLISH, "Wait all request complete , remained count [%s]", - GraceContext.INSTANCE.getGraceShutDownManager().getRequestCount())); - CommonUtils.sleep(shutdownCheckTimeUnit); - shutdownWaitTime -= shutdownCheckTimeUnit; - } - final int requestCount = GraceContext.INSTANCE.getGraceShutDownManager().getRequestCount(); - if (requestCount > 0) { - LOGGER.warning(String.format(Locale.ENGLISH, "Request num that does not completed is [%s] ", requestCount)); - } else { - LOGGER.fine("Graceful shutdown completed!"); - } - } -} diff --git a/sermant-plugins/sermant-service-registry/spring-cloud-registry-plugin/src/main/java/io/sermant/registry/entity/GraceShutdownHook.java b/sermant-plugins/sermant-service-registry/spring-cloud-registry-plugin/src/main/java/io/sermant/registry/entity/GraceShutdownHook.java deleted file mode 100644 index 2e2dae0798..0000000000 --- a/sermant-plugins/sermant-service-registry/spring-cloud-registry-plugin/src/main/java/io/sermant/registry/entity/GraceShutdownHook.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * Based on org/apache/dubbo/config/DubboShutdownHook.java - * from the Apache Dubbo project. - */ - -package io.sermant.registry.entity; - -/** - * Elegant offline HOOK - * - * @author zhouss - * @since 2022-06-24 - */ -public class GraceShutdownHook extends Thread { - private static final String GRACE_SHUTDOWN_THREAD = "SermantGraceShutdown"; - - /** - * Constructor Default thread name - */ - public GraceShutdownHook() { - super(new GraceShutdownBehavior(), GRACE_SHUTDOWN_THREAD); - } -} diff --git a/sermant-plugins/sermant-service-registry/spring-cloud-registry-plugin/src/main/java/io/sermant/registry/utils/RefreshUtils.java b/sermant-plugins/sermant-service-registry/spring-cloud-registry-plugin/src/main/java/io/sermant/registry/utils/RefreshUtils.java index a8397968bc..a62b354546 100644 --- a/sermant-plugins/sermant-service-registry/spring-cloud-registry-plugin/src/main/java/io/sermant/registry/utils/RefreshUtils.java +++ b/sermant-plugins/sermant-service-registry/spring-cloud-registry-plugin/src/main/java/io/sermant/registry/utils/RefreshUtils.java @@ -96,7 +96,7 @@ private static void refreshWithSpringLb(String serviceName, Collection r "Can not refresh service [%s] instance cache with spring loadbalancer!", curServiceName)); return; } - LOGGER.fine(String.format(Locale.ENGLISH, + LOGGER.info(String.format(Locale.ENGLISH, "Start refresh target service [%s] spring loadbalancer instance cache!", curServiceName)); final Optional cacheOptional = ReflectUtils.invokeMethod(loadBalancerCacheManager, "getCache", new Class[]{String.class}, new Object[]{GraceConstants.SPRING_CACHE_MANAGER_LOADBALANCER_CACHE_NAME}); @@ -112,7 +112,7 @@ private static void refreshWithSpringLb(String serviceName, Collection r private static void refreshWithRibbon(Object loadbalancer) { final Optional serviceName = ReflectUtils.getFieldValue(loadbalancer, "name"); - LOGGER.fine(String.format(Locale.ENGLISH, + LOGGER.info(String.format(Locale.ENGLISH, "Start refresh target service [%s] ribbon instance cache!", serviceName.orElse("unKnow service"))); ReflectUtils.invokeMethod(loadbalancer, "updateListOfServers", null, null); } diff --git a/sermant-plugins/sermant-service-registry/spring-cloud-registry-plugin/src/test/java/io/sermant/registry/entity/GraceShutdownHookTest.java b/sermant-plugins/sermant-service-registry/spring-cloud-registry-plugin/src/test/java/io/sermant/registry/entity/GraceShutdownHookTest.java deleted file mode 100644 index ee9195a3e0..0000000000 --- a/sermant-plugins/sermant-service-registry/spring-cloud-registry-plugin/src/test/java/io/sermant/registry/entity/GraceShutdownHookTest.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright (C) 2022-2022 Huawei Technologies Co., Ltd. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.sermant.registry.entity; - -import io.sermant.registry.config.ConfigConstants; -import io.sermant.registry.config.GraceConfig; -import io.sermant.registry.config.grace.GraceContext; - -import io.sermant.core.plugin.config.PluginConfigManager; - -import org.junit.Assert; -import org.junit.Test; -import org.mockito.MockedStatic; -import org.mockito.Mockito; - -/** - * Gracefully close the test - * - * @author zhouss - * @since 2022-09-06 - */ -public class GraceShutdownHookTest { - @Test - public void test() { - try (final MockedStatic pluginConfigManagerMockedStatic = Mockito - .mockStatic(PluginConfigManager.class)) { - final GraceConfig graceConfig = new GraceConfig(); - pluginConfigManagerMockedStatic.when(() -> PluginConfigManager.getPluginConfig(GraceConfig.class)) - .thenReturn(graceConfig); - final GraceShutdownBehavior graceShutdownBehavior = new GraceShutdownBehavior(); - graceConfig.setEnableSpring(true); - graceConfig.setEnableGraceShutdown(true); - graceConfig.setShutdownWaitTime(1); - final long start = System.currentTimeMillis(); - GraceContext.INSTANCE.getGraceShutDownManager().increaseRequestCount(); - graceShutdownBehavior.run(); - Assert.assertTrue(System.currentTimeMillis() - start - >= graceConfig.getShutdownWaitTime() * ConfigConstants.SEC_DELTA); - Assert.assertTrue(GraceContext.INSTANCE.getGraceShutDownManager().isShutDown()); - GraceContext.INSTANCE.getGraceShutDownManager().setShutDown(false); - GraceContext.INSTANCE.getGraceShutDownManager().decreaseRequestCount(); - } - } -} diff --git a/sermant-plugins/sermant-service-registry/spring-cloud-registry-plugin/src/test/java/io/sermant/registry/grace/interceptors/SpringWebHandlerInterceptorTest.java b/sermant-plugins/sermant-service-registry/spring-cloud-registry-plugin/src/test/java/io/sermant/registry/grace/interceptors/SpringWebHandlerInterceptorTest.java index c820a1c082..0b090b6a52 100644 --- a/sermant-plugins/sermant-service-registry/spring-cloud-registry-plugin/src/test/java/io/sermant/registry/grace/interceptors/SpringWebHandlerInterceptorTest.java +++ b/sermant-plugins/sermant-service-registry/spring-cloud-registry-plugin/src/test/java/io/sermant/registry/grace/interceptors/SpringWebHandlerInterceptorTest.java @@ -17,13 +17,12 @@ package io.sermant.registry.grace.interceptors; -import io.sermant.registry.config.GraceConfig; -import io.sermant.registry.config.RegisterConfig; -import io.sermant.registry.services.GraceService; - import io.sermant.core.plugin.agent.entity.ExecuteContext; import io.sermant.core.plugin.config.PluginConfigManager; import io.sermant.core.plugin.service.PluginServiceManager; +import io.sermant.registry.config.GraceConfig; +import io.sermant.registry.config.RegisterConfig; +import io.sermant.registry.services.GraceService; import org.junit.After; import org.junit.Assert; diff --git a/sermant-plugins/sermant-service-registry/spring-cloud-registry-plugin/src/test/java/io/sermant/registry/inject/grace/SpringRequestInterceptorTest.java b/sermant-plugins/sermant-service-registry/spring-cloud-registry-plugin/src/test/java/io/sermant/registry/inject/grace/SpringRequestInterceptorTest.java index f730a4194e..c082508ed2 100644 --- a/sermant-plugins/sermant-service-registry/spring-cloud-registry-plugin/src/test/java/io/sermant/registry/inject/grace/SpringRequestInterceptorTest.java +++ b/sermant-plugins/sermant-service-registry/spring-cloud-registry-plugin/src/test/java/io/sermant/registry/inject/grace/SpringRequestInterceptorTest.java @@ -16,14 +16,13 @@ package io.sermant.registry.inject.grace; +import io.sermant.core.plugin.config.PluginConfigManager; +import io.sermant.core.plugin.service.PluginServiceManager; import io.sermant.registry.config.GraceConfig; import io.sermant.registry.config.grace.GraceConstants; import io.sermant.registry.config.grace.GraceContext; import io.sermant.registry.services.GraceService; -import io.sermant.core.plugin.config.PluginConfigManager; -import io.sermant.core.plugin.service.PluginServiceManager; - import org.junit.After; import org.junit.Assert; import org.junit.Before; diff --git a/sermant-plugins/sermant-service-registry/spring-cloud-registry-service/src/main/java/io/sermant/registry/service/cache/AddressCache.java b/sermant-plugins/sermant-service-registry/spring-cloud-registry-service/src/main/java/io/sermant/registry/service/cache/AddressCache.java index e69ceba624..b296c72836 100644 --- a/sermant-plugins/sermant-service-registry/spring-cloud-registry-service/src/main/java/io/sermant/registry/service/cache/AddressCache.java +++ b/sermant-plugins/sermant-service-registry/spring-cloud-registry-service/src/main/java/io/sermant/registry/service/cache/AddressCache.java @@ -18,12 +18,17 @@ import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; +import com.google.common.cache.RemovalListener; +import com.google.common.cache.RemovalNotification; +import io.sermant.core.common.LoggerFactory; import io.sermant.core.plugin.config.PluginConfigManager; import io.sermant.registry.config.GraceConfig; import java.util.Set; import java.util.concurrent.TimeUnit; +import java.util.logging.Level; +import java.util.logging.Logger; /** * Upstream address caching @@ -36,6 +41,7 @@ public enum AddressCache { * Singleton */ INSTANCE; + private final Cache cache; AddressCache() { @@ -43,6 +49,7 @@ public enum AddressCache { cache = CacheBuilder.newBuilder() .maximumSize(pluginConfig.getUpstreamAddressMaxSize()) // 设置缓存的最大容量 .expireAfterWrite(pluginConfig.getUpstreamAddressExpiredTime(), TimeUnit.SECONDS) // 设置缓存失效时间 + .removalListener(new CacheRemovalListener()) .build(); } @@ -70,4 +77,30 @@ public Set getAddressSet() { public void cleanCache() { cache.invalidateAll(); } + + /** + * get cache size + * + * @return size + */ + public int size() { + cache.cleanUp(); + return (int) cache.size(); + } + + /** + * Cache Removal Listener + * + * @author provenceee + * @since 2024-09-27 + */ + private static class CacheRemovalListener implements RemovalListener { + private static final Logger LOGGER = LoggerFactory.getLogger(); + + @Override + public void onRemoval(RemovalNotification notification) { + LOGGER.log(Level.INFO, "[{0}] will remove, type is [{1}]", new Object[]{notification.getKey(), + notification.getCause()}); + } + } } \ No newline at end of file diff --git a/sermant-plugins/sermant-service-registry/spring-cloud-registry-service/src/main/java/io/sermant/registry/service/impl/GraceServiceImpl.java b/sermant-plugins/sermant-service-registry/spring-cloud-registry-service/src/main/java/io/sermant/registry/service/impl/GraceServiceImpl.java index efdd452acf..90259bf522 100644 --- a/sermant-plugins/sermant-service-registry/spring-cloud-registry-service/src/main/java/io/sermant/registry/service/impl/GraceServiceImpl.java +++ b/sermant-plugins/sermant-service-registry/spring-cloud-registry-service/src/main/java/io/sermant/registry/service/impl/GraceServiceImpl.java @@ -22,6 +22,8 @@ import io.sermant.core.plugin.config.PluginConfigManager; import io.sermant.core.plugin.service.PluginServiceManager; import io.sermant.core.utils.ReflectUtils; +import io.sermant.registry.config.ConfigConstants; +import io.sermant.registry.config.GraceConfig; import io.sermant.registry.config.RegisterConfig; import io.sermant.registry.config.grace.GraceConstants; import io.sermant.registry.config.grace.GraceContext; @@ -32,12 +34,16 @@ import io.sermant.registry.service.utils.HttpClientUtils; import io.sermant.registry.services.GraceService; import io.sermant.registry.services.RegisterCenterService; +import io.sermant.registry.utils.CommonUtils; import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashMap; +import java.util.Locale; import java.util.Map; +import java.util.Set; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.Executor; import java.util.concurrent.Executors; import java.util.concurrent.atomic.AtomicBoolean; @@ -65,27 +71,74 @@ public class GraceServiceImpl implements GraceService { private static final int RETRY_TIME = 3; + private final GraceConfig graceConfig; + + private CountDownLatch latch; + + /** + * Constructor + */ + public GraceServiceImpl() { + this.graceConfig = PluginConfigManager.getPluginConfig(GraceConfig.class); + } + /** * Offline notifications */ @Override public void shutdown() { if (SHUTDOWN.compareAndSet(false, true)) { - Object registration = GraceContext.INSTANCE.getGraceShutDownManager().getRegistration(); - ReflectUtils.invokeMethodWithNoneParameter(registration, REGISTRATION_DEREGISTER_METHOD_NAME); - checkAndCloseSc(); GraceContext.INSTANCE.getGraceShutDownManager().setShutDown(true); - ClientInfo clientInfo = RegisterContext.INSTANCE.getClientInfo(); - Map> header = new HashMap<>(); - header.put(GraceConstants.MARK_SHUTDOWN_SERVICE_NAME, - Collections.singletonList(clientInfo.getServiceName())); - header.put(GraceConstants.MARK_SHUTDOWN_SERVICE_ENDPOINT, - Arrays.asList(clientInfo.getIp() + ":" + clientInfo.getPort(), - clientInfo.getHost() + ":" + clientInfo.getPort())); - AddressCache.INSTANCE.getAddressSet().forEach(address -> notifyToGraceHttpServer(address, header)); + notifyToGraceHttpServer(); + graceShutDown(); + } + } + + private boolean isCompleteNotify() { + return latch != null && latch.getCount() == 0; + } + + private void graceShutDown() { + if (!graceConfig.isEnableSpring() || !graceConfig.isEnableGraceShutdown()) { + return; + } + + // wait notify consumer complete + CommonUtils.sleep(ConfigConstants.SEC_DELTA); + long shutdownWaitTime = graceConfig.getShutdownWaitTime() * ConfigConstants.SEC_DELTA; + final long shutdownCheckTimeUnit = graceConfig.getShutdownCheckTimeUnit() * ConfigConstants.SEC_DELTA; + while ((GraceContext.INSTANCE.getGraceShutDownManager().getRequestCount() > 0 && shutdownWaitTime > 0) + || !isCompleteNotify()) { + LOGGER.info(String.format(Locale.ENGLISH, "Wait all request complete , remained count [%s]", + GraceContext.INSTANCE.getGraceShutDownManager().getRequestCount())); + CommonUtils.sleep(shutdownCheckTimeUnit); + shutdownWaitTime -= shutdownCheckTimeUnit; + } + final int requestCount = GraceContext.INSTANCE.getGraceShutDownManager().getRequestCount(); + if (requestCount > 0) { + LOGGER.warning(String.format(Locale.ENGLISH, "Request num that does not completed is [%s] ", requestCount)); + } else { + LOGGER.info("Graceful shutdown completed!"); } } + private void notifyToGraceHttpServer() { + Object registration = GraceContext.INSTANCE.getGraceShutDownManager().getRegistration(); + ReflectUtils.invokeMethodWithNoneParameter(registration, REGISTRATION_DEREGISTER_METHOD_NAME); + checkAndCloseSc(); + GraceContext.INSTANCE.getGraceShutDownManager().setShutDown(true); + ClientInfo clientInfo = RegisterContext.INSTANCE.getClientInfo(); + Map> header = new HashMap<>(); + header.put(GraceConstants.MARK_SHUTDOWN_SERVICE_NAME, + Collections.singletonList(clientInfo.getServiceName())); + header.put(GraceConstants.MARK_SHUTDOWN_SERVICE_ENDPOINT, + Arrays.asList(clientInfo.getIp() + ":" + clientInfo.getPort(), + clientInfo.getHost() + ":" + clientInfo.getPort())); + Set addressSet = AddressCache.INSTANCE.getAddressSet(); + latch = new CountDownLatch(AddressCache.INSTANCE.size()); + addressSet.forEach(address -> notifyToGraceHttpServer(address, header)); + } + private void notifyToGraceHttpServer(String address, Map> header) { EXECUTOR.execute(() -> execute(address, header)); } @@ -100,6 +153,7 @@ private void execute(String address, Map> header) { } LOGGER.log(Level.WARNING, "Failed to notify before shutdown, address: {0}", address); } + latch.countDown(); } /** diff --git a/sermant-plugins/sermant-service-registry/spring-cloud-registry-service/src/test/java/io/sermant/registry/service/impl/GraceServiceImplTest.java b/sermant-plugins/sermant-service-registry/spring-cloud-registry-service/src/test/java/io/sermant/registry/service/impl/GraceServiceImplTest.java index adc8de5b6b..d4664b55da 100644 --- a/sermant-plugins/sermant-service-registry/spring-cloud-registry-service/src/test/java/io/sermant/registry/service/impl/GraceServiceImplTest.java +++ b/sermant-plugins/sermant-service-registry/spring-cloud-registry-service/src/test/java/io/sermant/registry/service/impl/GraceServiceImplTest.java @@ -19,8 +19,10 @@ import io.sermant.core.plugin.config.PluginConfigManager; import io.sermant.core.utils.ReflectUtils; +import io.sermant.registry.config.ConfigConstants; import io.sermant.registry.config.GraceConfig; import io.sermant.registry.config.RegisterConfig; +import io.sermant.registry.config.grace.GraceContext; import io.sermant.registry.service.cache.AddressCache; import org.junit.After; @@ -45,11 +47,13 @@ public class GraceServiceImplTest { */ public MockedStatic pluginConfigManagerMockedStatic; + private final GraceConfig graceConfig = new GraceConfig(); + @Before public void setUp() { pluginConfigManagerMockedStatic = Mockito.mockStatic(PluginConfigManager.class); pluginConfigManagerMockedStatic.when(() -> PluginConfigManager.getPluginConfig(GraceConfig.class)) - .thenReturn(new GraceConfig()); + .thenReturn(graceConfig); pluginConfigManagerMockedStatic.when(() -> PluginConfigManager.getPluginConfig(RegisterConfig.class)) .thenReturn(new RegisterConfig()); } @@ -66,8 +70,13 @@ public void tearDown() { @Test public void testShutDown() { final GraceServiceImpl spy = Mockito.spy(new GraceServiceImpl()); - spy.shutdown(); + graceConfig.setEnableSpring(true); + graceConfig.setEnableGraceShutdown(true); + graceConfig.setShutdownWaitTime(1); + final long start = System.currentTimeMillis(); + GraceContext.INSTANCE.getGraceShutDownManager().increaseRequestCount(); spy.addAddress("test"); + spy.shutdown(); Mockito.doCallRealMethod().when(spy).shutdown(); Mockito.verify(spy, Mockito.times(1)).shutdown(); Mockito.verify(spy, Mockito.times(1)).addAddress("test"); @@ -75,5 +84,10 @@ public void testShutDown() { final Optional shutdown = ReflectUtils.getFieldValue(spy, "SHUTDOWN"); Assert.assertTrue(shutdown.isPresent() && shutdown.get() instanceof AtomicBoolean); Assert.assertTrue(((AtomicBoolean) shutdown.get()).get()); + Assert.assertTrue(System.currentTimeMillis() - start + >= graceConfig.getShutdownWaitTime() * ConfigConstants.SEC_DELTA); + Assert.assertTrue(GraceContext.INSTANCE.getGraceShutDownManager().isShutDown()); + GraceContext.INSTANCE.getGraceShutDownManager().setShutDown(false); + GraceContext.INSTANCE.getGraceShutDownManager().decreaseRequestCount(); } } diff --git a/sermant-plugins/sermant-service-registry/spring-cloud-registry-service/src/test/java/io/sermant/registry/service/server/GraceHttpServerTest.java b/sermant-plugins/sermant-service-registry/spring-cloud-registry-service/src/test/java/io/sermant/registry/service/server/GraceHttpServerTest.java index 22425928ab..f0b5866639 100644 --- a/sermant-plugins/sermant-service-registry/spring-cloud-registry-service/src/test/java/io/sermant/registry/service/server/GraceHttpServerTest.java +++ b/sermant-plugins/sermant-service-registry/spring-cloud-registry-service/src/test/java/io/sermant/registry/service/server/GraceHttpServerTest.java @@ -63,8 +63,17 @@ public class GraceHttpServerTest { @Before public void setUp() { - pluginConfigManagerMockedStatic = Mockito.mockStatic(PluginConfigManager.class); pluginServiceManagerMockedStatic = Mockito.mockStatic(PluginServiceManager.class); + pluginServiceManagerMockedStatic.when(() -> PluginServiceManager.getPluginService(GraceService.class)) + .thenReturn(new GraceServiceImpl()); + pluginConfigManagerMockedStatic = Mockito.mockStatic(PluginConfigManager.class); + final GraceConfig graceConfig = new GraceConfig(); + graceConfig.setEnableSpring(true); + graceConfig.setEnableGraceShutdown(true); + pluginConfigManagerMockedStatic.when(() -> PluginConfigManager.getPluginConfig(RegisterConfig.class)) + .thenReturn(new RegisterConfig()); + pluginConfigManagerMockedStatic.when(() -> PluginConfigManager.getPluginConfig(GraceConfig.class)) + .thenReturn(graceConfig); } @After @@ -78,15 +87,6 @@ public void tearDown() { */ @Test public void testHttpServer() { - final GraceConfig graceConfig = new GraceConfig(); - graceConfig.setEnableSpring(true); - graceConfig.setEnableGraceShutdown(true); - pluginConfigManagerMockedStatic.when(() -> PluginConfigManager.getPluginConfig(RegisterConfig.class)) - .thenReturn(new RegisterConfig()); - pluginConfigManagerMockedStatic.when(() -> PluginConfigManager.getPluginConfig(GraceConfig.class)) - .thenReturn(graceConfig); - pluginServiceManagerMockedStatic.when(() -> PluginServiceManager.getPluginService(GraceService.class)) - .thenReturn(new GraceServiceImpl()); final GraceHttpServer graceHttpServer = new GraceHttpServer(); graceHttpServer.start(); try {