Support phase-based job shutdown (graceful shutdown, aggressive grace…

…ful shutdown, forced shutdown) to shut down gracefully and promptly

turms-server-common/src/main/java/im/turms/server/common/infra/application/TurmsApplicationContext.java

@@ -85,7 +85,8 @@ public class TurmsApplicationContext {
     private final String activeEnvProfile;
     private final BuildProperties buildProperties;
 
-    private long shutdownJobTimeoutMillis;
+    private long shutdownJobGracefulTimeoutMillis;
+    private long shutdownJobForcedTimeoutMillis;
     private final TreeMap<JobShutdownOrder, ShutdownHook> shutdownHooks = new TreeMap<>();
 
     public TurmsApplicationContext(Environment environment) {
@@ -175,7 +176,9 @@ public void handleContextRefreshedEvent(ContextRefreshedEvent event) {
                 .getBean(TurmsPropertiesManager.class);
         ShutdownProperties properties = propertiesManager.getLocalProperties()
                 .getShutdown();
-        shutdownJobTimeoutMillis = properties.getJobTimeoutMillis();
+        shutdownJobGracefulTimeoutMillis = properties.getJobGracefulTimeoutMillis();
+        shutdownJobForcedTimeoutMillis =
+                Math.max(properties.getJobForcedTimeoutMillis(), shutdownJobGracefulTimeoutMillis);
     }
 
     @EventListener(classes = ContextClosedEvent.class)
@@ -188,18 +191,18 @@ public void handleContextClosedEvent() {
             String jobName = key.name();
             boolean isClosingLogProcessor = key == JobShutdownOrder.CLOSE_LOG_PROCESSOR;
             Future<Mono<Void>> shutdownFuture = executor.submit(() -> orderAndJob.getValue()
-                    .run(shutdownJobTimeoutMillis));
+                    .run(shutdownJobGracefulTimeoutMillis));
             try {
                 long time = System.currentTimeMillis();
                 Mono<Void> mono =
-                        shutdownFuture.get(shutdownJobTimeoutMillis, TimeUnit.MILLISECONDS);
+                        shutdownFuture.get(shutdownJobForcedTimeoutMillis, TimeUnit.MILLISECONDS);
                 if (mono == null) {
                     throw new IllegalArgumentException(
                             "The result of the job \""
                                     + jobName
                                     + "\" must not be null");
                 }
-                time = shutdownJobTimeoutMillis - (System.currentTimeMillis() - time);
+                time = shutdownJobForcedTimeoutMillis - (System.currentTimeMillis() - time);
                 if (time <= 0) {
                     throw new TimeoutException();
                 }

turms-server-common/src/main/java/im/turms/server/common/infra/property/env/common/ShutdownProperties.java

@@ -35,9 +35,15 @@
 @NoArgsConstructor
 public class ShutdownProperties {
 
-    @Description("Wait for a job 2 minutes at most for extreme cases by default. "
-            + "Though it is a long time, graceful shutdown is usually better than force shutdown.")
+    @Description("The graceful shutdown timeout in milliseconds. After the timeout, the job will try to shut down gracefully in a more aggressive way "
+            + "before being forced to be shutdown when reaching \"jobForcedTimeoutMillis\"")
     @Min(0)
-    private long jobTimeoutMillis = 120 * 1000;
+    private long jobGracefulTimeoutMillis = 60 * 1000;
+
+    @Description("The forced shutdown timeout in milliseconds. After the timeout, the job will be forced to be shutdown immediately no matter "
+            + "whether it is running or not. If this value is equal to or less than \"jobGracefulTimeoutMillis\", the job will be forced to be shutdown when "
+            + "reaching \"jobGracefulTimeoutMillis\"")
+    @Min(0)
+    private long jobForcedTimeoutMillis = 90 * 1000;
 
 }

turms-server-common/src/test/resources/application-full-example.yaml

@@ -2474,10 +2474,14 @@ turms:
         # mutable property: true
         ignore-unknown-settings-on-upsert: false
   shutdown:
-    # Wait for a job 2 minutes at most for extreme cases by default. Though it is a long time, graceful shutdown is usually better than force shutdown.
+    # The forced shutdown timeout in milliseconds. After the timeout, the job will be forced to be shutdown immediately no matter whether it is running or not. If this value is equal to or less than "jobGracefulTimeoutMillis", the job will be forced to be shutdown when reaching "jobGracefulTimeoutMillis".
     # global property: false
     # mutable property: false
-    job-timeout-millis: 120000
+    job-forced-timeout-millis: 90000
+    # The graceful shutdown timeout in milliseconds. After the timeout, the job will try to shut down gracefully in a more aggressive way before being forced to be shutdown when reaching "jobForcedTimeoutMillis".
+    # global property: false
+    # mutable property: false
+    job-graceful-timeout-millis: 60000
   user-status:
     # Whether to cache the user sessions status.
     # global property: false

turms-server-common/src/test/resources/turms-properties-metadata-with-property-value.json

@@ -4528,7 +4528,7 @@
               "type": "boolean",
               "value": false
             },
-            "roleId": {
+            "roleIds": {
               "deprecated": false,
               "global": false,
               "mutable": false,
@@ -6189,14 +6189,23 @@
     }
   },
   "shutdown": {
-    "jobTimeoutMillis": {
+    "jobForcedTimeoutMillis": {
       "deprecated": false,
-      "description": "Wait for a job 2 minutes at most for extreme cases by default. Though it is a long time, graceful shutdown is usually better than force shutdown.",
+      "description": "The forced shutdown timeout in milliseconds. After the timeout, the job will be forced to be shutdown immediately no matter whether it is running or not. If this value is equal to or less than \"jobGracefulTimeoutMillis\", the job will be forced to be shutdown when reaching \"jobGracefulTimeoutMillis\"",
       "global": false,
       "mutable": false,
       "sensitive": false,
       "type": "long",
-      "value": 120000
+      "value": 90000
+    },
+    "jobGracefulTimeoutMillis": {
+      "deprecated": false,
+      "description": "The graceful shutdown timeout in milliseconds. After the timeout, the job will try to shut down gracefully in a more aggressive way before being forced to be shutdown when reaching \"jobForcedTimeoutMillis\"",
+      "global": false,
+      "mutable": false,
+      "sensitive": false,
+      "type": "long",
+      "value": 60000
     }
   },
   "userStatus": {

turms-server-common/src/test/resources/turms-properties-metadata.json

@@ -3972,7 +3972,7 @@
               "sensitive": false,
               "type": "boolean"
             },
-            "roleId": {
+            "roleIds": {
               "deprecated": false,
               "global": false,
               "mutable": false,
@@ -5435,9 +5435,17 @@
     }
   },
   "shutdown": {
-    "jobTimeoutMillis": {
+    "jobForcedTimeoutMillis": {
       "deprecated": false,
-      "description": "Wait for a job 2 minutes at most for extreme cases by default. Though it is a long time, graceful shutdown is usually better than force shutdown.",
+      "description": "The forced shutdown timeout in milliseconds. After the timeout, the job will be forced to be shutdown immediately no matter whether it is running or not. If this value is equal to or less than \"jobGracefulTimeoutMillis\", the job will be forced to be shutdown when reaching \"jobGracefulTimeoutMillis\"",
+      "global": false,
+      "mutable": false,
+      "sensitive": false,
+      "type": "long"
+    },
+    "jobGracefulTimeoutMillis": {
+      "deprecated": false,
+      "description": "The graceful shutdown timeout in milliseconds. After the timeout, the job will try to shut down gracefully in a more aggressive way before being forced to be shutdown when reaching \"jobForcedTimeoutMillis\"",
       "global": false,
       "mutable": false,
       "sensitive": false,