From c01d9f5ba0fc72c7733a487adc56023eda92ab96 Mon Sep 17 00:00:00 2001 From: guentherm Date: Wed, 2 Mar 2022 14:45:07 +0100 Subject: [PATCH 1/6] SDCISA-3814 Redisques queue montitoring and statistics integration --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 97b366723..14e2382fa 100644 --- a/pom.xml +++ b/pom.xml @@ -313,7 +313,7 @@ org.swisspush redisques - 3.0.1 + 3.0.2 org.swisspush From 33188733e15330b001efa99de9ded6d7f8b7c9c3 Mon Sep 17 00:00:00 2001 From: guentherm Date: Thu, 14 Apr 2022 14:50:48 +0200 Subject: [PATCH 2/6] SDCISA-3814 Redisques queue montitoring and statistics integration --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index fd3f94a18..988a771ed 100644 --- a/pom.xml +++ b/pom.xml @@ -313,7 +313,7 @@ org.swisspush redisques - 3.0.3 + 3.0.4 org.swisspush From e0207e0a35ecea0109124fda94564e82b20f67c9 Mon Sep 17 00:00:00 2001 From: guentherm Date: Thu, 22 Sep 2022 18:49:27 +0200 Subject: [PATCH 3/6] #473 Provide route hook timeout property --- gateleen-hook/README_hook.md | 26 +++++------ .../swisspush/gateleen/hook/HookHandler.java | 8 +++- .../org/swisspush/gateleen/hook/HttpHook.java | 44 ++++++++++++++----- .../org/swisspush/gateleen/hook/Route.java | 12 ++++- .../resources/gateleen_hooking_schema_hook | 7 ++- .../gateleen/hook/HookHandlerTest.java | 35 +++++++++++++++ .../gateleen/hook/HookSchemaTest.java | 3 +- .../org/swisspush/gateleen/routing/Rule.java | 2 + .../gateleen/routing/RuleFactory.java | 3 +- 9 files changed, 109 insertions(+), 31 deletions(-) diff --git a/gateleen-hook/README_hook.md b/gateleen-hook/README_hook.md index 7cbf6fcba..556c2a841 100644 --- a/gateleen-hook/README_hook.md +++ b/gateleen-hook/README_hook.md @@ -20,20 +20,20 @@ Adding hooks requires a validation against a schema to be positive. Check the sc **Payload** -| Property | Required | Description | -|:------------------ | :------: | :--------------------------------------- | -| destination | yes | A valid URL (absolute) where the requests should be redirected to. | -| methods | no | An array of valid HTTP methods (PUT, GET, DELETE, ...) to define for which requests the redirection should be performed. As a default all requests will be redirected. | -| *expireAfter* | no | *DEPRECATED - Hooks don't manipulate or set the x-expire-after header any more. Use HeaderFunctions instead* | -| staticHeaders | no | (*deprecated - use headers*) This property allows you to set static headers passed down to every request. The defined static headers will overwrite given ones! | -| headers | no | array of request header manipulations: set, remove, complete (set-if-absent) and override (set-if-present)| -| headersFilter | no | A regular expression to define which requests headers must be present for the route to be used. Each request header entry is validated in the format `: `, so you are able to filter for request header names and values.| -| collection | no | This property specifies if the given URL is a resource or a collection. If this property is not set, the default is true (collection). | -| listable | no | This property specifies if a route is listed if a GET is performed on its parent or not. The default is false or set by the value passed during the initialization. | -| translateStatus | no | This property defines a mapping to translate http respond status values. | -| connectionPoolSize | no | This property defines the max number of TCP connections which will be open in parallel to the destination system (Default 50) | +| Property | Required | Description | +|:-------------------| :------: |:----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| destination | yes | A valid URL (absolute) where the requests should be redirected to. | +| methods | no | An array of valid HTTP methods (PUT, GET, DELETE, ...) to define for which requests the redirection should be performed. As a default all requests will be redirected. | +| *expireAfter* | no | *DEPRECATED - Hooks don't manipulate or set the x-expire-after header any more. Use HeaderFunctions instead* | +| staticHeaders | no | (*deprecated - use headers*) This property allows you to set static headers passed down to every request. The defined static headers will overwrite given ones! | +| headers | no | array of request header manipulations: set, remove, complete (set-if-absent) and override (set-if-present) | +| headersFilter | no | A regular expression to define which requests headers must be present for the route to be used. Each request header entry is validated in the format `: `, so you are able to filter for request header names and values. | +| collection | no | This property specifies if the given URL is a resource or a collection. If this property is not set, the default is true (collection). | +| listable | no | This property specifies if a route is listed if a GET is performed on its parent or not. The default is false or set by the value passed during the initialization. | +| translateStatus | no | This property defines a mapping to translate http respond status values. | +| connectionPoolSize | no | This property defines the max number of TCP connections which will be open in parallel to the destination system (Default 50) | | maxWaitQueueSize | no | This property defines the max number of requests allowed in the wait queue of the connection pool to the destination. If set to 0 then new incoming requests are rejected immediately if the pool is full (max connectionPoolSize reached), otherwise the requests are queued until the maxWaitQueueSize is reached and dispatched once connections in the pool become available. (Default -1, unbounded) | - +| timeout | no | This property defines the request timeout in seconds (Default 30 seconds) applied to requests established over the route | > Attention: A route has a default expiration time of **1 hour**. After this time the route will expire and be removed from the storage, as well as the HookHandler.
To update / refresh a route, simply perform another registration.
To change the expiration time of a route, just pass a _X-Expire-After_ header with the registration PUT request. diff --git a/gateleen-hook/src/main/java/org/swisspush/gateleen/hook/HookHandler.java b/gateleen-hook/src/main/java/org/swisspush/gateleen/hook/HookHandler.java index cbd6608df..a15c65728 100755 --- a/gateleen-hook/src/main/java/org/swisspush/gateleen/hook/HookHandler.java +++ b/gateleen-hook/src/main/java/org/swisspush/gateleen/hook/HookHandler.java @@ -1501,6 +1501,11 @@ private void registerRoute(Buffer buffer) { hook.setConnectionPoolSize(jsonHook.getInteger(HttpHook.CONNECTION_POOL_SIZE_PROPERTY_NAME)); hook.setMaxWaitQueueSize(jsonHook.getInteger(HttpHook.CONNECTION_MAX_WAIT_QUEUE_SIZE_PROPERTY_NAME)); + // Configure request timeout + Integer timeout = jsonHook.getInteger(HttpHook.CONNECTION_TIMEOUT_SEC_PROPERTY_NAME); + if (timeout != null) { + hook.setTimeout(1000 * timeout); + } boolean mustCreateNewRoute = true; Route existingRoute = routeRepository.getRoutes().get(routedUrl); @@ -1536,7 +1541,8 @@ private boolean mustCreateNewRouteForHook(Route existingRoute, HttpHook newHook) same &= oldHook.isCollection () == newHook.isCollection () ; same &= oldHook.isCollection () == newHook.isCollection () ; same &= Objects.equals(oldHook.getConnectionPoolSize() , newHook.getConnectionPoolSize()); - same &= Objects.equals(oldHook.getMaxWaitQueueSize() , newHook.getMaxWaitQueueSize()); + same &= Objects.equals(oldHook.getMaxWaitQueueSize() , newHook.getMaxWaitQueueSize ()); + same &= Objects.equals(oldHook.getTimeout(), newHook.getTimeout ()); same &= headersFilterPatternEquals(oldHook.getHeadersFilterPattern(), newHook.getHeadersFilterPattern()); // queueingStrategy, filter, queueExpireAfter and hookTriggerType are not relevant for Route-Hooks diff --git a/gateleen-hook/src/main/java/org/swisspush/gateleen/hook/HttpHook.java b/gateleen-hook/src/main/java/org/swisspush/gateleen/hook/HttpHook.java index e2248c161..7d947c753 100755 --- a/gateleen-hook/src/main/java/org/swisspush/gateleen/hook/HttpHook.java +++ b/gateleen-hook/src/main/java/org/swisspush/gateleen/hook/HttpHook.java @@ -13,7 +13,7 @@ /** * Represents a hook. - * + * * @author https://github.com/ljucam [Mario Ljuca] */ public class HttpHook { @@ -21,6 +21,8 @@ public class HttpHook { public static final String CONNECTION_MAX_WAIT_QUEUE_SIZE_PROPERTY_NAME = Rule.MAX_WAIT_QUEUE_SIZE_PROPERTY_NAME; public static final int CONNECTION_POOL_SIZE_DEFAULT_VALUE = Rule.CONNECTION_POOL_SIZE_DEFAULT_VALUE; public static final int CONNECTION_MAX_WAIT_QUEUE_SIZE_DEFAULT_VALUE = Rule.MAX_WAIT_QUEUE_SIZE_DEFAULT_VALUE; + public static final String CONNECTION_TIMEOUT_SEC_PROPERTY_NAME = Rule.CONNECTION_TIMEOUT_SEC_PROPERTY_NAME; + public static final int CONNECTION_TIMEOUT_SEC_DEFAULT_VALUE = Rule.CONNECTION_TIMEOUT_SEC_DEFAULT_VALUE; private String destination; private List methods; private Pattern headersFilterPattern; @@ -37,10 +39,10 @@ public class HttpHook { private Integer connectionPoolSize = null; private Integer maxWaitQueueSize = null; private ProxyOptions proxyOptions = null; - + private Integer timeout = null; /** * Creates a new hook. - * + * * @param destination destination */ public HttpHook(String destination) { @@ -53,7 +55,7 @@ public HttpHook(String destination) { /** * The destination of the hook. - * + * * @return String */ public String getDestination() { @@ -62,7 +64,7 @@ public String getDestination() { /** * Sets the destination of the hook. - * + * * @param destination destination */ public void setDestination(String destination) { @@ -71,7 +73,7 @@ public void setDestination(String destination) { /** * Returns the methods which should pass the hook. - * + * * @return a list of HTTP methods or empty, if all methods do pass. */ public List getMethods() { @@ -80,7 +82,7 @@ public List getMethods() { /** * Sets the methods which should pass the hook. - * + * * @param methods a list of HTTP methods or empty, if all methods do pass. */ public void setMethods(List methods) { @@ -114,7 +116,7 @@ public Optional getExpirationTime() { /** * Sets the expiration time of this hook. - * + * * @param expirationTime expirationTime */ public void setExpirationTime(DateTime expirationTime) { @@ -123,7 +125,7 @@ public void setExpirationTime(DateTime expirationTime) { /** * Returns whether the hook forwards using the full initial url or only the appendix. - * + * * @return fullUrl */ public boolean isFullUrl() { @@ -132,7 +134,7 @@ public boolean isFullUrl() { /** * Sets whether the hook forwards using the full initial url or only the appendix. - * + * * @param fullUrl fullUrl */ public void setFullUrl(boolean fullUrl) { @@ -156,7 +158,7 @@ public void setFullUrl(boolean fullUrl) { /** * Returns the precompiled pattern, to match * a given url. - * + * * @return - a precompiled pattern */ public Pattern getFilter() { @@ -165,7 +167,7 @@ public Pattern getFilter() { /** * Set a regexp to filter the hook.
- * + * * @param regex - a regular expression */ public void setFilter(String regex) { @@ -302,6 +304,24 @@ public void setMaxWaitQueueSize(Integer maxWaitQueueSize) { this.maxWaitQueueSize = maxWaitQueueSize; } + /** + * @return Timeout for request. This may returning null in case there's no value + * specified. Callers may catch that by fall back to a default. + */ + public Integer getTimeout() { + return this.timeout; + } + + /** + * See {@link #getTimeout()}. + */ + public void setTimeout(Integer timeout) { + if (timeout != null && timeout < -1){ + throw new IllegalArgumentException("Values below -1 not valid."); + } + this.timeout = timeout; + } + /** * Get custom proxy options for this hook * diff --git a/gateleen-hook/src/main/java/org/swisspush/gateleen/hook/Route.java b/gateleen-hook/src/main/java/org/swisspush/gateleen/hook/Route.java index 48e8a3bd1..36d75f1c8 100755 --- a/gateleen-hook/src/main/java/org/swisspush/gateleen/hook/Route.java +++ b/gateleen-hook/src/main/java/org/swisspush/gateleen/hook/Route.java @@ -138,6 +138,16 @@ private void createRule() { rule.setMaxWaitQueueSize(maxWaitQueueSize); } } + { // Evaluate timeout + Integer timeout = httpHook.getTimeout(); + if (timeout == null) { + LOG.trace("No timeout specified for route '{}' using default of {}.", rule.getRuleIdentifier(), HttpHook.CONNECTION_TIMEOUT_SEC_DEFAULT_VALUE); + rule.setTimeout(1000 * HttpHook.CONNECTION_TIMEOUT_SEC_DEFAULT_VALUE); + } else { + LOG.trace("Using timeout {} for route '{}'.", timeout, rule.getRuleIdentifier()); + rule.setTimeout(timeout); + } + } if (!httpHook.getMethods().isEmpty()) { rule.setMethods(httpHook.getMethods().toArray(new String[httpHook.getMethods().size()])); @@ -250,7 +260,7 @@ public void cleanup() { /** * Returns the hook associated * with this route. - * + * * @return the hook */ public HttpHook getHook() { diff --git a/gateleen-hook/src/main/resources/gateleen_hooking_schema_hook b/gateleen-hook/src/main/resources/gateleen_hooking_schema_hook index ef5111d8c..aa126b297 100644 --- a/gateleen-hook/src/main/resources/gateleen_hooking_schema_hook +++ b/gateleen-hook/src/main/resources/gateleen_hooking_schema_hook @@ -94,7 +94,12 @@ "description": "The maximum number of requests allowed in the wait queue.", "type": "integer", "default": -1 - }, + }, + "timeout": { + "description": "The request timeout applied in seconds.", + "type": "integer", + "default": 30 + }, "collection": { "description": "only used for route hook", "type": "boolean" diff --git a/gateleen-hook/src/test/java/org/swisspush/gateleen/hook/HookHandlerTest.java b/gateleen-hook/src/test/java/org/swisspush/gateleen/hook/HookHandlerTest.java index ddf309bd5..2de34c20f 100644 --- a/gateleen-hook/src/test/java/org/swisspush/gateleen/hook/HookHandlerTest.java +++ b/gateleen-hook/src/test/java/org/swisspush/gateleen/hook/HookHandlerTest.java @@ -378,6 +378,41 @@ public void hookRegistration_usesDefaultExpiryIfExpireAfterHeaderIsNegativeNumbe } } + @Test + public void hookRegistration_RouteWithTimeout(TestContext testContext) { + // Initialize mock + final int[] statusCodePtr = new int[]{0}; + final String[] statusMessagePtr = new String[]{null}; + final HttpServerRequest request; + { // Mock request + final MultiMap requestHeaders = MultiMap.caseInsensitiveMultiMap(); + final Buffer requestBody = new BufferImpl(); + requestBody.setBytes(0, ("{" + + " \"methods\": [ \"PUT\" , \"DELETE\" ]," + + " \"destination\": \"/an/example/destination/\"," + + " \"timeout\": 42" + + "}").getBytes()); + + request = createSimpleRequest(HttpMethod.PUT, "/gateleen/example/_hooks/route/http/my-service/my-hook", + requestHeaders, requestBody, statusCodePtr, statusMessagePtr + ); + } + + // Trigger work + hookHandler.handle(request); + + // Assert request was ok + testContext.assertEquals(200, statusCodePtr[0]); + + { // Assert expiration time has same length as a valid date (including time zone) + final String storedHook = storage.getMockData().get(HOOK_ROOT_URI + "registrations/routes/+gateleen+example+_hooks+route+http+my-service+my-hook"); + testContext.assertNotNull(storedHook); + final Integer timeout = new JsonObject(storedHook).getJsonObject("hook").getInteger("timeout"); + testContext.assertNotNull(timeout); + testContext.assertEquals(42, timeout); + } + } + @Test public void hookRegistration_usesDefaultExpiryWhenHeaderContainsCorruptValue(TestContext testContext) { // In my opinion gateleen should answer with 400 in this case. But that's another topic. diff --git a/gateleen-hook/src/test/java/org/swisspush/gateleen/hook/HookSchemaTest.java b/gateleen-hook/src/test/java/org/swisspush/gateleen/hook/HookSchemaTest.java index d872ac8aa..8e2439b13 100644 --- a/gateleen-hook/src/test/java/org/swisspush/gateleen/hook/HookSchemaTest.java +++ b/gateleen-hook/src/test/java/org/swisspush/gateleen/hook/HookSchemaTest.java @@ -39,7 +39,8 @@ public void validMaximalHook() { " 'queueingStrategy':{'type':'reducedPropagation','intervalMs':1000}," + " 'collection':false," + " 'listable':true," + - " 'proxyOptions':{'type':'HTTP', 'host':'someHost', 'port':1234, 'username':'johndoe', 'password':'secret'}" + + " 'proxyOptions':{'type':'HTTP', 'host':'someHost', 'port':1234, 'username':'johndoe', 'password':'secret'}," + + " 'timeout':60" + "}"); Set valMsgs = schema.validate(json); diff --git a/gateleen-routing/src/main/java/org/swisspush/gateleen/routing/Rule.java b/gateleen-routing/src/main/java/org/swisspush/gateleen/routing/Rule.java index 186f0420c..5cc927712 100755 --- a/gateleen-routing/src/main/java/org/swisspush/gateleen/routing/Rule.java +++ b/gateleen-routing/src/main/java/org/swisspush/gateleen/routing/Rule.java @@ -13,8 +13,10 @@ public class Rule { public static final String CONNECTION_POOL_SIZE_PROPERTY_NAME = "connectionPoolSize"; public static final String MAX_WAIT_QUEUE_SIZE_PROPERTY_NAME = "maxWaitQueueSize"; + public static final String CONNECTION_TIMEOUT_SEC_PROPERTY_NAME = "timeout"; public static final int CONNECTION_POOL_SIZE_DEFAULT_VALUE = 50; public static final int MAX_WAIT_QUEUE_SIZE_DEFAULT_VALUE = -1; + public static final int CONNECTION_TIMEOUT_SEC_DEFAULT_VALUE = 30; private String scheme; private String host; private String metricName; diff --git a/gateleen-routing/src/main/java/org/swisspush/gateleen/routing/RuleFactory.java b/gateleen-routing/src/main/java/org/swisspush/gateleen/routing/RuleFactory.java index 793f65ea3..3350c759d 100755 --- a/gateleen-routing/src/main/java/org/swisspush/gateleen/routing/RuleFactory.java +++ b/gateleen-routing/src/main/java/org/swisspush/gateleen/routing/RuleFactory.java @@ -86,8 +86,7 @@ public List createRules(JsonObject rules) throws ValidationException { ruleObj.setPassword(basicAuth.getString("password")); } - int defaultTimeoutSec = 30; - ruleObj.setTimeout(1000 * rule.getInteger("timeout", defaultTimeoutSec)); + ruleObj.setTimeout(1000 * rule.getInteger(Rule.CONNECTION_TIMEOUT_SEC_PROPERTY_NAME, Rule.CONNECTION_TIMEOUT_SEC_DEFAULT_VALUE)); ruleObj.setKeepAliveTimeout(rule.getInteger("keepAliveTimeout", HttpClientOptions.DEFAULT_KEEP_ALIVE_TIMEOUT)); ruleObj.setPoolSize(rule.getInteger(Rule.CONNECTION_POOL_SIZE_PROPERTY_NAME, Rule.CONNECTION_POOL_SIZE_DEFAULT_VALUE)); ruleObj.setMaxWaitQueueSize(rule.getInteger(Rule.MAX_WAIT_QUEUE_SIZE_PROPERTY_NAME, Rule.MAX_WAIT_QUEUE_SIZE_DEFAULT_VALUE)); From 4158488fc89bb63749a50b2dcd9f4fe160480a66 Mon Sep 17 00:00:00 2001 From: guentherm Date: Fri, 23 Sep 2022 08:07:45 +0200 Subject: [PATCH 4/6] #473 PR improvements --- .../src/main/java/org/swisspush/gateleen/hook/HttpHook.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/gateleen-hook/src/main/java/org/swisspush/gateleen/hook/HttpHook.java b/gateleen-hook/src/main/java/org/swisspush/gateleen/hook/HttpHook.java index 7d947c753..1bdc81171 100755 --- a/gateleen-hook/src/main/java/org/swisspush/gateleen/hook/HttpHook.java +++ b/gateleen-hook/src/main/java/org/swisspush/gateleen/hook/HttpHook.java @@ -1,6 +1,7 @@ package org.swisspush.gateleen.hook; import io.vertx.core.net.ProxyOptions; +import javax.annotation.Nullable; import org.joda.time.DateTime; import org.swisspush.gateleen.core.http.HeaderFunction; import org.swisspush.gateleen.core.http.HeaderFunctions; @@ -305,9 +306,12 @@ public void setMaxWaitQueueSize(Integer maxWaitQueueSize) { } /** - * @return Timeout for request. This may returning null in case there's no value + * @return Timeout for request in milliseconds. Defines the time up until + * an ongoing request will be aborted if there was no reply given before. + * This may returning null in case there's no value * specified. Callers may catch that by fall back to a default. */ + @Nullable public Integer getTimeout() { return this.timeout; } From 6c725c291a271f33e0b956485205f3abdc24907d Mon Sep 17 00:00:00 2001 From: guentherm Date: Fri, 23 Sep 2022 13:46:31 +0200 Subject: [PATCH 5/6] #473 PR improvements --- .../src/main/java/org/swisspush/gateleen/hook/HttpHook.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gateleen-hook/src/main/java/org/swisspush/gateleen/hook/HttpHook.java b/gateleen-hook/src/main/java/org/swisspush/gateleen/hook/HttpHook.java index 1bdc81171..71751f55a 100755 --- a/gateleen-hook/src/main/java/org/swisspush/gateleen/hook/HttpHook.java +++ b/gateleen-hook/src/main/java/org/swisspush/gateleen/hook/HttpHook.java @@ -320,8 +320,8 @@ public Integer getTimeout() { * See {@link #getTimeout()}. */ public void setTimeout(Integer timeout) { - if (timeout != null && timeout < -1){ - throw new IllegalArgumentException("Values below -1 not valid."); + if (timeout != null && timeout <= 0){ + throw new IllegalArgumentException("Values <= 0 are not valid."); } this.timeout = timeout; } From 592dc71175c2626671408b563ff84457864a4585 Mon Sep 17 00:00:00 2001 From: markus Date: Tue, 23 Jan 2024 21:08:14 +0100 Subject: [PATCH 6/6] Allow File attachments on Playground Server by default --- .../main/java/org/swisspush/gateleen/playground/Server.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/gateleen-playground/src/main/java/org/swisspush/gateleen/playground/Server.java b/gateleen-playground/src/main/java/org/swisspush/gateleen/playground/Server.java index 6decb174b..e2353acb5 100755 --- a/gateleen-playground/src/main/java/org/swisspush/gateleen/playground/Server.java +++ b/gateleen-playground/src/main/java/org/swisspush/gateleen/playground/Server.java @@ -237,7 +237,8 @@ public void start() { SERVER_ROOT + "/admin/v1/contentTypeConstraints", Arrays.asList( new PatternHolder("application/json"), - new PatternHolder("application/x-www-form-urlencoded") + new PatternHolder("application/x-www-form-urlencoded"), + new PatternHolder("multipart/form-data") )); contentTypeConstraintHandler.initialize();