From dffde374546d7288e8c829fab715e28cc60a91aa Mon Sep 17 00:00:00 2001 From: Giannandrea Castaldi Date: Mon, 22 Jan 2024 11:40:02 +0100 Subject: [PATCH 01/18] #550 added QueueSplitterConfigurationParser --- .../splitter/QueueSplitterConfiguration.java | 93 ++++++++++ .../QueueSplitterConfigurationParser.java | 80 +++++++++ .../splitter/QueueSplitterHandler.java | 9 + .../QueueSplitterConfigurationParserTest.java | 161 ++++++++++++++++++ ...source_queuesplitter_configuration_invalid | 13 ++ ...e_queuesplitter_configuration_invalid_json | 10 ++ ...resource_queuesplitter_configuration_valid | 18 ++ ...rce_queuesplitter_configuration_with_props | 6 + 8 files changed, 390 insertions(+) create mode 100644 gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterConfiguration.java create mode 100644 gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterConfigurationParser.java create mode 100644 gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterHandler.java create mode 100644 gateleen-queue/src/test/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterConfigurationParserTest.java create mode 100644 gateleen-queue/src/test/resources/testresource_queuesplitter_configuration_invalid create mode 100644 gateleen-queue/src/test/resources/testresource_queuesplitter_configuration_invalid_json create mode 100644 gateleen-queue/src/test/resources/testresource_queuesplitter_configuration_valid create mode 100644 gateleen-queue/src/test/resources/testresource_queuesplitter_configuration_with_props diff --git a/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterConfiguration.java b/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterConfiguration.java new file mode 100644 index 000000000..430193088 --- /dev/null +++ b/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterConfiguration.java @@ -0,0 +1,93 @@ +package org.swisspush.gateleen.queue.queuing.splitter; + +import javax.annotation.Nullable; +import java.util.List; +import java.util.Objects; +import java.util.regex.Pattern; + +/** + * Container holding configuration values for {@link QueueSplitterHandler} identified + * by a queue pattern. + * + * @author https://github.com/gcastaldi [Giannandrea Castaldi] + */ +public class QueueSplitterConfiguration { + + private final Pattern queue; + + private final String postfixDelimiter; + + @Nullable + private final List postfixFromStatic; + + @Nullable + private final String postfixFromHeader; + + @Nullable + private final String postfixFromUrl; + + + public QueueSplitterConfiguration( + Pattern queue, + String postfixDelimiter, + @Nullable List postfixFromStatic, + @Nullable String postfixFromHeader, + @Nullable String postfixFromUrl) { + this.queue = queue; + this.postfixDelimiter = postfixDelimiter; + this.postfixFromStatic = postfixFromStatic; + this.postfixFromHeader = postfixFromHeader; + this.postfixFromUrl = postfixFromUrl; + } + + public Pattern getQueue() { + return queue; + } + + public String getPostfixDelimiter() { + return postfixDelimiter; + } + + @Nullable + public List getPostfixFromStatic() { + return postfixFromStatic; + } + + @Nullable + public String getPostfixFromHeader() { + return postfixFromHeader; + } + + @Nullable + public String getPostfixFromUrl() { + return postfixFromUrl; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + QueueSplitterConfiguration that = (QueueSplitterConfiguration) o; + return Objects.equals(queue, that.queue) && + Objects.equals(postfixDelimiter, that.postfixDelimiter) && + Objects.equals(postfixFromStatic, that.postfixFromStatic) && + Objects.equals(postfixFromHeader, that.postfixFromHeader) && + Objects.equals(postfixFromUrl, that.postfixFromUrl); + } + + @Override + public int hashCode() { + return Objects.hash(queue, postfixDelimiter, postfixFromStatic, postfixFromHeader, postfixFromUrl); + } + + @Override + public String toString() { + return "QueueSplitterConfiguration{" + + "queue=" + queue + + ", postfixDelimiter='" + postfixDelimiter + '\'' + + ", postfixFromStatic=" + postfixFromStatic + + ", postfixFromHeader='" + postfixFromHeader + '\'' + + ", postfixFromUrl='" + postfixFromUrl + '\'' + + '}'; + } +} diff --git a/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterConfigurationParser.java b/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterConfigurationParser.java new file mode 100644 index 000000000..c27251bca --- /dev/null +++ b/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterConfigurationParser.java @@ -0,0 +1,80 @@ +package org.swisspush.gateleen.queue.queuing.splitter; + +import io.vertx.core.buffer.Buffer; +import io.vertx.core.json.JsonArray; +import io.vertx.core.json.JsonObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.swisspush.gateleen.core.util.StringUtils; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +/** + * Parses the splitter configuration resource in to a list of {@link QueueSplitterConfiguration} + * + * @author https://github.com/gcastaldi [Giannandrea Castaldi] + */ +public class QueueSplitterConfigurationParser { + + private static final Logger log = LoggerFactory.getLogger(QueueSplitterConfigurationParser.class); + public static final String POSTFIX_FROM_STATIC_KEY = "postfixFromStatic"; + public static final String POSTFIX_FROM_HEADER_KEY = "postfixFromHeader"; + public static final String POSTFIX_FROM_URL_KEY = "postfixFromUrl"; + public static final String POSTFIX_DELIMITER_KEY = "postfixDelimiter"; + public static final String DEFAULT_POSTFIX_DELIMITER = "-"; + + static List parse(Buffer configurationResourceBuffer, Map properties) { + + JsonObject config; + List queueSplitterConfigurations = new ArrayList<>(); + try { + String resolvedConfiguration = StringUtils.replaceWildcardConfigs( + configurationResourceBuffer.toString(StandardCharsets.UTF_8), + properties + ); + config = new JsonObject(Buffer.buffer(resolvedConfiguration)); + } catch (Exception ex) { + log.warn("Could not replace wildcards with environment properties for queue splitter configurations or json invalid. Here the reason: {}", + ex.getMessage()); + return queueSplitterConfigurations; + } + + for (String queuePattern : config.fieldNames()) { + Pattern pattern = Pattern.compile(queuePattern); + JsonObject queueConfig = config.getJsonObject(queuePattern); + JsonArray postfixFromStatic = queueConfig.getJsonArray(POSTFIX_FROM_STATIC_KEY); + if (postfixFromStatic != null) { + List staticPostfixes = postfixFromStatic.stream().map(Object::toString).collect(Collectors.toList()); + queueSplitterConfigurations.add(new QueueSplitterConfiguration( + pattern, + queueConfig.getString("postfixDelimiter", DEFAULT_POSTFIX_DELIMITER), + staticPostfixes, + null, + null + )); + continue; + } + String postfixFromHeader = queueConfig.getString(POSTFIX_FROM_HEADER_KEY); + String postfixFromUrl = queueConfig.getString(POSTFIX_FROM_URL_KEY); + if (postfixFromHeader != null || postfixFromUrl != null) { + queueSplitterConfigurations.add(new QueueSplitterConfiguration( + pattern, + queueConfig.getString(POSTFIX_DELIMITER_KEY, DEFAULT_POSTFIX_DELIMITER), + null, + postfixFromHeader, + postfixFromUrl + )); + } else { + log.warn("Queue splitter configuration without a postfix definition"); + } + } + + return queueSplitterConfigurations; + } + +} diff --git a/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterHandler.java b/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterHandler.java new file mode 100644 index 000000000..2de873a43 --- /dev/null +++ b/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterHandler.java @@ -0,0 +1,9 @@ +package org.swisspush.gateleen.queue.queuing.splitter; + +/** + * Handler class for queues configured to be split in sub-queues. + * + * @author https://github.com/gcastaldi [Giannandrea Castaldi] + */ +public class QueueSplitterHandler { +} diff --git a/gateleen-queue/src/test/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterConfigurationParserTest.java b/gateleen-queue/src/test/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterConfigurationParserTest.java new file mode 100644 index 000000000..b08eefebc --- /dev/null +++ b/gateleen-queue/src/test/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterConfigurationParserTest.java @@ -0,0 +1,161 @@ +package org.swisspush.gateleen.queue.queuing.splitter; + +import io.vertx.core.buffer.Buffer; +import io.vertx.ext.unit.TestContext; +import io.vertx.ext.unit.junit.VertxUnitRunner; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.swisspush.gateleen.core.util.ResourcesUtils; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.regex.Pattern; + +/** + * Tests for {@link QueueSplitterConfigurationParser} class + * + * @author https://github.com/gcastaldi [Giannandrea Castaldi] + */ +@RunWith(VertxUnitRunner.class) +public class QueueSplitterConfigurationParserTest { + + private final String CONFIGURATION_VALID = ResourcesUtils.loadResource( + "testresource_queuesplitter_configuration_valid", + true + ); + + private final String CONFIGURATION_INVALID = ResourcesUtils.loadResource( + "testresource_queuesplitter_configuration_invalid", + true + ); + + private final String CONFIGURATION_INVALID_JSON = ResourcesUtils.loadResource( + "testresource_queuesplitter_configuration_invalid_json", + true + ); + + private final String CONFIGURATION_WITH_PROPS = ResourcesUtils.loadResource( + "testresource_queuesplitter_configuration_with_props", + true + ); + + @Test + public void parseWithAllValid(TestContext context) { + + // Given + Buffer configurationResourceBuffer = Buffer.buffer(CONFIGURATION_VALID); + HashMap properties = new HashMap<>(); + + // When + List configurations = QueueSplitterConfigurationParser.parse( + configurationResourceBuffer, + properties + ); + + // Then + context.assertEquals(3, configurations.size()); + + // Note that the order of the parsed configurations matters! + QueueSplitterConfiguration config_1 = configurations.get(0); + context.assertEquals(Pattern.compile("my-queue-1").pattern(), config_1.getQueue().pattern()); + context.assertEquals("-", config_1.getPostfixDelimiter()); + context.assertEquals(List.of("A", "B", "C", "D"), config_1.getPostfixFromStatic()); + context.assertNull(config_1.getPostfixFromHeader()); + context.assertNull(config_1.getPostfixFromUrl()); + + QueueSplitterConfiguration config_2 = configurations.get(1); + context.assertEquals(Pattern.compile("my-queue-[0-9]+").pattern(), config_2.getQueue().pattern()); + context.assertEquals("+", config_2.getPostfixDelimiter()); + context.assertNull(config_2.getPostfixFromStatic()); + context.assertEquals("{x-rp-deviceid}", config_2.getPostfixFromHeader()); + context.assertNull(config_2.getPostfixFromUrl()); + + QueueSplitterConfiguration config_3 = configurations.get(2); + context.assertEquals(Pattern.compile("my-queue-[a-zA-Z]+").pattern(), config_3.getQueue().pattern()); + context.assertEquals("_", config_3.getPostfixDelimiter()); + context.assertNull(config_3.getPostfixFromStatic()); + context.assertNull(config_3.getPostfixFromHeader()); + context.assertEquals(".*/path1/(.*)/path3/path4/.*", config_3.getPostfixFromUrl()); + } + + @Test + public void parseWithOneValidOneNot(TestContext context) { + + // Given + Buffer configurationResourceBuffer = Buffer.buffer(CONFIGURATION_INVALID); + HashMap properties = new HashMap<>(); + + // When + List configurations = QueueSplitterConfigurationParser.parse( + configurationResourceBuffer, + properties + ); + + // Then + context.assertEquals(1, configurations.size()); + + QueueSplitterConfiguration config_1 = configurations.get(0); + context.assertEquals(Pattern.compile("my-queue-1").pattern(), config_1.getQueue().pattern()); + context.assertEquals("-", config_1.getPostfixDelimiter()); + context.assertEquals(List.of("A", "B", "C", "D"), config_1.getPostfixFromStatic()); + context.assertNull(config_1.getPostfixFromHeader()); + context.assertNull(config_1.getPostfixFromUrl()); + } + @Test + public void parseWithInvalidJson(TestContext context) { + + // Given + Buffer configurationResourceBuffer = Buffer.buffer(CONFIGURATION_INVALID_JSON); + HashMap properties = new HashMap<>(); + + // When + List configurations = QueueSplitterConfigurationParser.parse( + configurationResourceBuffer, + properties + ); + + // Then + context.assertEquals(0, configurations.size()); + } + + @Test + public void parseWithValidAndProps(TestContext context) { + + // Given + Buffer configurationResourceBuffer = Buffer.buffer(CONFIGURATION_WITH_PROPS); + Map properties = Map.of("queue.splitter.delimiter", "_"); + + // When + List configurations = QueueSplitterConfigurationParser.parse( + configurationResourceBuffer, + properties + ); + + // Then + context.assertEquals(1, configurations.size()); + + QueueSplitterConfiguration config_1 = configurations.get(0); + context.assertEquals(Pattern.compile("my-queue-[0-9]+").pattern(), config_1.getQueue().pattern()); + context.assertEquals("_", config_1.getPostfixDelimiter()); + context.assertNull(config_1.getPostfixFromStatic()); + context.assertEquals("{x-rp-deviceid}", config_1.getPostfixFromHeader()); + context.assertNull(config_1.getPostfixFromUrl()); + } + @Test + public void parseWithValidAndMissingProps(TestContext context) { + + // Given + Buffer configurationResourceBuffer = Buffer.buffer(CONFIGURATION_WITH_PROPS); + Map properties = new HashMap<>(); + + // When + List configurations = QueueSplitterConfigurationParser.parse( + configurationResourceBuffer, + properties + ); + + // Then + context.assertEquals(0, configurations.size()); + } +} diff --git a/gateleen-queue/src/test/resources/testresource_queuesplitter_configuration_invalid b/gateleen-queue/src/test/resources/testresource_queuesplitter_configuration_invalid new file mode 100644 index 000000000..f4522fb7a --- /dev/null +++ b/gateleen-queue/src/test/resources/testresource_queuesplitter_configuration_invalid @@ -0,0 +1,13 @@ +{ + "my-queue-1" : { + "postfixFromStatic": [ + "A", + "B", + "C", + "D" + ] + }, + "my-queue-[0-9]+" : { + "postfixDelimiter": "+" + } +} \ No newline at end of file diff --git a/gateleen-queue/src/test/resources/testresource_queuesplitter_configuration_invalid_json b/gateleen-queue/src/test/resources/testresource_queuesplitter_configuration_invalid_json new file mode 100644 index 000000000..3fdbfbc82 --- /dev/null +++ b/gateleen-queue/src/test/resources/testresource_queuesplitter_configuration_invalid_json @@ -0,0 +1,10 @@ +{ + "my-queue-1" : { + "postfixFromStatic": [ + "A" + "B" + "C" + "D" + ] + } +} \ No newline at end of file diff --git a/gateleen-queue/src/test/resources/testresource_queuesplitter_configuration_valid b/gateleen-queue/src/test/resources/testresource_queuesplitter_configuration_valid new file mode 100644 index 000000000..c4ae7d7c1 --- /dev/null +++ b/gateleen-queue/src/test/resources/testresource_queuesplitter_configuration_valid @@ -0,0 +1,18 @@ +{ + "my-queue-1" : { + "postfixFromStatic": [ + "A", + "B", + "C", + "D" + ] + }, + "my-queue-[0-9]+" : { + "postfixDelimiter": "+", + "postfixFromHeader": "{x-rp-deviceid}" + }, + "my-queue-[a-zA-Z]+" : { + "postfixDelimiter": "_", + "postfixFromUrl": ".*/path1/(.*)/path3/path4/.*" + } +} \ No newline at end of file diff --git a/gateleen-queue/src/test/resources/testresource_queuesplitter_configuration_with_props b/gateleen-queue/src/test/resources/testresource_queuesplitter_configuration_with_props new file mode 100644 index 000000000..ae03103c8 --- /dev/null +++ b/gateleen-queue/src/test/resources/testresource_queuesplitter_configuration_with_props @@ -0,0 +1,6 @@ +{ + "my-queue-[0-9]+" : { + "postfixDelimiter": "${queue.splitter.delimiter}", + "postfixFromHeader": "{x-rp-deviceid}" + } +} \ No newline at end of file From 761da46dbf58ccaf9ce20aed7912e4a6301202aa Mon Sep 17 00:00:00 2001 From: Giannandrea Castaldi Date: Tue, 23 Jan 2024 15:02:22 +0100 Subject: [PATCH 02/18] #550 added QueueSplitter and used in QueuingHandler --- .../swisspush/gateleen/playground/Server.java | 6 +++ .../playground/server/admin/v1/queueSplitters | 18 ++++++++ .../queue/queuing/QueuingHandler.java | 25 ++++++++--- .../queue/queuing/splitter/QueueSplitter.java | 42 +++++++++++++++++++ .../splitter/QueueSplitterConfiguration.java | 2 +- .../splitter/QueueSplitterHandler.java | 9 ---- ...teleen_queue_splitter_configuration_schema | 32 ++++++++++++++ .../gateleen/runconfig/RunConfig.java | 16 +++++-- 8 files changed, 132 insertions(+), 18 deletions(-) create mode 100644 gateleen-playground/src/main/resources/playground/server/admin/v1/queueSplitters create mode 100644 gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitter.java delete mode 100644 gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterHandler.java create mode 100644 gateleen-queue/src/main/resources/gateleen_queue_splitter_configuration_schema diff --git a/gateleen-playground/src/main/java/org/swisspush/gateleen/playground/Server.java b/gateleen-playground/src/main/java/org/swisspush/gateleen/playground/Server.java index 70dad222d..1fa31a7aa 100755 --- a/gateleen-playground/src/main/java/org/swisspush/gateleen/playground/Server.java +++ b/gateleen-playground/src/main/java/org/swisspush/gateleen/playground/Server.java @@ -64,6 +64,7 @@ import org.swisspush.gateleen.queue.queuing.circuitbreaker.impl.QueueCircuitBreakerImpl; import org.swisspush.gateleen.queue.queuing.circuitbreaker.impl.RedisQueueCircuitBreakerStorage; import org.swisspush.gateleen.queue.queuing.circuitbreaker.util.QueueCircuitBreakerRulePatternToCircuitMapping; +import org.swisspush.gateleen.queue.queuing.splitter.QueueSplitter; import org.swisspush.gateleen.routing.*; import org.swisspush.gateleen.routing.auth.DefaultOAuthProvider; import org.swisspush.gateleen.runconfig.RunConfig; @@ -151,6 +152,8 @@ public class Server extends AbstractVerticle { private ContentTypeConstraintHandler contentTypeConstraintHandler; private CacheHandler cacheHandler; + private QueueSplitter queueSplitter; + public static void main(String[] args) { Vertx.vertx().deployVerticle("org.swisspush.gateleen.playground.Server", event -> LoggerFactory.getLogger(Server.class).info("[_] Gateleen - http://localhost:7012/gateleen/") @@ -320,6 +323,8 @@ public void start() { final QueueBrowser queueBrowser = new QueueBrowser(vertx, SERVER_ROOT + "/queuing", Address.redisquesAddress(), monitoringHandler); + queueSplitter = new QueueSplitter(configurationResourceManager, SERVER_ROOT + "/admin/v1/queueSplitters"); + LogController logController = new LogController(); logController.registerLogConfiguratorMBean(JMX_DOMAIN); @@ -344,6 +349,7 @@ public void start() { .loggingResourceManager(loggingResourceManager) .configurationResourceManager(configurationResourceManager) .queueCircuitBreakerConfigurationResourceManager(queueCircuitBreakerConfigurationResourceManager) + .queueSplitterHandler(queueSplitter) .schedulerResourceManager(schedulerResourceManager) .zipExtractHandler(zipExtractHandler) .delegateHandler(delegateHandler) diff --git a/gateleen-playground/src/main/resources/playground/server/admin/v1/queueSplitters b/gateleen-playground/src/main/resources/playground/server/admin/v1/queueSplitters new file mode 100644 index 000000000..c4ae7d7c1 --- /dev/null +++ b/gateleen-playground/src/main/resources/playground/server/admin/v1/queueSplitters @@ -0,0 +1,18 @@ +{ + "my-queue-1" : { + "postfixFromStatic": [ + "A", + "B", + "C", + "D" + ] + }, + "my-queue-[0-9]+" : { + "postfixDelimiter": "+", + "postfixFromHeader": "{x-rp-deviceid}" + }, + "my-queue-[a-zA-Z]+" : { + "postfixDelimiter": "_", + "postfixFromUrl": ".*/path1/(.*)/path3/path4/.*" + } +} \ No newline at end of file diff --git a/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/QueuingHandler.java b/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/QueuingHandler.java index a44fb7311..87bb1d5a4 100755 --- a/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/QueuingHandler.java +++ b/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/QueuingHandler.java @@ -11,6 +11,7 @@ import org.swisspush.gateleen.core.util.StatusCode; import org.swisspush.gateleen.monitoring.MonitoringHandler; import org.swisspush.gateleen.queue.duplicate.DuplicateCheckHandler; +import org.swisspush.gateleen.queue.queuing.splitter.QueueSplitter; import static org.swisspush.redisques.util.RedisquesAPI.buildCheckOperation; @@ -34,16 +35,30 @@ public static boolean isQueued(HttpServerRequest request) { private final HttpServerRequest request; private final Vertx vertx; private final RedisProvider redisProvider; + private final QueueSplitter queueSplitter; - public QueuingHandler(Vertx vertx, RedisProvider redisProvider, HttpServerRequest request, MonitoringHandler monitoringHandler) { - this(vertx, redisProvider, request, new QueueClient(vertx, monitoringHandler)); + public QueuingHandler( + Vertx vertx, + RedisProvider redisProvider, + HttpServerRequest request, + MonitoringHandler monitoringHandler, + QueueSplitter queueSplitter + ) { + this(vertx, redisProvider, request, new QueueClient(vertx, monitoringHandler), queueSplitter); } - public QueuingHandler(Vertx vertx, RedisProvider redisProvider, HttpServerRequest request, RequestQueue requestQueue) { + public QueuingHandler( + Vertx vertx, + RedisProvider redisProvider, + HttpServerRequest request, + RequestQueue requestQueue, + QueueSplitter queueSplitter + ) { this.request = request; this.vertx = vertx; this.redisProvider = redisProvider; this.requestQueue = requestQueue; + this.queueSplitter = queueSplitter; } @Override @@ -61,12 +76,12 @@ public void handle(final Buffer buffer) { request.response().setStatusMessage(StatusCode.ACCEPTED.getStatusMessage()); request.response().end(); } else { - requestQueue.enqueue(request, headers, buffer, queue); + requestQueue.enqueue(request, headers, buffer, queueSplitter.handle(queue)); } }); } else { - requestQueue.enqueue(request, headers, buffer, queue); + requestQueue.enqueue(request, headers, buffer, queueSplitter.handle(queue)); } } diff --git a/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitter.java b/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitter.java new file mode 100644 index 000000000..b49001e6a --- /dev/null +++ b/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitter.java @@ -0,0 +1,42 @@ +package org.swisspush.gateleen.queue.queuing.splitter; + +import io.vertx.core.buffer.Buffer; +import org.swisspush.gateleen.core.configuration.ConfigurationResourceConsumer; +import org.swisspush.gateleen.core.configuration.ConfigurationResourceManager; + +/** + * Class for queues configured to be split in sub-queues. + * + * @author https://github.com/gcastaldi [Giannandrea Castaldi] + */ +public class QueueSplitter extends ConfigurationResourceConsumer { + + public QueueSplitter( + ConfigurationResourceManager configurationResourceManager, + String configResourceUri + ) { + this(configurationResourceManager, configResourceUri, "gateleen_queue_splitter_configuration_schema"); + } + + public QueueSplitter( + ConfigurationResourceManager configurationResourceManager, + String configResourceUri, + String schemaResourceName + ) { + super(configurationResourceManager, configResourceUri, schemaResourceName); + } + + @Override + public void resourceChanged(String resourceUri, Buffer resource) { + + } + + @Override + public void resourceRemoved(String resourceUri) { + + } + + public String handle(final String queue) { + return queue + "-1"; + } +} diff --git a/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterConfiguration.java b/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterConfiguration.java index 430193088..d336679c4 100644 --- a/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterConfiguration.java +++ b/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterConfiguration.java @@ -6,7 +6,7 @@ import java.util.regex.Pattern; /** - * Container holding configuration values for {@link QueueSplitterHandler} identified + * Container holding configuration values for {@link QueueSplitter} identified * by a queue pattern. * * @author https://github.com/gcastaldi [Giannandrea Castaldi] diff --git a/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterHandler.java b/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterHandler.java deleted file mode 100644 index 2de873a43..000000000 --- a/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterHandler.java +++ /dev/null @@ -1,9 +0,0 @@ -package org.swisspush.gateleen.queue.queuing.splitter; - -/** - * Handler class for queues configured to be split in sub-queues. - * - * @author https://github.com/gcastaldi [Giannandrea Castaldi] - */ -public class QueueSplitterHandler { -} diff --git a/gateleen-queue/src/main/resources/gateleen_queue_splitter_configuration_schema b/gateleen-queue/src/main/resources/gateleen_queue_splitter_configuration_schema new file mode 100644 index 000000000..ab8dbb66b --- /dev/null +++ b/gateleen-queue/src/main/resources/gateleen_queue_splitter_configuration_schema @@ -0,0 +1,32 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/QueueSplitter" + }, + "definitions": { + "QueueSplitter": { + "description": "A single queue splitter configuration", + "type": "object", + "properties": { + "postfixDelimiter": { + "type": "string" + }, + "postfixFromStatic": { + "type": "array", + "items": { + "type": "string" + }, + "minItems": 1, + "uniqueItems": true + }, + "postfixFromHeader": { + "type": "string" + } + }, + "postfixFromUrl": { + "type": "string" + } + } + } +} \ No newline at end of file diff --git a/gateleen-runconfig/src/main/java/org/swisspush/gateleen/runconfig/RunConfig.java b/gateleen-runconfig/src/main/java/org/swisspush/gateleen/runconfig/RunConfig.java index c5e9c593d..6858e5b55 100755 --- a/gateleen-runconfig/src/main/java/org/swisspush/gateleen/runconfig/RunConfig.java +++ b/gateleen-runconfig/src/main/java/org/swisspush/gateleen/runconfig/RunConfig.java @@ -38,6 +38,7 @@ import org.swisspush.gateleen.queue.queuing.QueueBrowser; import org.swisspush.gateleen.queue.queuing.QueuingHandler; import org.swisspush.gateleen.queue.queuing.circuitbreaker.configuration.QueueCircuitBreakerConfigurationResourceManager; +import org.swisspush.gateleen.queue.queuing.splitter.QueueSplitter; import org.swisspush.gateleen.routing.CustomHttpResponseHandler; import org.swisspush.gateleen.routing.Router; import org.swisspush.gateleen.scheduler.SchedulerResourceManager; @@ -91,6 +92,7 @@ public class RunConfig { private final LoggingResourceManager loggingResourceManager; private final ConfigurationResourceManager configurationResourceManager; private final QueueCircuitBreakerConfigurationResourceManager queueCircuitBreakerConfigurationResourceManager; + private QueueSplitter queueSplitter; private final EventBusHandler eventBusHandler; private final ValidationHandler validationHandler; private final HookHandler hookHandler; @@ -115,7 +117,7 @@ public RunConfig(Vertx vertx, RedisProvider redisProvider, Class verticleClass, ValidationResourceManager validationResourceManager, LoggingResourceManager loggingResourceManager, ConfigurationResourceManager configurationResourceManager, QueueCircuitBreakerConfigurationResourceManager queueCircuitBreakerConfigurationResourceManager, - EventBusHandler eventBusHandler, ValidationHandler validationHandler, HookHandler hookHandler, + QueueSplitter queueSplitter, EventBusHandler eventBusHandler, ValidationHandler validationHandler, HookHandler hookHandler, UserProfileHandler userProfileHandler, RoleProfileHandler roleProfileHandler, ExpansionHandler expansionHandler, DeltaHandler deltaHandler, Authorizer authorizer, CopyResourceHandler copyResourceHandler, QoSHandler qosHandler, PropertyHandler propertyHandler, ZipExtractHandler zipExtractHandler, @@ -134,6 +136,7 @@ public RunConfig(Vertx vertx, RedisProvider redisProvider, Class verticleClass, this.loggingResourceManager = loggingResourceManager; this.configurationResourceManager = configurationResourceManager; this.queueCircuitBreakerConfigurationResourceManager = queueCircuitBreakerConfigurationResourceManager; + this.queueSplitter = queueSplitter; this.eventBusHandler = eventBusHandler; this.validationHandler = validationHandler; this.hookHandler = hookHandler; @@ -168,6 +171,7 @@ private RunConfig(RunConfigBuilder builder) { builder.loggingResourceManager, builder.configurationResourceManager, builder.queueCircuitBreakerConfigurationResourceManager, + builder.queueSplitter, builder.eventBusHandler, builder.validationHandler, builder.hookHandler, @@ -225,6 +229,7 @@ public static class RunConfigBuilder { private LoggingResourceManager loggingResourceManager; private ConfigurationResourceManager configurationResourceManager; private QueueCircuitBreakerConfigurationResourceManager queueCircuitBreakerConfigurationResourceManager; + private QueueSplitter queueSplitter; private EventBusHandler eventBusHandler; private KafkaHandler kafkaHandler; private CustomHttpResponseHandler customHttpResponseHandler; @@ -277,6 +282,11 @@ public RunConfigBuilder queueCircuitBreakerConfigurationResourceManager(QueueCir return this; } + public RunConfigBuilder queueSplitterHandler(QueueSplitter queueSplitter) { + this.queueSplitter = queueSplitter; + return this; + } + public RunConfigBuilder eventBusHandler(EventBusHandler eventBusHandler) { this.eventBusHandler = eventBusHandler; return this; @@ -593,11 +603,11 @@ private void handleRequest(final RoutingContext ctx) { return; } if (PackingHandler.isPacked(request)) { - request.bodyHandler(new PackingHandler(request, new QueuingHandler(vertx, redisProvider, request, monitoringHandler))); + request.bodyHandler(new PackingHandler(request, new QueuingHandler(vertx, redisProvider, request, monitoringHandler, queueSplitter))); } else { if (QueuingHandler.isQueued(request)) { setISO8601Timestamps(request); - request.bodyHandler(new QueuingHandler(vertx, redisProvider, request, monitoringHandler)); + request.bodyHandler(new QueuingHandler(vertx, redisProvider, request, monitoringHandler, queueSplitter)); } else { if (cacheHandler != null && cacheHandler.handle(request)) { return; From 2c28f271c01a723fbc668eec528d2b1beeb65b91 Mon Sep 17 00:00:00 2001 From: Giannandrea Castaldi Date: Tue, 23 Jan 2024 16:05:16 +0100 Subject: [PATCH 03/18] #550 extracted interface QueueSplitter with also NoOp implementation --- .../swisspush/gateleen/playground/Server.java | 5 ++- .../queue/queuing/QueuingHandler.java | 13 +++++- .../queuing/splitter/NoOpQueueSplitter.java | 15 +++++++ .../queue/queuing/splitter/QueueSplitter.java | 45 +++++-------------- .../splitter/QueueSplitterConfiguration.java | 2 +- .../queuing/splitter/QueueSplitterImpl.java | 44 ++++++++++++++++++ .../gateleen/runconfig/RunConfig.java | 7 +-- 7 files changed, 88 insertions(+), 43 deletions(-) create mode 100644 gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/NoOpQueueSplitter.java create mode 100644 gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterImpl.java diff --git a/gateleen-playground/src/main/java/org/swisspush/gateleen/playground/Server.java b/gateleen-playground/src/main/java/org/swisspush/gateleen/playground/Server.java index 1fa31a7aa..8b62d42d3 100755 --- a/gateleen-playground/src/main/java/org/swisspush/gateleen/playground/Server.java +++ b/gateleen-playground/src/main/java/org/swisspush/gateleen/playground/Server.java @@ -65,6 +65,7 @@ import org.swisspush.gateleen.queue.queuing.circuitbreaker.impl.RedisQueueCircuitBreakerStorage; import org.swisspush.gateleen.queue.queuing.circuitbreaker.util.QueueCircuitBreakerRulePatternToCircuitMapping; import org.swisspush.gateleen.queue.queuing.splitter.QueueSplitter; +import org.swisspush.gateleen.queue.queuing.splitter.QueueSplitterImpl; import org.swisspush.gateleen.routing.*; import org.swisspush.gateleen.routing.auth.DefaultOAuthProvider; import org.swisspush.gateleen.runconfig.RunConfig; @@ -323,7 +324,7 @@ public void start() { final QueueBrowser queueBrowser = new QueueBrowser(vertx, SERVER_ROOT + "/queuing", Address.redisquesAddress(), monitoringHandler); - queueSplitter = new QueueSplitter(configurationResourceManager, SERVER_ROOT + "/admin/v1/queueSplitters"); + queueSplitter = new QueueSplitterImpl(configurationResourceManager, SERVER_ROOT + "/admin/v1/queueSplitters"); LogController logController = new LogController(); logController.registerLogConfiguratorMBean(JMX_DOMAIN); @@ -349,7 +350,7 @@ public void start() { .loggingResourceManager(loggingResourceManager) .configurationResourceManager(configurationResourceManager) .queueCircuitBreakerConfigurationResourceManager(queueCircuitBreakerConfigurationResourceManager) - .queueSplitterHandler(queueSplitter) + .queueSplitter(queueSplitter) .schedulerResourceManager(schedulerResourceManager) .zipExtractHandler(zipExtractHandler) .delegateHandler(delegateHandler) diff --git a/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/QueuingHandler.java b/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/QueuingHandler.java index 87bb1d5a4..72f4417c3 100755 --- a/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/QueuingHandler.java +++ b/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/QueuingHandler.java @@ -11,6 +11,7 @@ import org.swisspush.gateleen.core.util.StatusCode; import org.swisspush.gateleen.monitoring.MonitoringHandler; import org.swisspush.gateleen.queue.duplicate.DuplicateCheckHandler; +import org.swisspush.gateleen.queue.queuing.splitter.NoOpQueueSplitter; import org.swisspush.gateleen.queue.queuing.splitter.QueueSplitter; import static org.swisspush.redisques.util.RedisquesAPI.buildCheckOperation; @@ -37,6 +38,14 @@ public static boolean isQueued(HttpServerRequest request) { private final RedisProvider redisProvider; private final QueueSplitter queueSplitter; + public QueuingHandler( + Vertx vertx, + RedisProvider redisProvider, + HttpServerRequest request, + MonitoringHandler monitoringHandler + ) { + this(vertx, redisProvider, request, new QueueClient(vertx, monitoringHandler), new NoOpQueueSplitter()); + } public QueuingHandler( Vertx vertx, RedisProvider redisProvider, @@ -76,12 +85,12 @@ public void handle(final Buffer buffer) { request.response().setStatusMessage(StatusCode.ACCEPTED.getStatusMessage()); request.response().end(); } else { - requestQueue.enqueue(request, headers, buffer, queueSplitter.handle(queue)); + requestQueue.enqueue(request, headers, buffer, queueSplitter.convertToSubQueue(queue)); } }); } else { - requestQueue.enqueue(request, headers, buffer, queueSplitter.handle(queue)); + requestQueue.enqueue(request, headers, buffer, queueSplitter.convertToSubQueue(queue)); } } diff --git a/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/NoOpQueueSplitter.java b/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/NoOpQueueSplitter.java new file mode 100644 index 000000000..cbfa2326e --- /dev/null +++ b/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/NoOpQueueSplitter.java @@ -0,0 +1,15 @@ +package org.swisspush.gateleen.queue.queuing.splitter; + +/** + * {@inheritDoc} + */ +public class NoOpQueueSplitter implements QueueSplitter { + + @Override + /** + * {@inheritDoc} + */ + public String convertToSubQueue(String queue) { + return queue; + } +} diff --git a/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitter.java b/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitter.java index b49001e6a..bb3159866 100644 --- a/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitter.java +++ b/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitter.java @@ -1,42 +1,17 @@ package org.swisspush.gateleen.queue.queuing.splitter; -import io.vertx.core.buffer.Buffer; -import org.swisspush.gateleen.core.configuration.ConfigurationResourceConsumer; -import org.swisspush.gateleen.core.configuration.ConfigurationResourceManager; - /** - * Class for queues configured to be split in sub-queues. + * Interface for queues configured to be split in sub-queues. The method {@link QueueSplitter#convertToSubQueue(String)} + * evaluates the convert of the queue name in a sub-queue name. * * @author https://github.com/gcastaldi [Giannandrea Castaldi] */ -public class QueueSplitter extends ConfigurationResourceConsumer { - - public QueueSplitter( - ConfigurationResourceManager configurationResourceManager, - String configResourceUri - ) { - this(configurationResourceManager, configResourceUri, "gateleen_queue_splitter_configuration_schema"); - } - - public QueueSplitter( - ConfigurationResourceManager configurationResourceManager, - String configResourceUri, - String schemaResourceName - ) { - super(configurationResourceManager, configResourceUri, schemaResourceName); - } - - @Override - public void resourceChanged(String resourceUri, Buffer resource) { - - } - - @Override - public void resourceRemoved(String resourceUri) { - - } - - public String handle(final String queue) { - return queue + "-1"; - } +public interface QueueSplitter { + /** + * Convert the queue name in a sub-queue name. If not necessary maintains the initial queue name. + * + * @param queue + * @return sub-queue name + */ + String convertToSubQueue(String queue); } diff --git a/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterConfiguration.java b/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterConfiguration.java index d336679c4..025060cdf 100644 --- a/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterConfiguration.java +++ b/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterConfiguration.java @@ -6,7 +6,7 @@ import java.util.regex.Pattern; /** - * Container holding configuration values for {@link QueueSplitter} identified + * Container holding configuration values for {@link QueueSplitterImpl} identified * by a queue pattern. * * @author https://github.com/gcastaldi [Giannandrea Castaldi] diff --git a/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterImpl.java b/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterImpl.java new file mode 100644 index 000000000..6a6d44d68 --- /dev/null +++ b/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterImpl.java @@ -0,0 +1,44 @@ +package org.swisspush.gateleen.queue.queuing.splitter; + +import io.vertx.core.buffer.Buffer; +import org.swisspush.gateleen.core.configuration.ConfigurationResourceConsumer; +import org.swisspush.gateleen.core.configuration.ConfigurationResourceManager; + +/** + * {@inheritDoc} + */ +public class QueueSplitterImpl extends ConfigurationResourceConsumer implements QueueSplitter { + + public QueueSplitterImpl( + ConfigurationResourceManager configurationResourceManager, + String configResourceUri + ) { + this(configurationResourceManager, configResourceUri, "gateleen_queue_splitter_configuration_schema"); + } + + public QueueSplitterImpl( + ConfigurationResourceManager configurationResourceManager, + String configResourceUri, + String schemaResourceName + ) { + super(configurationResourceManager, configResourceUri, schemaResourceName); + } + + @Override + public void resourceChanged(String resourceUri, Buffer resource) { + + } + + @Override + public void resourceRemoved(String resourceUri) { + + } + + /** + * {@inheritDoc} + */ + @Override + public String convertToSubQueue(final String queue) { + return queue + "-1"; + } +} diff --git a/gateleen-runconfig/src/main/java/org/swisspush/gateleen/runconfig/RunConfig.java b/gateleen-runconfig/src/main/java/org/swisspush/gateleen/runconfig/RunConfig.java index 6858e5b55..98ec64e84 100755 --- a/gateleen-runconfig/src/main/java/org/swisspush/gateleen/runconfig/RunConfig.java +++ b/gateleen-runconfig/src/main/java/org/swisspush/gateleen/runconfig/RunConfig.java @@ -38,6 +38,7 @@ import org.swisspush.gateleen.queue.queuing.QueueBrowser; import org.swisspush.gateleen.queue.queuing.QueuingHandler; import org.swisspush.gateleen.queue.queuing.circuitbreaker.configuration.QueueCircuitBreakerConfigurationResourceManager; +import org.swisspush.gateleen.queue.queuing.splitter.NoOpQueueSplitter; import org.swisspush.gateleen.queue.queuing.splitter.QueueSplitter; import org.swisspush.gateleen.routing.CustomHttpResponseHandler; import org.swisspush.gateleen.routing.Router; @@ -92,7 +93,7 @@ public class RunConfig { private final LoggingResourceManager loggingResourceManager; private final ConfigurationResourceManager configurationResourceManager; private final QueueCircuitBreakerConfigurationResourceManager queueCircuitBreakerConfigurationResourceManager; - private QueueSplitter queueSplitter; + private final QueueSplitter queueSplitter; private final EventBusHandler eventBusHandler; private final ValidationHandler validationHandler; private final HookHandler hookHandler; @@ -229,7 +230,7 @@ public static class RunConfigBuilder { private LoggingResourceManager loggingResourceManager; private ConfigurationResourceManager configurationResourceManager; private QueueCircuitBreakerConfigurationResourceManager queueCircuitBreakerConfigurationResourceManager; - private QueueSplitter queueSplitter; + private QueueSplitter queueSplitter = new NoOpQueueSplitter(); private EventBusHandler eventBusHandler; private KafkaHandler kafkaHandler; private CustomHttpResponseHandler customHttpResponseHandler; @@ -282,7 +283,7 @@ public RunConfigBuilder queueCircuitBreakerConfigurationResourceManager(QueueCir return this; } - public RunConfigBuilder queueSplitterHandler(QueueSplitter queueSplitter) { + public RunConfigBuilder queueSplitter(QueueSplitter queueSplitter) { this.queueSplitter = queueSplitter; return this; } From c385935ddcb072f089aad255b03258145b6022f8 Mon Sep 17 00:00:00 2001 From: Giannandrea Castaldi Date: Wed, 24 Jan 2024 17:01:42 +0100 Subject: [PATCH 04/18] #550 added split executors --- .../swisspush/gateleen/playground/Server.java | 1 + .../queue/queuing/QueuingHandler.java | 4 +- .../queuing/splitter/NoOpQueueSplitter.java | 13 +- .../queue/queuing/splitter/QueueSplitter.java | 23 ++- .../splitter/QueueSplitterConfiguration.java | 8 + .../queuing/splitter/QueueSplitterImpl.java | 67 ++++++++- .../executors/QueueSplitExecutor.java | 10 ++ .../executors/QueueSplitExecutorBase.java | 17 +++ .../executors/QueueSplitExecutorFromList.java | 30 ++++ .../QueueSplitExecutorFromRequest.java | 35 +++++ .../QueueSplitterConfigurationParserTest.java | 8 +- .../QueueSplitExecutorFromListTest.java | 92 ++++++++++++ .../QueueSplitExecutorFromRequestTest.java | 141 ++++++++++++++++++ ...resource_queuesplitter_configuration_valid | 2 +- 14 files changed, 430 insertions(+), 21 deletions(-) create mode 100644 gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/executors/QueueSplitExecutor.java create mode 100644 gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/executors/QueueSplitExecutorBase.java create mode 100644 gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/executors/QueueSplitExecutorFromList.java create mode 100644 gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/executors/QueueSplitExecutorFromRequest.java create mode 100644 gateleen-queue/src/test/java/org/swisspush/gateleen/queue/queuing/splitter/executors/QueueSplitExecutorFromListTest.java create mode 100644 gateleen-queue/src/test/java/org/swisspush/gateleen/queue/queuing/splitter/executors/QueueSplitExecutorFromRequestTest.java diff --git a/gateleen-playground/src/main/java/org/swisspush/gateleen/playground/Server.java b/gateleen-playground/src/main/java/org/swisspush/gateleen/playground/Server.java index 8b62d42d3..a339adb67 100755 --- a/gateleen-playground/src/main/java/org/swisspush/gateleen/playground/Server.java +++ b/gateleen-playground/src/main/java/org/swisspush/gateleen/playground/Server.java @@ -325,6 +325,7 @@ public void start() { monitoringHandler); queueSplitter = new QueueSplitterImpl(configurationResourceManager, SERVER_ROOT + "/admin/v1/queueSplitters"); + queueSplitter.initialize(); LogController logController = new LogController(); logController.registerLogConfiguratorMBean(JMX_DOMAIN); diff --git a/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/QueuingHandler.java b/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/QueuingHandler.java index 72f4417c3..b548a9c76 100755 --- a/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/QueuingHandler.java +++ b/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/QueuingHandler.java @@ -85,12 +85,12 @@ public void handle(final Buffer buffer) { request.response().setStatusMessage(StatusCode.ACCEPTED.getStatusMessage()); request.response().end(); } else { - requestQueue.enqueue(request, headers, buffer, queueSplitter.convertToSubQueue(queue)); + requestQueue.enqueue(request, headers, buffer, queueSplitter.convertToSubQueue(queue, request)); } }); } else { - requestQueue.enqueue(request, headers, buffer, queueSplitter.convertToSubQueue(queue)); + requestQueue.enqueue(request, headers, buffer, queueSplitter.convertToSubQueue(queue, request)); } } diff --git a/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/NoOpQueueSplitter.java b/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/NoOpQueueSplitter.java index cbfa2326e..b494c7b68 100644 --- a/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/NoOpQueueSplitter.java +++ b/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/NoOpQueueSplitter.java @@ -1,15 +1,26 @@ package org.swisspush.gateleen.queue.queuing.splitter; +import io.vertx.core.Future; +import io.vertx.core.Promise; +import io.vertx.core.http.HttpServerRequest; + /** * {@inheritDoc} */ public class NoOpQueueSplitter implements QueueSplitter { + @Override + public Future initialize() { + Promise promise = Promise.promise(); + promise.complete(); + return promise.future(); + } + @Override /** * {@inheritDoc} */ - public String convertToSubQueue(String queue) { + public String convertToSubQueue(String queue, HttpServerRequest request) { return queue; } } diff --git a/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitter.java b/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitter.java index bb3159866..7d94b1b64 100644 --- a/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitter.java +++ b/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitter.java @@ -1,17 +1,24 @@ package org.swisspush.gateleen.queue.queuing.splitter; +import io.vertx.core.Future; +import io.vertx.core.http.HttpServerRequest; + /** - * Interface for queues configured to be split in sub-queues. The method {@link QueueSplitter#convertToSubQueue(String)} + * Interface for queues configured to be split in sub-queues. The method {@link QueueSplitter#convertToSubQueue(String, HttpServerRequest)} * evaluates the convert of the queue name in a sub-queue name. * * @author https://github.com/gcastaldi [Giannandrea Castaldi] */ public interface QueueSplitter { - /** - * Convert the queue name in a sub-queue name. If not necessary maintains the initial queue name. - * - * @param queue - * @return sub-queue name - */ - String convertToSubQueue(String queue); + + public Future initialize(); + + /** + * Convert the queue name in a sub-queue name. If not necessary maintains the initial queue name. + * + * @param queue + * @param request + * @return sub-queue name + */ + String convertToSubQueue(String queue, HttpServerRequest request); } diff --git a/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterConfiguration.java b/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterConfiguration.java index 025060cdf..16cc164a3 100644 --- a/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterConfiguration.java +++ b/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterConfiguration.java @@ -90,4 +90,12 @@ public String toString() { ", postfixFromUrl='" + postfixFromUrl + '\'' + '}'; } + + public boolean isSplitStatic() { + return postfixFromStatic != null && !postfixFromStatic.isEmpty(); + } + + public boolean isSplitFromRequest() { + return postfixFromHeader != null || postfixFromUrl != null; + } } diff --git a/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterImpl.java b/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterImpl.java index 6a6d44d68..7b9463562 100644 --- a/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterImpl.java +++ b/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterImpl.java @@ -1,44 +1,95 @@ package org.swisspush.gateleen.queue.queuing.splitter; +import io.vertx.core.Future; +import io.vertx.core.Promise; import io.vertx.core.buffer.Buffer; +import io.vertx.core.http.HttpServerRequest; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.swisspush.gateleen.core.configuration.ConfigurationResourceConsumer; import org.swisspush.gateleen.core.configuration.ConfigurationResourceManager; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + /** * {@inheritDoc} */ public class QueueSplitterImpl extends ConfigurationResourceConsumer implements QueueSplitter { + private final Logger log = LoggerFactory.getLogger(QueueSplitterImpl.class); + + private final Map properties; + + private boolean initialized = false; + public QueueSplitterImpl( ConfigurationResourceManager configurationResourceManager, String configResourceUri ) { - this(configurationResourceManager, configResourceUri, "gateleen_queue_splitter_configuration_schema"); + this(configurationResourceManager, configResourceUri, new HashMap<>()); } public QueueSplitterImpl( ConfigurationResourceManager configurationResourceManager, String configResourceUri, - String schemaResourceName + Map properties ) { - super(configurationResourceManager, configResourceUri, schemaResourceName); + super(configurationResourceManager, configResourceUri, "gateleen_queue_splitter_configuration_schema"); + this.properties = properties; } - @Override - public void resourceChanged(String resourceUri, Buffer resource) { + public Future initialize() { + Promise promise = Promise.promise(); + configurationResourceManager().getRegisteredResource(configResourceUri()).onComplete((event -> { + if (event.succeeded() && event.result().isPresent()) { + initializeQueueSplitterConfiguration(event.result().get()).onComplete((event1 -> promise.complete())); + } else { + log.warn("No queue splitter configuration resource with uri '{}' found. Unable to setup kafka configuration correctly", configResourceUri()); + promise.complete(); + } + })); + return promise.future(); + } + public boolean isInitialized() { + return initialized; } - @Override - public void resourceRemoved(String resourceUri) { + private Future initializeQueueSplitterConfiguration(Buffer configuration) { + Promise promise = Promise.promise(); + final List kafkaConfigurations = QueueSplitterConfigurationParser.parse(configuration, properties); + + // TODO: release all splitters and creates new ones + return promise.future(); } /** * {@inheritDoc} */ @Override - public String convertToSubQueue(final String queue) { + public String convertToSubQueue(final String queue, HttpServerRequest request) { return queue + "-1"; } + + @Override + public void resourceChanged(String resourceUri, Buffer resource) { + if (configResourceUri() != null && configResourceUri().equals(resourceUri)) { + log.info("Queue splitter configuration resource {} was updated. Going to initialize with new configuration", resourceUri); + initializeQueueSplitterConfiguration(resource); + } + } + + @Override + public void resourceRemoved(String resourceUri) { + if (configResourceUri() != null && configResourceUri().equals(resourceUri)) { + log.info("Queue splitter configuration resource {} was removed. Going to close all kafka producers", resourceUri); + + // TODO: release all splitters and creates new ones + + initialized = false; + } + } } diff --git a/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/executors/QueueSplitExecutor.java b/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/executors/QueueSplitExecutor.java new file mode 100644 index 000000000..7f56062d0 --- /dev/null +++ b/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/executors/QueueSplitExecutor.java @@ -0,0 +1,10 @@ +package org.swisspush.gateleen.queue.queuing.splitter.executors; + +import io.vertx.core.http.HttpServerRequest; + +public interface QueueSplitExecutor { + + boolean matches(String queue); + + String executeSplit(String queue, HttpServerRequest request); +} diff --git a/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/executors/QueueSplitExecutorBase.java b/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/executors/QueueSplitExecutorBase.java new file mode 100644 index 000000000..1c15edf3c --- /dev/null +++ b/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/executors/QueueSplitExecutorBase.java @@ -0,0 +1,17 @@ +package org.swisspush.gateleen.queue.queuing.splitter.executors; + +import org.swisspush.gateleen.queue.queuing.splitter.QueueSplitterConfiguration; + +public abstract class QueueSplitExecutorBase implements QueueSplitExecutor { + + protected final QueueSplitterConfiguration configuration; + + protected QueueSplitExecutorBase(QueueSplitterConfiguration configuration) { + this.configuration = configuration; + } + + @Override + public boolean matches(String queue) { + return configuration.getQueue().matcher(queue).matches(); + } +} diff --git a/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/executors/QueueSplitExecutorFromList.java b/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/executors/QueueSplitExecutorFromList.java new file mode 100644 index 000000000..e5b94c586 --- /dev/null +++ b/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/executors/QueueSplitExecutorFromList.java @@ -0,0 +1,30 @@ +package org.swisspush.gateleen.queue.queuing.splitter.executors; + +import io.vertx.core.http.HttpServerRequest; +import org.swisspush.gateleen.queue.queuing.splitter.QueueSplitterConfiguration; + +import java.util.concurrent.atomic.AtomicInteger; + +public class QueueSplitExecutorFromList extends QueueSplitExecutorBase { + + AtomicInteger atomicInteger = new AtomicInteger(0); + + protected QueueSplitExecutorFromList(QueueSplitterConfiguration configuration) { + super(configuration); + } + + @Override + public String executeSplit(String queue, HttpServerRequest request) { + StringBuilder stringBuilder = new StringBuilder(queue); + if (matches(queue)) { + stringBuilder.append(configuration.getPostfixDelimiter()); + stringBuilder.append(configuration.getPostfixFromStatic().get( + atomicInteger.getAndAccumulate( + 1, + (left, right) -> (left + right) % configuration.getPostfixFromStatic().size() + ) + )); + } + return stringBuilder.toString(); + } +} diff --git a/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/executors/QueueSplitExecutorFromRequest.java b/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/executors/QueueSplitExecutorFromRequest.java new file mode 100644 index 000000000..68341b9ed --- /dev/null +++ b/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/executors/QueueSplitExecutorFromRequest.java @@ -0,0 +1,35 @@ +package org.swisspush.gateleen.queue.queuing.splitter.executors; + +import io.vertx.core.http.HttpServerRequest; +import org.swisspush.gateleen.queue.queuing.splitter.QueueSplitterConfiguration; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class QueueSplitExecutorFromRequest extends QueueSplitExecutorBase { + protected QueueSplitExecutorFromRequest(QueueSplitterConfiguration configuration) { + super(configuration); + } + + @Override + public String executeSplit(String queue, HttpServerRequest request) { + StringBuilder stringBuilder = new StringBuilder(queue); + if (matches(queue)) { + if (configuration.getPostfixFromUrl() != null) { + Pattern pattern = Pattern.compile(configuration.getPostfixFromUrl()); + Matcher matcher = pattern.matcher(request.uri()); + if (matcher.matches()) { + for (int i = 0; i < matcher.groupCount(); i++) { + stringBuilder.append(configuration.getPostfixDelimiter()); + stringBuilder.append(matcher.group(i + 1)); + } + } + } + if (configuration.getPostfixFromHeader() != null) { + stringBuilder.append(configuration.getPostfixDelimiter()); + stringBuilder.append(request.headers().get(configuration.getPostfixFromHeader())); + } + } + return stringBuilder.toString(); + } +} diff --git a/gateleen-queue/src/test/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterConfigurationParserTest.java b/gateleen-queue/src/test/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterConfigurationParserTest.java index b08eefebc..82b6e2173 100644 --- a/gateleen-queue/src/test/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterConfigurationParserTest.java +++ b/gateleen-queue/src/test/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterConfigurationParserTest.java @@ -63,13 +63,17 @@ public void parseWithAllValid(TestContext context) { context.assertEquals(List.of("A", "B", "C", "D"), config_1.getPostfixFromStatic()); context.assertNull(config_1.getPostfixFromHeader()); context.assertNull(config_1.getPostfixFromUrl()); + context.assertTrue(config_1.isSplitStatic()); + context.assertFalse(config_1.isSplitFromRequest()); QueueSplitterConfiguration config_2 = configurations.get(1); context.assertEquals(Pattern.compile("my-queue-[0-9]+").pattern(), config_2.getQueue().pattern()); context.assertEquals("+", config_2.getPostfixDelimiter()); context.assertNull(config_2.getPostfixFromStatic()); - context.assertEquals("{x-rp-deviceid}", config_2.getPostfixFromHeader()); + context.assertEquals("x-rp-deviceid", config_2.getPostfixFromHeader()); context.assertNull(config_2.getPostfixFromUrl()); + context.assertFalse(config_2.isSplitStatic()); + context.assertTrue(config_2.isSplitFromRequest()); QueueSplitterConfiguration config_3 = configurations.get(2); context.assertEquals(Pattern.compile("my-queue-[a-zA-Z]+").pattern(), config_3.getQueue().pattern()); @@ -77,6 +81,8 @@ public void parseWithAllValid(TestContext context) { context.assertNull(config_3.getPostfixFromStatic()); context.assertNull(config_3.getPostfixFromHeader()); context.assertEquals(".*/path1/(.*)/path3/path4/.*", config_3.getPostfixFromUrl()); + context.assertFalse(config_3.isSplitStatic()); + context.assertTrue(config_3.isSplitFromRequest()); } @Test diff --git a/gateleen-queue/src/test/java/org/swisspush/gateleen/queue/queuing/splitter/executors/QueueSplitExecutorFromListTest.java b/gateleen-queue/src/test/java/org/swisspush/gateleen/queue/queuing/splitter/executors/QueueSplitExecutorFromListTest.java new file mode 100644 index 000000000..e815df2e0 --- /dev/null +++ b/gateleen-queue/src/test/java/org/swisspush/gateleen/queue/queuing/splitter/executors/QueueSplitExecutorFromListTest.java @@ -0,0 +1,92 @@ +package org.swisspush.gateleen.queue.queuing.splitter.executors; + +import io.vertx.core.http.HttpServerRequest; +import org.junit.Test; +import org.swisspush.gateleen.queue.queuing.splitter.QueueSplitterConfiguration; + +import java.util.List; +import java.util.regex.Pattern; + +import static org.junit.Assert.*; +import static org.mockito.Mockito.mock; + +public class QueueSplitExecutorFromListTest { + + @Test + public void testMatchesWithStaticQueueName() { + + // Given + QueueSplitExecutorFromList executor = new QueueSplitExecutorFromList(new QueueSplitterConfiguration( + Pattern.compile("queue-1"), + "-", + List.of("A", "B", "C", "D"), + null, + null + )); + + // Then + assertTrue(executor.matches("queue-1")); + assertFalse(executor.matches("queue-2")); + } + + @Test + public void testMatchesWithWildCharQueueName() { + + // Given + QueueSplitExecutorFromList executor = new QueueSplitExecutorFromList(new QueueSplitterConfiguration( + Pattern.compile("queue-[0-9]+"), + "-", + List.of("A", "B", "C", "D"), + null, + null + )); + + // Then + assertTrue(executor.matches("queue-1")); + assertTrue(executor.matches("queue-2")); + assertTrue(executor.matches("queue-30")); + assertFalse(executor.matches("queue-a")); + } + + @Test + public void testExecuteSplit() { + + // Given + QueueSplitExecutorFromList executor = new QueueSplitExecutorFromList(new QueueSplitterConfiguration( + Pattern.compile("queue-1"), + "-", + List.of("A", "B", "C", "D"), + null, + null + )); + HttpServerRequest request = mock(HttpServerRequest.class); + + // When + + // Then + assertEquals("queue-1-A", executor.executeSplit("queue-1", request)); + assertEquals("queue-1-B", executor.executeSplit("queue-1", request)); + assertEquals("queue-1-C", executor.executeSplit("queue-1", request)); + assertEquals("queue-1-D", executor.executeSplit("queue-1", request)); + assertEquals("queue-1-A", executor.executeSplit("queue-1", request)); + assertEquals("queue-1-B", executor.executeSplit("queue-1", request)); + assertEquals("queue-1-C", executor.executeSplit("queue-1", request)); + } + + @Test + public void testExecuteSplitForWrongQueue() { + + // Given + QueueSplitExecutorFromList executor = new QueueSplitExecutorFromList(new QueueSplitterConfiguration( + Pattern.compile("queue-1"), + "-", + List.of("A", "B", "C", "D"), + null, + null + )); + HttpServerRequest request = mock(HttpServerRequest.class); + + // Then + assertEquals("queue-2", executor.executeSplit("queue-2", request)); + } +} diff --git a/gateleen-queue/src/test/java/org/swisspush/gateleen/queue/queuing/splitter/executors/QueueSplitExecutorFromRequestTest.java b/gateleen-queue/src/test/java/org/swisspush/gateleen/queue/queuing/splitter/executors/QueueSplitExecutorFromRequestTest.java new file mode 100644 index 000000000..8fa15e0e2 --- /dev/null +++ b/gateleen-queue/src/test/java/org/swisspush/gateleen/queue/queuing/splitter/executors/QueueSplitExecutorFromRequestTest.java @@ -0,0 +1,141 @@ +package org.swisspush.gateleen.queue.queuing.splitter.executors; + +import io.vertx.core.http.HttpServerRequest; +import io.vertx.core.http.impl.headers.HeadersMultiMap; +import org.junit.Test; +import org.swisspush.gateleen.queue.queuing.splitter.QueueSplitterConfiguration; + +import java.util.regex.Pattern; + +import static org.junit.Assert.*; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class QueueSplitExecutorFromRequestTest { + + @Test + public void testMatchesWithStaticQueueName() { + + // Given + QueueSplitExecutorFromRequest executor = new QueueSplitExecutorFromRequest(new QueueSplitterConfiguration( + Pattern.compile("queue-1"), + "-", + null, + "x-rp-deviceid", + null + )); + + // Then + assertTrue(executor.matches("queue-1")); + assertFalse(executor.matches("queue-2")); + } + + @Test + public void testMatchesWithWildCharQueueName() { + + // Given + QueueSplitExecutorFromRequest executor = new QueueSplitExecutorFromRequest(new QueueSplitterConfiguration( + Pattern.compile("queue-[0-9]+"), + "-", + null, + "x-rp-deviceid", + null + )); + + // Then + assertTrue(executor.matches("queue-1")); + assertTrue(executor.matches("queue-2")); + assertTrue(executor.matches("queue-30")); + assertFalse(executor.matches("queue-a")); + } + + @Test + public void testExecuteSplitWithHeader() { + + // Given + QueueSplitExecutorFromRequest executor = new QueueSplitExecutorFromRequest(new QueueSplitterConfiguration( + Pattern.compile("queue-1"), + "-", + null, + "x-rp-deviceid", + null + )); + HttpServerRequest request = mock(HttpServerRequest.class); + when(request.headers()).thenReturn(new HeadersMultiMap().add("x-rp-deviceid", "A1B2C3D4E5F6")); + + // Then + assertEquals("queue-1-A1B2C3D4E5F6", executor.executeSplit("queue-1", request)); + } + + @Test + public void testExecuteSplitWithHeaderButMissingInRequest() { + + // Given + QueueSplitExecutorFromRequest executor = new QueueSplitExecutorFromRequest(new QueueSplitterConfiguration( + Pattern.compile("queue-1"), + "-", + null, + "x-rp-deviceid", + null + )); + HttpServerRequest request = mock(HttpServerRequest.class); + when(request.headers()).thenReturn(new HeadersMultiMap()); + + // Then + assertEquals("queue-1-null", executor.executeSplit("queue-1", request)); + } + + @Test + public void testExecuteSplitWithUrl() { + + // Given + QueueSplitExecutorFromRequest executor = new QueueSplitExecutorFromRequest(new QueueSplitterConfiguration( + Pattern.compile("queue-1"), + "-", + null, + null, + "/path1/(.+)/path3/(.+)" + )); + HttpServerRequest request = mock(HttpServerRequest.class); + when(request.headers()).thenReturn(new HeadersMultiMap()); + when(request.uri()).thenReturn("/path1/path2/path3/path4"); + + // Then + assertEquals("queue-1-path2-path4", executor.executeSplit("queue-1", request)); + } + @Test + public void testExecuteSplitWithUrlAndHeader() { + + // Given + QueueSplitExecutorFromRequest executor = new QueueSplitExecutorFromRequest(new QueueSplitterConfiguration( + Pattern.compile("queue-1"), + "-", + null, + "x-rp-deviceid", + "/path1/(.+)/path3/(.+)" + )); + HttpServerRequest request = mock(HttpServerRequest.class); + when(request.headers()).thenReturn(new HeadersMultiMap().add("x-rp-deviceid", "A1B2C3D4E5F6")); + when(request.uri()).thenReturn("/path1/path2/path3/path4"); + + // Then + assertEquals("queue-1-path2-path4-A1B2C3D4E5F6", executor.executeSplit("queue-1", request)); + } + + @Test + public void testExecuteSplitForWrongQueue() { + + // Given + QueueSplitExecutorFromList executor = new QueueSplitExecutorFromList(new QueueSplitterConfiguration( + Pattern.compile("queue-1"), + "-", + null, + "x-rp-deviceid", + null + )); + HttpServerRequest request = mock(HttpServerRequest.class); + + // Then + assertEquals("queue-2", executor.executeSplit("queue-2", request)); + } +} diff --git a/gateleen-queue/src/test/resources/testresource_queuesplitter_configuration_valid b/gateleen-queue/src/test/resources/testresource_queuesplitter_configuration_valid index c4ae7d7c1..bd359d385 100644 --- a/gateleen-queue/src/test/resources/testresource_queuesplitter_configuration_valid +++ b/gateleen-queue/src/test/resources/testresource_queuesplitter_configuration_valid @@ -9,7 +9,7 @@ }, "my-queue-[0-9]+" : { "postfixDelimiter": "+", - "postfixFromHeader": "{x-rp-deviceid}" + "postfixFromHeader": "x-rp-deviceid" }, "my-queue-[a-zA-Z]+" : { "postfixDelimiter": "_", From 12b48f17b3752a8ae0d1de2fc08df94c7c35caaf Mon Sep 17 00:00:00 2001 From: Giannandrea Castaldi Date: Thu, 25 Jan 2024 13:30:45 +0100 Subject: [PATCH 05/18] #550 implemented QueueSplitterImpl --- .../queuing/splitter/QueueSplitterImpl.java | 41 ++-- .../executors/QueueSplitExecutorFromList.java | 2 +- .../QueueSplitExecutorFromRequest.java | 2 +- .../splitter/QueueSplitterImplTest.java | 190 ++++++++++++++++++ 4 files changed, 219 insertions(+), 16 deletions(-) create mode 100644 gateleen-queue/src/test/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterImplTest.java diff --git a/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterImpl.java b/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterImpl.java index 7b9463562..4bb04b7cb 100644 --- a/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterImpl.java +++ b/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterImpl.java @@ -8,10 +8,12 @@ import org.slf4j.LoggerFactory; import org.swisspush.gateleen.core.configuration.ConfigurationResourceConsumer; import org.swisspush.gateleen.core.configuration.ConfigurationResourceManager; +import org.swisspush.gateleen.queue.queuing.splitter.executors.QueueSplitExecutor; +import org.swisspush.gateleen.queue.queuing.splitter.executors.QueueSplitExecutorFromList; +import org.swisspush.gateleen.queue.queuing.splitter.executors.QueueSplitExecutorFromRequest; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; +import java.util.stream.Collectors; /** * {@inheritDoc} @@ -24,6 +26,8 @@ public class QueueSplitterImpl extends ConfigurationResourceConsumer implements private boolean initialized = false; + private List queueSplitExecutors = new ArrayList<>(); + public QueueSplitterImpl( ConfigurationResourceManager configurationResourceManager, String configResourceUri @@ -40,11 +44,16 @@ public QueueSplitterImpl( this.properties = properties; } + public List getQueueSplitExecutors() { + return queueSplitExecutors; + } + public Future initialize() { Promise promise = Promise.promise(); configurationResourceManager().getRegisteredResource(configResourceUri()).onComplete((event -> { if (event.succeeded() && event.result().isPresent()) { - initializeQueueSplitterConfiguration(event.result().get()).onComplete((event1 -> promise.complete())); + initializeQueueSplitterConfiguration(event.result().get()); + promise.complete(); } else { log.warn("No queue splitter configuration resource with uri '{}' found. Unable to setup kafka configuration correctly", configResourceUri()); promise.complete(); @@ -57,13 +66,18 @@ public boolean isInitialized() { return initialized; } - private Future initializeQueueSplitterConfiguration(Buffer configuration) { + private void initializeQueueSplitterConfiguration(Buffer configuration) { Promise promise = Promise.promise(); - final List kafkaConfigurations = QueueSplitterConfigurationParser.parse(configuration, properties); - - // TODO: release all splitters and creates new ones - - return promise.future(); + final List configurations = QueueSplitterConfigurationParser.parse(configuration, properties); + queueSplitExecutors.clear(); + queueSplitExecutors = configurations.stream().map(queueSplitterConfiguration -> { + if (queueSplitterConfiguration.isSplitStatic()) { + return new QueueSplitExecutorFromList(queueSplitterConfiguration); + } else { + return new QueueSplitExecutorFromRequest(queueSplitterConfiguration); + } + }).collect(Collectors.toList()); + initialized = true; } /** @@ -71,7 +85,8 @@ private Future initializeQueueSplitterConfiguration(Buffer configuration) */ @Override public String convertToSubQueue(final String queue, HttpServerRequest request) { - return queue + "-1"; + Optional executor = queueSplitExecutors.stream().filter(splitExecutor -> splitExecutor.matches(queue)).findFirst(); + return executor.isPresent() ? executor.get().executeSplit(queue, request) : queue; } @Override @@ -86,9 +101,7 @@ public void resourceChanged(String resourceUri, Buffer resource) { public void resourceRemoved(String resourceUri) { if (configResourceUri() != null && configResourceUri().equals(resourceUri)) { log.info("Queue splitter configuration resource {} was removed. Going to close all kafka producers", resourceUri); - - // TODO: release all splitters and creates new ones - + queueSplitExecutors.clear(); initialized = false; } } diff --git a/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/executors/QueueSplitExecutorFromList.java b/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/executors/QueueSplitExecutorFromList.java index e5b94c586..8124f85a3 100644 --- a/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/executors/QueueSplitExecutorFromList.java +++ b/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/executors/QueueSplitExecutorFromList.java @@ -9,7 +9,7 @@ public class QueueSplitExecutorFromList extends QueueSplitExecutorBase { AtomicInteger atomicInteger = new AtomicInteger(0); - protected QueueSplitExecutorFromList(QueueSplitterConfiguration configuration) { + public QueueSplitExecutorFromList(QueueSplitterConfiguration configuration) { super(configuration); } diff --git a/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/executors/QueueSplitExecutorFromRequest.java b/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/executors/QueueSplitExecutorFromRequest.java index 68341b9ed..117303924 100644 --- a/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/executors/QueueSplitExecutorFromRequest.java +++ b/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/executors/QueueSplitExecutorFromRequest.java @@ -7,7 +7,7 @@ import java.util.regex.Pattern; public class QueueSplitExecutorFromRequest extends QueueSplitExecutorBase { - protected QueueSplitExecutorFromRequest(QueueSplitterConfiguration configuration) { + public QueueSplitExecutorFromRequest(QueueSplitterConfiguration configuration) { super(configuration); } diff --git a/gateleen-queue/src/test/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterImplTest.java b/gateleen-queue/src/test/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterImplTest.java new file mode 100644 index 000000000..b916589ed --- /dev/null +++ b/gateleen-queue/src/test/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterImplTest.java @@ -0,0 +1,190 @@ +package org.swisspush.gateleen.queue.queuing.splitter; + +import io.vertx.core.Vertx; +import io.vertx.core.http.HttpServerRequest; +import io.vertx.core.http.impl.headers.HeadersMultiMap; +import io.vertx.core.json.JsonObject; +import io.vertx.ext.unit.Async; +import io.vertx.ext.unit.TestContext; +import io.vertx.ext.unit.junit.VertxUnitRunner; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.swisspush.gateleen.core.configuration.ConfigurationResourceManager; +import org.swisspush.gateleen.core.storage.MockResourceStorage; +import org.swisspush.gateleen.core.util.ResourcesUtils; +import org.swisspush.gateleen.queue.queuing.splitter.executors.QueueSplitExecutor; +import org.swisspush.gateleen.queue.queuing.splitter.executors.QueueSplitExecutorFromList; +import org.swisspush.gateleen.queue.queuing.splitter.executors.QueueSplitExecutorFromRequest; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.swisspush.gateleen.core.configuration.ConfigurationResourceManager.CONFIG_RESOURCE_CHANGED_ADDRESS; + +@RunWith(VertxUnitRunner.class) +public class QueueSplitterImplTest { + + private Vertx vertx; + private MockResourceStorage storage; + private final String configResourceUri = "/queueSplitters"; + private ConfigurationResourceManager configurationResourceManager; + private QueueSplitterImpl queueSplitter; + + private final String CONFIG_RESOURCE_VALID = ResourcesUtils.loadResource("testresource_queuesplitter_configuration_valid", true); + private final String CONFIG_RESOURCE_INVALID = ResourcesUtils.loadResource("testresource_queuesplitter_configuration_invalid", true); + + @Before + public void setUp() { + vertx = Vertx.vertx(); + storage = new MockResourceStorage(); + configurationResourceManager = new ConfigurationResourceManager(vertx, storage); + queueSplitter = new QueueSplitterImpl(configurationResourceManager, configResourceUri); + } + + @Test + public void initWithMissingConfigResource(TestContext context) { + Async async = context.async(); + context.assertFalse(queueSplitter.isInitialized()); + queueSplitter.initialize().onComplete(event -> { + context.assertFalse(queueSplitter.isInitialized()); + context.assertTrue(queueSplitter.getQueueSplitExecutors().isEmpty()); + async.complete(); + }); + } + + @Test + public void initWithExistingConfigResource(TestContext context) { + Async async = context.async(); + storage.putMockData(configResourceUri, CONFIG_RESOURCE_VALID); + context.assertFalse(queueSplitter.isInitialized()); + queueSplitter.initialize().onComplete(event -> { + context.assertTrue(queueSplitter.isInitialized()); + context.assertEquals(3, queueSplitter.getQueueSplitExecutors().size()); + + QueueSplitExecutor executor_1 = queueSplitter.getQueueSplitExecutors().get(0); + context.assertTrue(executor_1 instanceof QueueSplitExecutorFromList); + + QueueSplitExecutor executor_2 = queueSplitter.getQueueSplitExecutors().get(1); + context.assertTrue(executor_2 instanceof QueueSplitExecutorFromRequest); + + QueueSplitExecutor executor_3 = queueSplitter.getQueueSplitExecutors().get(2); + context.assertTrue(executor_3 instanceof QueueSplitExecutorFromRequest); + + async.complete(); + }); + } + + @Test + public void initWithExistingInvalidConfigResource(TestContext context) { + Async async = context.async(); + storage.putMockData(configResourceUri, CONFIG_RESOURCE_INVALID); + context.assertFalse(queueSplitter.isInitialized()); + queueSplitter.initialize().onComplete(event -> { + context.assertTrue(queueSplitter.isInitialized()); + context.assertEquals(1, queueSplitter.getQueueSplitExecutors().size()); + + QueueSplitExecutor executor_1 = queueSplitter.getQueueSplitExecutors().get(0); + context.assertTrue(executor_1 instanceof QueueSplitExecutorFromList); + + async.complete(); + }); + } + + @Test + @Ignore("verify with timeout and await don't work, review with Marc") + public void configResourceRemovedTriggerRemoveAllExecutors(TestContext context) { + Async async = context.async(); + storage.putMockData(configResourceUri, CONFIG_RESOURCE_VALID); + context.assertFalse(queueSplitter.isInitialized()); + queueSplitter.initialize().onComplete(event -> { + context.assertTrue(queueSplitter.isInitialized()); + context.assertEquals(3, queueSplitter.getQueueSplitExecutors().size()); + JsonObject object = new JsonObject(); + object.put("requestUri", configResourceUri); + object.put("type", "remove"); + vertx.eventBus().publish(CONFIG_RESOURCE_CHANGED_ADDRESS, object); + // try { Thread.sleep(2000);} catch (InterruptedException e) {} + // verify(storage, timeout(100).times(1)).delete(eq(CONFIG_RESOURCE_CHANGED_ADDRESS), any()); + // await().atMost(TWO_SECONDS).until(() -> queueSplitter.getQueueSplitExecutors(), equalTo(0)); + context.assertEquals(0, queueSplitter.getQueueSplitExecutors().size()); + async.complete(); + }); + } + + @Test + @Ignore("verify with timeout and await don't work, review with Marc") + public void configResourceChangedTriggerNewInitOfExecutors(TestContext context) { + Async async = context.async(); + storage.putMockData(configResourceUri, CONFIG_RESOURCE_VALID); + context.assertFalse(queueSplitter.isInitialized()); + queueSplitter.initialize().onComplete(event -> { + context.assertTrue(queueSplitter.isInitialized()); + context.assertEquals(3, queueSplitter.getQueueSplitExecutors().size()); + + JsonObject object = new JsonObject(); + object.put("requestUri", configResourceUri); + object.put("type", "change"); + vertx.eventBus().publish(CONFIG_RESOURCE_CHANGED_ADDRESS, object); + // try { Thread.sleep(2000);} catch (InterruptedException e) {} + // verify(storage, timeout(100).times(1)).delete(eq(CONFIG_RESOURCE_CHANGED_ADDRESS), any()); + // await().atMost(TWO_SECONDS).until(() -> queueSplitter.getQueueSplitExecutors(), equalTo(0)); + context.assertEquals(0, queueSplitter.getQueueSplitExecutors().size()); + async.complete(); + }); + } + + @Test + public void testConvertToSubQueueWithPostfixFromStatic(TestContext context) { + Async async = context.async(); + storage.putMockData(configResourceUri, CONFIG_RESOURCE_VALID); + context.assertFalse(queueSplitter.isInitialized()); + queueSplitter.initialize().onComplete(event -> { + HttpServerRequest request = mock(HttpServerRequest.class); + when(request.headers()).thenReturn(new HeadersMultiMap()); + context.assertEquals("my-queue-1-A", queueSplitter.convertToSubQueue("my-queue-1", request)); + context.assertEquals("my-queue-1-B", queueSplitter.convertToSubQueue("my-queue-1", request)); + context.assertEquals("my-queue-1-C", queueSplitter.convertToSubQueue("my-queue-1", request)); + async.complete(); + }); + } + + @Test + public void testConvertToSubQueueWithPostfixFromHeader(TestContext context) { + Async async = context.async(); + storage.putMockData(configResourceUri, CONFIG_RESOURCE_VALID); + context.assertFalse(queueSplitter.isInitialized()); + queueSplitter.initialize().onComplete(event -> { + HttpServerRequest request = mock(HttpServerRequest.class); + when(request.headers()).thenReturn(new HeadersMultiMap().add("x-rp-deviceid", "A1B2C3D4E5F6")); + context.assertEquals("my-queue-2+A1B2C3D4E5F6", queueSplitter.convertToSubQueue("my-queue-2", request)); + async.complete(); + }); + } + @Test + public void testConvertToSubQueueWithPostfixFromUrl(TestContext context) { + Async async = context.async(); + storage.putMockData(configResourceUri, CONFIG_RESOURCE_VALID); + context.assertFalse(queueSplitter.isInitialized()); + queueSplitter.initialize().onComplete(event -> { + HttpServerRequest request = mock(HttpServerRequest.class); + when(request.headers()).thenReturn(new HeadersMultiMap()); + when(request.uri()).thenReturn("/path1/path2/path3/path4/"); + context.assertEquals("my-queue-a_path2", queueSplitter.convertToSubQueue("my-queue-a", request)); + async.complete(); + }); + } + + @Test + public void testConvertToSubQueueWithNoPostfixConfigured(TestContext context) { + Async async = context.async(); + storage.putMockData(configResourceUri, CONFIG_RESOURCE_VALID); + context.assertFalse(queueSplitter.isInitialized()); + queueSplitter.initialize().onComplete(event -> { + HttpServerRequest request = mock(HttpServerRequest.class); + when(request.headers()).thenReturn(new HeadersMultiMap()); + context.assertEquals("another-queue", queueSplitter.convertToSubQueue("another-queue", request)); + async.complete(); + }); + } +} From a834584ad5674497d45995859c2127a4d05e997c Mon Sep 17 00:00:00 2001 From: Giannandrea Castaldi Date: Thu, 25 Jan 2024 15:34:58 +0100 Subject: [PATCH 06/18] #550 converted postfixFromUrl in QueueSplitterConfiguration in Pattern --- .../splitter/QueueSplitterConfiguration.java | 6 +- .../QueueSplitterConfigurationParser.java | 57 ++++++++++--------- .../QueueSplitExecutorFromRequest.java | 3 +- .../QueueSplitterConfigurationParserTest.java | 4 +- .../QueueSplitExecutorFromRequestTest.java | 4 +- 5 files changed, 40 insertions(+), 34 deletions(-) diff --git a/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterConfiguration.java b/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterConfiguration.java index 16cc164a3..ca37d1047 100644 --- a/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterConfiguration.java +++ b/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterConfiguration.java @@ -24,7 +24,7 @@ public class QueueSplitterConfiguration { private final String postfixFromHeader; @Nullable - private final String postfixFromUrl; + private final Pattern postfixFromUrl; public QueueSplitterConfiguration( @@ -32,7 +32,7 @@ public QueueSplitterConfiguration( String postfixDelimiter, @Nullable List postfixFromStatic, @Nullable String postfixFromHeader, - @Nullable String postfixFromUrl) { + @Nullable Pattern postfixFromUrl) { this.queue = queue; this.postfixDelimiter = postfixDelimiter; this.postfixFromStatic = postfixFromStatic; @@ -59,7 +59,7 @@ public String getPostfixFromHeader() { } @Nullable - public String getPostfixFromUrl() { + public Pattern getPostfixFromUrl() { return postfixFromUrl; } diff --git a/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterConfigurationParser.java b/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterConfigurationParser.java index c27251bca..e892e8be0 100644 --- a/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterConfigurationParser.java +++ b/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterConfigurationParser.java @@ -12,6 +12,7 @@ import java.util.List; import java.util.Map; import java.util.regex.Pattern; +import java.util.regex.PatternSyntaxException; import java.util.stream.Collectors; /** @@ -45,32 +46,36 @@ static List parse(Buffer configurationResourceBuffer } for (String queuePattern : config.fieldNames()) { - Pattern pattern = Pattern.compile(queuePattern); - JsonObject queueConfig = config.getJsonObject(queuePattern); - JsonArray postfixFromStatic = queueConfig.getJsonArray(POSTFIX_FROM_STATIC_KEY); - if (postfixFromStatic != null) { - List staticPostfixes = postfixFromStatic.stream().map(Object::toString).collect(Collectors.toList()); - queueSplitterConfigurations.add(new QueueSplitterConfiguration( - pattern, - queueConfig.getString("postfixDelimiter", DEFAULT_POSTFIX_DELIMITER), - staticPostfixes, - null, - null - )); - continue; - } - String postfixFromHeader = queueConfig.getString(POSTFIX_FROM_HEADER_KEY); - String postfixFromUrl = queueConfig.getString(POSTFIX_FROM_URL_KEY); - if (postfixFromHeader != null || postfixFromUrl != null) { - queueSplitterConfigurations.add(new QueueSplitterConfiguration( - pattern, - queueConfig.getString(POSTFIX_DELIMITER_KEY, DEFAULT_POSTFIX_DELIMITER), - null, - postfixFromHeader, - postfixFromUrl - )); - } else { - log.warn("Queue splitter configuration without a postfix definition"); + try { + Pattern pattern = Pattern.compile(queuePattern); + JsonObject queueConfig = config.getJsonObject(queuePattern); + JsonArray postfixFromStatic = queueConfig.getJsonArray(POSTFIX_FROM_STATIC_KEY); + if (postfixFromStatic != null) { + List staticPostfixes = postfixFromStatic.stream().map(Object::toString).collect(Collectors.toList()); + queueSplitterConfigurations.add(new QueueSplitterConfiguration( + pattern, + queueConfig.getString("postfixDelimiter", DEFAULT_POSTFIX_DELIMITER), + staticPostfixes, + null, + null + )); + continue; + } + String postfixFromHeader = queueConfig.getString(POSTFIX_FROM_HEADER_KEY); + String postfixFromUrl = queueConfig.getString(POSTFIX_FROM_URL_KEY); + if (postfixFromHeader != null || postfixFromUrl != null) { + queueSplitterConfigurations.add(new QueueSplitterConfiguration( + pattern, + queueConfig.getString(POSTFIX_DELIMITER_KEY, DEFAULT_POSTFIX_DELIMITER), + null, + postfixFromHeader, + postfixFromUrl != null ? Pattern.compile(postfixFromUrl) : null + )); + } else { + log.warn("Queue splitter configuration without a postfix definition"); + } + } catch (PatternSyntaxException patternException) { + log.warn("Queue splitter '{}' is not a valid regex pattern. Discarding this queue splitter configuration", queuePattern); } } diff --git a/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/executors/QueueSplitExecutorFromRequest.java b/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/executors/QueueSplitExecutorFromRequest.java index 117303924..e907420ab 100644 --- a/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/executors/QueueSplitExecutorFromRequest.java +++ b/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/executors/QueueSplitExecutorFromRequest.java @@ -16,8 +16,7 @@ public String executeSplit(String queue, HttpServerRequest request) { StringBuilder stringBuilder = new StringBuilder(queue); if (matches(queue)) { if (configuration.getPostfixFromUrl() != null) { - Pattern pattern = Pattern.compile(configuration.getPostfixFromUrl()); - Matcher matcher = pattern.matcher(request.uri()); + Matcher matcher = configuration.getPostfixFromUrl().matcher(request.uri()); if (matcher.matches()) { for (int i = 0; i < matcher.groupCount(); i++) { stringBuilder.append(configuration.getPostfixDelimiter()); diff --git a/gateleen-queue/src/test/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterConfigurationParserTest.java b/gateleen-queue/src/test/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterConfigurationParserTest.java index 82b6e2173..e5dc3bcc0 100644 --- a/gateleen-queue/src/test/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterConfigurationParserTest.java +++ b/gateleen-queue/src/test/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterConfigurationParserTest.java @@ -80,7 +80,9 @@ public void parseWithAllValid(TestContext context) { context.assertEquals("_", config_3.getPostfixDelimiter()); context.assertNull(config_3.getPostfixFromStatic()); context.assertNull(config_3.getPostfixFromHeader()); - context.assertEquals(".*/path1/(.*)/path3/path4/.*", config_3.getPostfixFromUrl()); + context.assertEquals( + Pattern.compile(".*/path1/(.*)/path3/path4/.*").pattern(), + config_3.getPostfixFromUrl().pattern()); context.assertFalse(config_3.isSplitStatic()); context.assertTrue(config_3.isSplitFromRequest()); } diff --git a/gateleen-queue/src/test/java/org/swisspush/gateleen/queue/queuing/splitter/executors/QueueSplitExecutorFromRequestTest.java b/gateleen-queue/src/test/java/org/swisspush/gateleen/queue/queuing/splitter/executors/QueueSplitExecutorFromRequestTest.java index 8fa15e0e2..59a72e16a 100644 --- a/gateleen-queue/src/test/java/org/swisspush/gateleen/queue/queuing/splitter/executors/QueueSplitExecutorFromRequestTest.java +++ b/gateleen-queue/src/test/java/org/swisspush/gateleen/queue/queuing/splitter/executors/QueueSplitExecutorFromRequestTest.java @@ -94,7 +94,7 @@ public void testExecuteSplitWithUrl() { "-", null, null, - "/path1/(.+)/path3/(.+)" + Pattern.compile("/path1/(.+)/path3/(.+)") )); HttpServerRequest request = mock(HttpServerRequest.class); when(request.headers()).thenReturn(new HeadersMultiMap()); @@ -112,7 +112,7 @@ public void testExecuteSplitWithUrlAndHeader() { "-", null, "x-rp-deviceid", - "/path1/(.+)/path3/(.+)" + Pattern.compile("/path1/(.+)/path3/(.+)") )); HttpServerRequest request = mock(HttpServerRequest.class); when(request.headers()).thenReturn(new HeadersMultiMap().add("x-rp-deviceid", "A1B2C3D4E5F6")); From 1446d058cea16e2343f6f618f86af8ffcab4803e Mon Sep 17 00:00:00 2001 From: Giannandrea Castaldi Date: Fri, 26 Jan 2024 08:08:28 +0100 Subject: [PATCH 07/18] #550 removed method getQueueSplitExecutors() from QueueSplitterImpl --- .../queuing/splitter/QueueSplitterImpl.java | 4 - .../splitter/QueueSplitterImplTest.java | 148 +++++++++--------- 2 files changed, 71 insertions(+), 81 deletions(-) diff --git a/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterImpl.java b/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterImpl.java index 4bb04b7cb..34aaa139c 100644 --- a/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterImpl.java +++ b/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterImpl.java @@ -44,10 +44,6 @@ public QueueSplitterImpl( this.properties = properties; } - public List getQueueSplitExecutors() { - return queueSplitExecutors; - } - public Future initialize() { Promise promise = Promise.promise(); configurationResourceManager().getRegisteredResource(configResourceUri()).onComplete((event -> { diff --git a/gateleen-queue/src/test/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterImplTest.java b/gateleen-queue/src/test/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterImplTest.java index b916589ed..5e89bb6a9 100644 --- a/gateleen-queue/src/test/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterImplTest.java +++ b/gateleen-queue/src/test/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterImplTest.java @@ -14,12 +14,8 @@ import org.swisspush.gateleen.core.configuration.ConfigurationResourceManager; import org.swisspush.gateleen.core.storage.MockResourceStorage; import org.swisspush.gateleen.core.util.ResourcesUtils; -import org.swisspush.gateleen.queue.queuing.splitter.executors.QueueSplitExecutor; -import org.swisspush.gateleen.queue.queuing.splitter.executors.QueueSplitExecutorFromList; -import org.swisspush.gateleen.queue.queuing.splitter.executors.QueueSplitExecutorFromRequest; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; +import static org.mockito.Mockito.*; import static org.swisspush.gateleen.core.configuration.ConfigurationResourceManager.CONFIG_RESOURCE_CHANGED_ADDRESS; @RunWith(VertxUnitRunner.class) @@ -43,50 +39,56 @@ public void setUp() { } @Test - public void initWithMissingConfigResource(TestContext context) { + public void splitWithMissingConfigResource(TestContext context) { Async async = context.async(); context.assertFalse(queueSplitter.isInitialized()); queueSplitter.initialize().onComplete(event -> { context.assertFalse(queueSplitter.isInitialized()); - context.assertTrue(queueSplitter.getQueueSplitExecutors().isEmpty()); + verifySplitStaticNotExecuted(context); + verifySplitWithHeaderNotExecuted(context); + verifySplitWithUrlNotExecuted(context); async.complete(); }); } @Test - public void initWithExistingConfigResource(TestContext context) { + public void splitWithValidConfigResource(TestContext context) { Async async = context.async(); storage.putMockData(configResourceUri, CONFIG_RESOURCE_VALID); context.assertFalse(queueSplitter.isInitialized()); queueSplitter.initialize().onComplete(event -> { context.assertTrue(queueSplitter.isInitialized()); - context.assertEquals(3, queueSplitter.getQueueSplitExecutors().size()); - - QueueSplitExecutor executor_1 = queueSplitter.getQueueSplitExecutors().get(0); - context.assertTrue(executor_1 instanceof QueueSplitExecutorFromList); - - QueueSplitExecutor executor_2 = queueSplitter.getQueueSplitExecutors().get(1); - context.assertTrue(executor_2 instanceof QueueSplitExecutorFromRequest); - - QueueSplitExecutor executor_3 = queueSplitter.getQueueSplitExecutors().get(2); - context.assertTrue(executor_3 instanceof QueueSplitExecutorFromRequest); - + verifySplitStaticExecuted(context); + verifySplitWithHeaderExecuted(context); + verifySplitWithUrlExecuted(context); async.complete(); }); } @Test - public void initWithExistingInvalidConfigResource(TestContext context) { + public void splitWithPartiallyInvalidConfigResource(TestContext context) { Async async = context.async(); storage.putMockData(configResourceUri, CONFIG_RESOURCE_INVALID); context.assertFalse(queueSplitter.isInitialized()); queueSplitter.initialize().onComplete(event -> { context.assertTrue(queueSplitter.isInitialized()); - context.assertEquals(1, queueSplitter.getQueueSplitExecutors().size()); + verifySplitStaticExecuted(context); + verifySplitWithHeaderNotExecuted(context); + verifySplitWithUrlNotExecuted(context); + async.complete(); + }); + } - QueueSplitExecutor executor_1 = queueSplitter.getQueueSplitExecutors().get(0); - context.assertTrue(executor_1 instanceof QueueSplitExecutorFromList); + @Test + public void splitWithQueueNotMatchingAnyConfiguration(TestContext context) { + Async async = context.async(); + storage.putMockData(configResourceUri, CONFIG_RESOURCE_VALID); + context.assertFalse(queueSplitter.isInitialized()); + queueSplitter.initialize().onComplete(event -> { + HttpServerRequest request = mock(HttpServerRequest.class); + when(request.headers()).thenReturn(new HeadersMultiMap()); + context.assertEquals("another-queue", queueSplitter.convertToSubQueue("another-queue", request)); async.complete(); }); } @@ -99,15 +101,19 @@ public void configResourceRemovedTriggerRemoveAllExecutors(TestContext context) context.assertFalse(queueSplitter.isInitialized()); queueSplitter.initialize().onComplete(event -> { context.assertTrue(queueSplitter.isInitialized()); - context.assertEquals(3, queueSplitter.getQueueSplitExecutors().size()); + verifySplitStaticExecuted(context); + verifySplitWithHeaderExecuted(context); + verifySplitWithUrlExecuted(context); JsonObject object = new JsonObject(); object.put("requestUri", configResourceUri); object.put("type", "remove"); vertx.eventBus().publish(CONFIG_RESOURCE_CHANGED_ADDRESS, object); - // try { Thread.sleep(2000);} catch (InterruptedException e) {} + try { Thread.sleep(10000);} catch (InterruptedException e) {} // verify(storage, timeout(100).times(1)).delete(eq(CONFIG_RESOURCE_CHANGED_ADDRESS), any()); // await().atMost(TWO_SECONDS).until(() -> queueSplitter.getQueueSplitExecutors(), equalTo(0)); - context.assertEquals(0, queueSplitter.getQueueSplitExecutors().size()); + verifySplitStaticNotExecuted(context); + verifySplitWithHeaderNotExecuted(context); + verifySplitWithUrlNotExecuted(context); async.complete(); }); } @@ -120,71 +126,59 @@ public void configResourceChangedTriggerNewInitOfExecutors(TestContext context) context.assertFalse(queueSplitter.isInitialized()); queueSplitter.initialize().onComplete(event -> { context.assertTrue(queueSplitter.isInitialized()); - context.assertEquals(3, queueSplitter.getQueueSplitExecutors().size()); - + verifySplitStaticExecuted(context); + verifySplitWithHeaderExecuted(context); + verifySplitWithUrlExecuted(context); JsonObject object = new JsonObject(); object.put("requestUri", configResourceUri); object.put("type", "change"); vertx.eventBus().publish(CONFIG_RESOURCE_CHANGED_ADDRESS, object); // try { Thread.sleep(2000);} catch (InterruptedException e) {} - // verify(storage, timeout(100).times(1)).delete(eq(CONFIG_RESOURCE_CHANGED_ADDRESS), any()); + // verify(storage, timeout(100).times(1)).put(eq(CONFIG_RESOURCE_CHANGED_ADDRESS), any(), any()); + verify(storage, times(1)).get(anyString(), any()); // await().atMost(TWO_SECONDS).until(() -> queueSplitter.getQueueSplitExecutors(), equalTo(0)); - context.assertEquals(0, queueSplitter.getQueueSplitExecutors().size()); + verifySplitStaticExecuted(context); + verifySplitWithHeaderNotExecuted(context); + verifySplitWithUrlNotExecuted(context); async.complete(); }); } - @Test - public void testConvertToSubQueueWithPostfixFromStatic(TestContext context) { - Async async = context.async(); - storage.putMockData(configResourceUri, CONFIG_RESOURCE_VALID); - context.assertFalse(queueSplitter.isInitialized()); - queueSplitter.initialize().onComplete(event -> { - HttpServerRequest request = mock(HttpServerRequest.class); - when(request.headers()).thenReturn(new HeadersMultiMap()); - context.assertEquals("my-queue-1-A", queueSplitter.convertToSubQueue("my-queue-1", request)); - context.assertEquals("my-queue-1-B", queueSplitter.convertToSubQueue("my-queue-1", request)); - context.assertEquals("my-queue-1-C", queueSplitter.convertToSubQueue("my-queue-1", request)); - async.complete(); - }); + private void verifySplitStaticExecuted(TestContext context) { + HttpServerRequest request = mock(HttpServerRequest.class); + context.assertEquals("my-queue-1-A", queueSplitter.convertToSubQueue("my-queue-1", request)); + context.assertEquals("my-queue-1-B", queueSplitter.convertToSubQueue("my-queue-1", request)); + context.assertEquals("my-queue-1-C", queueSplitter.convertToSubQueue("my-queue-1", request)); } - @Test - public void testConvertToSubQueueWithPostfixFromHeader(TestContext context) { - Async async = context.async(); - storage.putMockData(configResourceUri, CONFIG_RESOURCE_VALID); - context.assertFalse(queueSplitter.isInitialized()); - queueSplitter.initialize().onComplete(event -> { - HttpServerRequest request = mock(HttpServerRequest.class); - when(request.headers()).thenReturn(new HeadersMultiMap().add("x-rp-deviceid", "A1B2C3D4E5F6")); - context.assertEquals("my-queue-2+A1B2C3D4E5F6", queueSplitter.convertToSubQueue("my-queue-2", request)); - async.complete(); - }); + private void verifySplitStaticNotExecuted(TestContext context) { + HttpServerRequest request = mock(HttpServerRequest.class); + context.assertEquals("my-queue-1", queueSplitter.convertToSubQueue("my-queue-1", request)); } - @Test - public void testConvertToSubQueueWithPostfixFromUrl(TestContext context) { - Async async = context.async(); - storage.putMockData(configResourceUri, CONFIG_RESOURCE_VALID); - context.assertFalse(queueSplitter.isInitialized()); - queueSplitter.initialize().onComplete(event -> { - HttpServerRequest request = mock(HttpServerRequest.class); - when(request.headers()).thenReturn(new HeadersMultiMap()); - when(request.uri()).thenReturn("/path1/path2/path3/path4/"); - context.assertEquals("my-queue-a_path2", queueSplitter.convertToSubQueue("my-queue-a", request)); - async.complete(); - }); + + private void verifySplitWithHeaderExecuted(TestContext context) { + HttpServerRequest request = mock(HttpServerRequest.class); + when(request.headers()).thenReturn(new HeadersMultiMap().add("x-rp-deviceid", "A1B2C3D4E5F6")); + context.assertEquals("my-queue-2+A1B2C3D4E5F6", queueSplitter.convertToSubQueue("my-queue-2", request)); } - @Test - public void testConvertToSubQueueWithNoPostfixConfigured(TestContext context) { - Async async = context.async(); - storage.putMockData(configResourceUri, CONFIG_RESOURCE_VALID); - context.assertFalse(queueSplitter.isInitialized()); - queueSplitter.initialize().onComplete(event -> { - HttpServerRequest request = mock(HttpServerRequest.class); - when(request.headers()).thenReturn(new HeadersMultiMap()); - context.assertEquals("another-queue", queueSplitter.convertToSubQueue("another-queue", request)); - async.complete(); - }); + private void verifySplitWithHeaderNotExecuted(TestContext context) { + HttpServerRequest request = mock(HttpServerRequest.class); + when(request.headers()).thenReturn(new HeadersMultiMap().add("x-rp-deviceid", "A1B2C3D4E5F6")); + context.assertEquals("my-queue-2", queueSplitter.convertToSubQueue("my-queue-2", request)); + } + + private void verifySplitWithUrlExecuted(TestContext context) { + HttpServerRequest request = mock(HttpServerRequest.class); + when(request.headers()).thenReturn(new HeadersMultiMap()); + when(request.uri()).thenReturn("/path1/path2/path3/path4/"); + context.assertEquals("my-queue-a_path2", queueSplitter.convertToSubQueue("my-queue-a", request)); + } + + private void verifySplitWithUrlNotExecuted(TestContext context) { + HttpServerRequest request = mock(HttpServerRequest.class); + when(request.headers()).thenReturn(new HeadersMultiMap()); + when(request.uri()).thenReturn("/path1/path2/path3/path4/"); + context.assertEquals("my-queue-a", queueSplitter.convertToSubQueue("my-queue-a", request)); } } From 9ab5c6feec2c24d640ab0ba8f7a5b3935bafb624 Mon Sep 17 00:00:00 2001 From: Giannandrea Castaldi Date: Tue, 30 Jan 2024 14:16:55 +0100 Subject: [PATCH 08/18] #550 commited some tests to review --- .../queuing/splitter/QueueSplitterImpl.java | 3 +- .../splitter/QueueSplitterImplTest.java | 163 ++++++++++++++++-- 2 files changed, 146 insertions(+), 20 deletions(-) diff --git a/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterImpl.java b/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterImpl.java index 34aaa139c..74e882413 100644 --- a/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterImpl.java +++ b/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterImpl.java @@ -63,7 +63,6 @@ public boolean isInitialized() { } private void initializeQueueSplitterConfiguration(Buffer configuration) { - Promise promise = Promise.promise(); final List configurations = QueueSplitterConfigurationParser.parse(configuration, properties); queueSplitExecutors.clear(); queueSplitExecutors = configurations.stream().map(queueSplitterConfiguration -> { @@ -89,6 +88,7 @@ public String convertToSubQueue(final String queue, HttpServerRequest request) { public void resourceChanged(String resourceUri, Buffer resource) { if (configResourceUri() != null && configResourceUri().equals(resourceUri)) { log.info("Queue splitter configuration resource {} was updated. Going to initialize with new configuration", resourceUri); + System.out.println("Queue splitter configuration resource changed"); initializeQueueSplitterConfiguration(resource); } } @@ -97,6 +97,7 @@ public void resourceChanged(String resourceUri, Buffer resource) { public void resourceRemoved(String resourceUri) { if (configResourceUri() != null && configResourceUri().equals(resourceUri)) { log.info("Queue splitter configuration resource {} was removed. Going to close all kafka producers", resourceUri); + System.out.println("Queue splitter configuration resource removed"); queueSplitExecutors.clear(); initialized = false; } diff --git a/gateleen-queue/src/test/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterImplTest.java b/gateleen-queue/src/test/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterImplTest.java index 5e89bb6a9..401ff948d 100644 --- a/gateleen-queue/src/test/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterImplTest.java +++ b/gateleen-queue/src/test/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterImplTest.java @@ -1,6 +1,9 @@ package org.swisspush.gateleen.queue.queuing.splitter; +import io.vertx.core.Handler; import io.vertx.core.Vertx; +import io.vertx.core.buffer.Buffer; +import io.vertx.core.eventbus.Message; import io.vertx.core.http.HttpServerRequest; import io.vertx.core.http.impl.headers.HeadersMultiMap; import io.vertx.core.json.JsonObject; @@ -12,9 +15,15 @@ import org.junit.Test; import org.junit.runner.RunWith; import org.swisspush.gateleen.core.configuration.ConfigurationResourceManager; +import org.swisspush.gateleen.core.configuration.ConfigurationResourceObserver; import org.swisspush.gateleen.core.storage.MockResourceStorage; import org.swisspush.gateleen.core.util.ResourcesUtils; +import java.util.concurrent.atomic.AtomicInteger; + +import static java.util.concurrent.TimeUnit.SECONDS; +import static org.awaitility.Awaitility.await; +import static org.hamcrest.Matchers.equalTo; import static org.mockito.Mockito.*; import static org.swisspush.gateleen.core.configuration.ConfigurationResourceManager.CONFIG_RESOURCE_CHANGED_ADDRESS; @@ -94,54 +103,170 @@ public void splitWithQueueNotMatchingAnyConfiguration(TestContext context) { } @Test - @Ignore("verify with timeout and await don't work, review with Marc") + public void configResourceRemovedTriggerRemoveAllExecutorsOld(TestContext context) { + Async async = context.async(); + storage.putMockData(configResourceUri, CONFIG_RESOURCE_VALID); + queueSplitter.initialize().onComplete(event -> { + context.assertTrue(queueSplitter.isInitialized()); + verifySplitStaticExecuted(context); + verifySplitWithHeaderExecuted(context); + verifySplitWithUrlExecuted(context); + + configurationResourceManager.registerObserver(new ConfigurationResourceObserver() { + + @Override + public void resourceChanged(String resourceUri, Buffer resource) { + } + + @Override + public void resourceRemoved(String resourceUri) { + context.assertFalse(queueSplitter.isInitialized()); + verifySplitStaticNotExecuted(context); + verifySplitWithHeaderNotExecuted(context); + verifySplitWithUrlNotExecuted(context); + async.complete(); + } + }, configResourceUri); + + JsonObject object = new JsonObject(); + object.put("requestUri", configResourceUri); + object.put("type", "remove"); + vertx.eventBus().publish(CONFIG_RESOURCE_CHANGED_ADDRESS, object); + }); + } + + @Test public void configResourceRemovedTriggerRemoveAllExecutors(TestContext context) { Async async = context.async(); storage.putMockData(configResourceUri, CONFIG_RESOURCE_VALID); - context.assertFalse(queueSplitter.isInitialized()); queueSplitter.initialize().onComplete(event -> { context.assertTrue(queueSplitter.isInitialized()); verifySplitStaticExecuted(context); verifySplitWithHeaderExecuted(context); verifySplitWithUrlExecuted(context); + + vertx.eventBus().consumer(CONFIG_RESOURCE_CHANGED_ADDRESS, (Handler>) message -> { + context.assertEquals("remove", message.body().getString("type")); + context.assertFalse(queueSplitter.isInitialized()); + verifySplitStaticNotExecuted(context); + verifySplitWithHeaderNotExecuted(context); + verifySplitWithUrlNotExecuted(context); + async.complete(); + }); JsonObject object = new JsonObject(); object.put("requestUri", configResourceUri); object.put("type", "remove"); vertx.eventBus().publish(CONFIG_RESOURCE_CHANGED_ADDRESS, object); - try { Thread.sleep(10000);} catch (InterruptedException e) {} - // verify(storage, timeout(100).times(1)).delete(eq(CONFIG_RESOURCE_CHANGED_ADDRESS), any()); - // await().atMost(TWO_SECONDS).until(() -> queueSplitter.getQueueSplitExecutors(), equalTo(0)); - verifySplitStaticNotExecuted(context); - verifySplitWithHeaderNotExecuted(context); - verifySplitWithUrlNotExecuted(context); - async.complete(); }); } @Test - @Ignore("verify with timeout and await don't work, review with Marc") - public void configResourceChangedTriggerNewInitOfExecutors(TestContext context) { + public void configResourceChangedTriggerNewInitOfExecutorsOld(TestContext context) { Async async = context.async(); storage.putMockData(configResourceUri, CONFIG_RESOURCE_VALID); - context.assertFalse(queueSplitter.isInitialized()); + queueSplitter.initialize().onComplete(event -> { context.assertTrue(queueSplitter.isInitialized()); verifySplitStaticExecuted(context); verifySplitWithHeaderExecuted(context); verifySplitWithUrlExecuted(context); + + configurationResourceManager.registerObserver(new ConfigurationResourceObserver() { + + private int resourceChangedCalls = 0; + + @Override + public void resourceChanged(String resourceUri, Buffer resource) { + resourceChangedCalls++; + if (resourceChangedCalls == 2) { + context.assertTrue(queueSplitter.isInitialized()); + verifySplitStaticExecuted(context); + verifySplitWithHeaderNotExecuted(context); + verifySplitWithUrlNotExecuted(context); + async.complete(); + } + } + + @Override + public void resourceRemoved(String resourceUri) { + } + }, configResourceUri); + + storage.putMockData(configResourceUri, CONFIG_RESOURCE_INVALID); JsonObject object = new JsonObject(); object.put("requestUri", configResourceUri); object.put("type", "change"); vertx.eventBus().publish(CONFIG_RESOURCE_CHANGED_ADDRESS, object); - // try { Thread.sleep(2000);} catch (InterruptedException e) {} - // verify(storage, timeout(100).times(1)).put(eq(CONFIG_RESOURCE_CHANGED_ADDRESS), any(), any()); - verify(storage, times(1)).get(anyString(), any()); - // await().atMost(TWO_SECONDS).until(() -> queueSplitter.getQueueSplitExecutors(), equalTo(0)); + }); + } + + @Test + @Ignore("to review") + public void configResourceChangedTriggerNewInitOfExecutors1(TestContext context) { + Async async = context.async(); + storage.putMockData(configResourceUri, CONFIG_RESOURCE_VALID); + + queueSplitter.initialize().onComplete(event -> { + + context.assertTrue(queueSplitter.isInitialized()); verifySplitStaticExecuted(context); - verifySplitWithHeaderNotExecuted(context); - verifySplitWithUrlNotExecuted(context); - async.complete(); + verifySplitWithHeaderExecuted(context); + verifySplitWithUrlExecuted(context); + + storage.putMockData(configResourceUri, CONFIG_RESOURCE_INVALID); + JsonObject object = new JsonObject(); + object.put("requestUri", configResourceUri); + object.put("type", "change"); + vertx.eventBus().publish(CONFIG_RESOURCE_CHANGED_ADDRESS, object); + + await().atMost(10, SECONDS).until( () -> { + HttpServerRequest request = mock(HttpServerRequest.class); + when(request.headers()).thenReturn(new HeadersMultiMap().add("x-rp-deviceid", "A1B2C3D4E5F6")); + String value = queueSplitter.convertToSubQueue("my-queue-2", request); + System.out.println("await called"); + return "my-queue-2".equals(value); + }, equalTo(Boolean.TRUE)); + + context.assertTrue(queueSplitter.isInitialized()); +// verifySplitStaticExecuted(context); +// verifySplitWithHeaderNotExecuted(context); +// verifySplitWithUrlNotExecuted(context); +// async.complete(); + }); + } + + @Test + @Ignore("to review") + public void configResourceChangedTriggerNewInitOfExecutors2(TestContext context) { + Async async = context.async(); + storage.putMockData(configResourceUri, CONFIG_RESOURCE_VALID); + + AtomicInteger resourceChangedCalls = new AtomicInteger(0); + vertx.eventBus().consumer(CONFIG_RESOURCE_CHANGED_ADDRESS, (Handler>) message -> { + + int counter = resourceChangedCalls.incrementAndGet(); + if (counter == 1) { + context.assertEquals("change", message.body().getString("type")); + context.assertFalse(queueSplitter.isInitialized()); + verifySplitStaticNotExecuted(context); + verifySplitWithHeaderNotExecuted(context); + verifySplitWithUrlNotExecuted(context); + async.complete(); + } else if (counter == 2) { + context.assertEquals("change", message.body().getString("type")); + context.assertTrue(queueSplitter.isInitialized()); + verifySplitStaticExecuted(context); + verifySplitWithHeaderExecuted(context); + verifySplitWithUrlExecuted(context); + async.complete(); + } }); + + storage.putMockData(configResourceUri, CONFIG_RESOURCE_INVALID); + JsonObject object = new JsonObject(); + object.put("requestUri", configResourceUri); + object.put("type", "change"); + vertx.eventBus().publish(CONFIG_RESOURCE_CHANGED_ADDRESS, object); } private void verifySplitStaticExecuted(TestContext context) { From 46ffb4ce38920717b7481c1efa34f398a283dfef Mon Sep 17 00:00:00 2001 From: Giannandrea Castaldi Date: Fri, 2 Feb 2024 17:25:17 +0100 Subject: [PATCH 09/18] #550 fixed log --- .../gateleen/queue/queuing/splitter/QueueSplitterImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterImpl.java b/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterImpl.java index 74e882413..aa5faef60 100644 --- a/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterImpl.java +++ b/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterImpl.java @@ -51,7 +51,7 @@ public Future initialize() { initializeQueueSplitterConfiguration(event.result().get()); promise.complete(); } else { - log.warn("No queue splitter configuration resource with uri '{}' found. Unable to setup kafka configuration correctly", configResourceUri()); + log.warn("No queue splitter configuration resource with uri '{}' found. Unable to setup splitter configuration correctly", configResourceUri()); promise.complete(); } })); From 3739ea36ec1c0f6abce504bbb1aff07d94058d1b Mon Sep 17 00:00:00 2001 From: Giannandrea Castaldi Date: Mon, 5 Feb 2024 16:16:33 +0100 Subject: [PATCH 10/18] #550 added in playground examples of splitters --- .../playground/server/admin/v1/queueSplitters | 15 ++++++++++----- .../executors/QueueSplitExecutorFromRequest.java | 7 +++++-- .../QueueSplitExecutorFromRequestTest.java | 2 +- 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/gateleen-playground/src/main/resources/playground/server/admin/v1/queueSplitters b/gateleen-playground/src/main/resources/playground/server/admin/v1/queueSplitters index c4ae7d7c1..c96ee0218 100644 --- a/gateleen-playground/src/main/resources/playground/server/admin/v1/queueSplitters +++ b/gateleen-playground/src/main/resources/playground/server/admin/v1/queueSplitters @@ -1,5 +1,5 @@ { - "my-queue-1" : { + "queue-static-split": { "postfixFromStatic": [ "A", "B", @@ -7,12 +7,17 @@ "D" ] }, - "my-queue-[0-9]+" : { + "queue-header-[a-z]+": { "postfixDelimiter": "+", - "postfixFromHeader": "{x-rp-deviceid}" + "postfixFromHeader": "x-rp-deviceid" }, - "my-queue-[a-zA-Z]+" : { + "queue-path-[a-z]+": { "postfixDelimiter": "_", - "postfixFromUrl": ".*/path1/(.*)/path3/path4/.*" + "postfixFromUrl": ".*/path1/(.*)/.*" + }, + "queue-header-and-path-[a-z]+": { + "postfixDelimiter": "_", + "postfixFromHeader": "x-rp-deviceid", + "postfixFromUrl": ".*/path1/(.*)/.*" } } \ No newline at end of file diff --git a/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/executors/QueueSplitExecutorFromRequest.java b/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/executors/QueueSplitExecutorFromRequest.java index e907420ab..98faa9c00 100644 --- a/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/executors/QueueSplitExecutorFromRequest.java +++ b/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/executors/QueueSplitExecutorFromRequest.java @@ -25,8 +25,11 @@ public String executeSplit(String queue, HttpServerRequest request) { } } if (configuration.getPostfixFromHeader() != null) { - stringBuilder.append(configuration.getPostfixDelimiter()); - stringBuilder.append(request.headers().get(configuration.getPostfixFromHeader())); + String headerValue = request.headers().get(configuration.getPostfixFromHeader()); + if (headerValue != null) { + stringBuilder.append(configuration.getPostfixDelimiter()); + stringBuilder.append(headerValue); + } } } return stringBuilder.toString(); diff --git a/gateleen-queue/src/test/java/org/swisspush/gateleen/queue/queuing/splitter/executors/QueueSplitExecutorFromRequestTest.java b/gateleen-queue/src/test/java/org/swisspush/gateleen/queue/queuing/splitter/executors/QueueSplitExecutorFromRequestTest.java index 59a72e16a..c79f2c8a0 100644 --- a/gateleen-queue/src/test/java/org/swisspush/gateleen/queue/queuing/splitter/executors/QueueSplitExecutorFromRequestTest.java +++ b/gateleen-queue/src/test/java/org/swisspush/gateleen/queue/queuing/splitter/executors/QueueSplitExecutorFromRequestTest.java @@ -82,7 +82,7 @@ public void testExecuteSplitWithHeaderButMissingInRequest() { when(request.headers()).thenReturn(new HeadersMultiMap()); // Then - assertEquals("queue-1-null", executor.executeSplit("queue-1", request)); + assertEquals("queue-1", executor.executeSplit("queue-1", request)); } @Test From 00b1b0336632211260998e5e421e8a4d54de9d4a Mon Sep 17 00:00:00 2001 From: Giannandrea Castaldi Date: Tue, 6 Feb 2024 09:03:02 +0100 Subject: [PATCH 11/18] #550 reviewed tests --- .../splitter/QueueSplitterImplTest.java | 130 +++++------------- 1 file changed, 34 insertions(+), 96 deletions(-) diff --git a/gateleen-queue/src/test/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterImplTest.java b/gateleen-queue/src/test/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterImplTest.java index 401ff948d..44e9fca04 100644 --- a/gateleen-queue/src/test/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterImplTest.java +++ b/gateleen-queue/src/test/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterImplTest.java @@ -49,9 +49,15 @@ public void setUp() { @Test public void splitWithMissingConfigResource(TestContext context) { + + // Given Async async = context.async(); context.assertFalse(queueSplitter.isInitialized()); + + // When queueSplitter.initialize().onComplete(event -> { + + // Then context.assertFalse(queueSplitter.isInitialized()); verifySplitStaticNotExecuted(context); verifySplitWithHeaderNotExecuted(context); @@ -62,10 +68,16 @@ public void splitWithMissingConfigResource(TestContext context) { @Test public void splitWithValidConfigResource(TestContext context) { + + // Given Async async = context.async(); storage.putMockData(configResourceUri, CONFIG_RESOURCE_VALID); context.assertFalse(queueSplitter.isInitialized()); + + // When queueSplitter.initialize().onComplete(event -> { + + // Then context.assertTrue(queueSplitter.isInitialized()); verifySplitStaticExecuted(context); verifySplitWithHeaderExecuted(context); @@ -76,10 +88,16 @@ public void splitWithValidConfigResource(TestContext context) { @Test public void splitWithPartiallyInvalidConfigResource(TestContext context) { + + // Given Async async = context.async(); storage.putMockData(configResourceUri, CONFIG_RESOURCE_INVALID); context.assertFalse(queueSplitter.isInitialized()); + + // When queueSplitter.initialize().onComplete(event -> { + + // Then context.assertTrue(queueSplitter.isInitialized()); verifySplitStaticExecuted(context); verifySplitWithHeaderNotExecuted(context); @@ -91,10 +109,16 @@ public void splitWithPartiallyInvalidConfigResource(TestContext context) { @Test public void splitWithQueueNotMatchingAnyConfiguration(TestContext context) { + + // Given Async async = context.async(); storage.putMockData(configResourceUri, CONFIG_RESOURCE_VALID); context.assertFalse(queueSplitter.isInitialized()); + + // When queueSplitter.initialize().onComplete(event -> { + + // Then HttpServerRequest request = mock(HttpServerRequest.class); when(request.headers()).thenReturn(new HeadersMultiMap()); context.assertEquals("another-queue", queueSplitter.convertToSubQueue("another-queue", request)); @@ -103,7 +127,9 @@ public void splitWithQueueNotMatchingAnyConfiguration(TestContext context) { } @Test - public void configResourceRemovedTriggerRemoveAllExecutorsOld(TestContext context) { + public void configResourceRemovedTriggerRemoveAllExecutors(TestContext context) { + + // Given Async async = context.async(); storage.putMockData(configResourceUri, CONFIG_RESOURCE_VALID); queueSplitter.initialize().onComplete(event -> { @@ -120,6 +146,8 @@ public void resourceChanged(String resourceUri, Buffer resource) { @Override public void resourceRemoved(String resourceUri) { + + // Then context.assertFalse(queueSplitter.isInitialized()); verifySplitStaticNotExecuted(context); verifySplitWithHeaderNotExecuted(context); @@ -128,6 +156,7 @@ public void resourceRemoved(String resourceUri) { } }, configResourceUri); + // When JsonObject object = new JsonObject(); object.put("requestUri", configResourceUri); object.put("type", "remove"); @@ -136,35 +165,11 @@ public void resourceRemoved(String resourceUri) { } @Test - public void configResourceRemovedTriggerRemoveAllExecutors(TestContext context) { - Async async = context.async(); - storage.putMockData(configResourceUri, CONFIG_RESOURCE_VALID); - queueSplitter.initialize().onComplete(event -> { - context.assertTrue(queueSplitter.isInitialized()); - verifySplitStaticExecuted(context); - verifySplitWithHeaderExecuted(context); - verifySplitWithUrlExecuted(context); - - vertx.eventBus().consumer(CONFIG_RESOURCE_CHANGED_ADDRESS, (Handler>) message -> { - context.assertEquals("remove", message.body().getString("type")); - context.assertFalse(queueSplitter.isInitialized()); - verifySplitStaticNotExecuted(context); - verifySplitWithHeaderNotExecuted(context); - verifySplitWithUrlNotExecuted(context); - async.complete(); - }); - JsonObject object = new JsonObject(); - object.put("requestUri", configResourceUri); - object.put("type", "remove"); - vertx.eventBus().publish(CONFIG_RESOURCE_CHANGED_ADDRESS, object); - }); - } + public void configResourceChangedTriggerNewInitOfExecutors(TestContext context) { - @Test - public void configResourceChangedTriggerNewInitOfExecutorsOld(TestContext context) { + // Given Async async = context.async(); storage.putMockData(configResourceUri, CONFIG_RESOURCE_VALID); - queueSplitter.initialize().onComplete(event -> { context.assertTrue(queueSplitter.isInitialized()); verifySplitStaticExecuted(context); @@ -177,6 +182,7 @@ public void configResourceChangedTriggerNewInitOfExecutorsOld(TestContext contex @Override public void resourceChanged(String resourceUri, Buffer resource) { + // Then resourceChangedCalls++; if (resourceChangedCalls == 2) { context.assertTrue(queueSplitter.isInitialized()); @@ -192,6 +198,7 @@ public void resourceRemoved(String resourceUri) { } }, configResourceUri); + // When storage.putMockData(configResourceUri, CONFIG_RESOURCE_INVALID); JsonObject object = new JsonObject(); object.put("requestUri", configResourceUri); @@ -200,75 +207,6 @@ public void resourceRemoved(String resourceUri) { }); } - @Test - @Ignore("to review") - public void configResourceChangedTriggerNewInitOfExecutors1(TestContext context) { - Async async = context.async(); - storage.putMockData(configResourceUri, CONFIG_RESOURCE_VALID); - - queueSplitter.initialize().onComplete(event -> { - - context.assertTrue(queueSplitter.isInitialized()); - verifySplitStaticExecuted(context); - verifySplitWithHeaderExecuted(context); - verifySplitWithUrlExecuted(context); - - storage.putMockData(configResourceUri, CONFIG_RESOURCE_INVALID); - JsonObject object = new JsonObject(); - object.put("requestUri", configResourceUri); - object.put("type", "change"); - vertx.eventBus().publish(CONFIG_RESOURCE_CHANGED_ADDRESS, object); - - await().atMost(10, SECONDS).until( () -> { - HttpServerRequest request = mock(HttpServerRequest.class); - when(request.headers()).thenReturn(new HeadersMultiMap().add("x-rp-deviceid", "A1B2C3D4E5F6")); - String value = queueSplitter.convertToSubQueue("my-queue-2", request); - System.out.println("await called"); - return "my-queue-2".equals(value); - }, equalTo(Boolean.TRUE)); - - context.assertTrue(queueSplitter.isInitialized()); -// verifySplitStaticExecuted(context); -// verifySplitWithHeaderNotExecuted(context); -// verifySplitWithUrlNotExecuted(context); -// async.complete(); - }); - } - - @Test - @Ignore("to review") - public void configResourceChangedTriggerNewInitOfExecutors2(TestContext context) { - Async async = context.async(); - storage.putMockData(configResourceUri, CONFIG_RESOURCE_VALID); - - AtomicInteger resourceChangedCalls = new AtomicInteger(0); - vertx.eventBus().consumer(CONFIG_RESOURCE_CHANGED_ADDRESS, (Handler>) message -> { - - int counter = resourceChangedCalls.incrementAndGet(); - if (counter == 1) { - context.assertEquals("change", message.body().getString("type")); - context.assertFalse(queueSplitter.isInitialized()); - verifySplitStaticNotExecuted(context); - verifySplitWithHeaderNotExecuted(context); - verifySplitWithUrlNotExecuted(context); - async.complete(); - } else if (counter == 2) { - context.assertEquals("change", message.body().getString("type")); - context.assertTrue(queueSplitter.isInitialized()); - verifySplitStaticExecuted(context); - verifySplitWithHeaderExecuted(context); - verifySplitWithUrlExecuted(context); - async.complete(); - } - }); - - storage.putMockData(configResourceUri, CONFIG_RESOURCE_INVALID); - JsonObject object = new JsonObject(); - object.put("requestUri", configResourceUri); - object.put("type", "change"); - vertx.eventBus().publish(CONFIG_RESOURCE_CHANGED_ADDRESS, object); - } - private void verifySplitStaticExecuted(TestContext context) { HttpServerRequest request = mock(HttpServerRequest.class); context.assertEquals("my-queue-1-A", queueSplitter.convertToSubQueue("my-queue-1", request)); From 4a4eecd06d61d9e10dc1165f31e2f982578eb8f5 Mon Sep 17 00:00:00 2001 From: Giannandrea Castaldi Date: Tue, 6 Feb 2024 10:26:09 +0100 Subject: [PATCH 12/18] #550 refactoring --- .../QueueSplitterConfigurationParser.java | 26 +++++------ .../queuing/splitter/QueueSplitterImpl.java | 4 +- ... => QueueSplitExecutorFromStaticList.java} | 4 +- .../QueueSplitExecutorFromRequestTest.java | 2 +- ...QueueSplitExecutorFromStaticListTest.java} | 10 ++-- .../gateleen/runconfig/RunConfig.java | 46 ++++++++++++------- 6 files changed, 53 insertions(+), 39 deletions(-) rename gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/executors/{QueueSplitExecutorFromList.java => QueueSplitExecutorFromStaticList.java} (85%) rename gateleen-queue/src/test/java/org/swisspush/gateleen/queue/queuing/splitter/executors/{QueueSplitExecutorFromListTest.java => QueueSplitExecutorFromStaticListTest.java} (82%) diff --git a/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterConfigurationParser.java b/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterConfigurationParser.java index e892e8be0..d84b0672c 100644 --- a/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterConfigurationParser.java +++ b/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterConfigurationParser.java @@ -59,20 +59,20 @@ static List parse(Buffer configurationResourceBuffer null, null )); - continue; - } - String postfixFromHeader = queueConfig.getString(POSTFIX_FROM_HEADER_KEY); - String postfixFromUrl = queueConfig.getString(POSTFIX_FROM_URL_KEY); - if (postfixFromHeader != null || postfixFromUrl != null) { - queueSplitterConfigurations.add(new QueueSplitterConfiguration( - pattern, - queueConfig.getString(POSTFIX_DELIMITER_KEY, DEFAULT_POSTFIX_DELIMITER), - null, - postfixFromHeader, - postfixFromUrl != null ? Pattern.compile(postfixFromUrl) : null - )); } else { - log.warn("Queue splitter configuration without a postfix definition"); + String postfixFromHeader = queueConfig.getString(POSTFIX_FROM_HEADER_KEY); + String postfixFromUrl = queueConfig.getString(POSTFIX_FROM_URL_KEY); + if (postfixFromHeader != null || postfixFromUrl != null) { + queueSplitterConfigurations.add(new QueueSplitterConfiguration( + pattern, + queueConfig.getString(POSTFIX_DELIMITER_KEY, DEFAULT_POSTFIX_DELIMITER), + null, + postfixFromHeader, + postfixFromUrl != null ? Pattern.compile(postfixFromUrl) : null + )); + } else { + log.warn("Queue splitter configuration without a postfix definition"); + } } } catch (PatternSyntaxException patternException) { log.warn("Queue splitter '{}' is not a valid regex pattern. Discarding this queue splitter configuration", queuePattern); diff --git a/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterImpl.java b/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterImpl.java index aa5faef60..085cb3849 100644 --- a/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterImpl.java +++ b/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterImpl.java @@ -9,7 +9,7 @@ import org.swisspush.gateleen.core.configuration.ConfigurationResourceConsumer; import org.swisspush.gateleen.core.configuration.ConfigurationResourceManager; import org.swisspush.gateleen.queue.queuing.splitter.executors.QueueSplitExecutor; -import org.swisspush.gateleen.queue.queuing.splitter.executors.QueueSplitExecutorFromList; +import org.swisspush.gateleen.queue.queuing.splitter.executors.QueueSplitExecutorFromStaticList; import org.swisspush.gateleen.queue.queuing.splitter.executors.QueueSplitExecutorFromRequest; import java.util.*; @@ -67,7 +67,7 @@ private void initializeQueueSplitterConfiguration(Buffer configuration) { queueSplitExecutors.clear(); queueSplitExecutors = configurations.stream().map(queueSplitterConfiguration -> { if (queueSplitterConfiguration.isSplitStatic()) { - return new QueueSplitExecutorFromList(queueSplitterConfiguration); + return new QueueSplitExecutorFromStaticList(queueSplitterConfiguration); } else { return new QueueSplitExecutorFromRequest(queueSplitterConfiguration); } diff --git a/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/executors/QueueSplitExecutorFromList.java b/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/executors/QueueSplitExecutorFromStaticList.java similarity index 85% rename from gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/executors/QueueSplitExecutorFromList.java rename to gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/executors/QueueSplitExecutorFromStaticList.java index 8124f85a3..d23d80561 100644 --- a/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/executors/QueueSplitExecutorFromList.java +++ b/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/executors/QueueSplitExecutorFromStaticList.java @@ -5,11 +5,11 @@ import java.util.concurrent.atomic.AtomicInteger; -public class QueueSplitExecutorFromList extends QueueSplitExecutorBase { +public class QueueSplitExecutorFromStaticList extends QueueSplitExecutorBase { AtomicInteger atomicInteger = new AtomicInteger(0); - public QueueSplitExecutorFromList(QueueSplitterConfiguration configuration) { + public QueueSplitExecutorFromStaticList(QueueSplitterConfiguration configuration) { super(configuration); } diff --git a/gateleen-queue/src/test/java/org/swisspush/gateleen/queue/queuing/splitter/executors/QueueSplitExecutorFromRequestTest.java b/gateleen-queue/src/test/java/org/swisspush/gateleen/queue/queuing/splitter/executors/QueueSplitExecutorFromRequestTest.java index c79f2c8a0..9a44b5206 100644 --- a/gateleen-queue/src/test/java/org/swisspush/gateleen/queue/queuing/splitter/executors/QueueSplitExecutorFromRequestTest.java +++ b/gateleen-queue/src/test/java/org/swisspush/gateleen/queue/queuing/splitter/executors/QueueSplitExecutorFromRequestTest.java @@ -126,7 +126,7 @@ public void testExecuteSplitWithUrlAndHeader() { public void testExecuteSplitForWrongQueue() { // Given - QueueSplitExecutorFromList executor = new QueueSplitExecutorFromList(new QueueSplitterConfiguration( + QueueSplitExecutorFromRequest executor = new QueueSplitExecutorFromRequest(new QueueSplitterConfiguration( Pattern.compile("queue-1"), "-", null, diff --git a/gateleen-queue/src/test/java/org/swisspush/gateleen/queue/queuing/splitter/executors/QueueSplitExecutorFromListTest.java b/gateleen-queue/src/test/java/org/swisspush/gateleen/queue/queuing/splitter/executors/QueueSplitExecutorFromStaticListTest.java similarity index 82% rename from gateleen-queue/src/test/java/org/swisspush/gateleen/queue/queuing/splitter/executors/QueueSplitExecutorFromListTest.java rename to gateleen-queue/src/test/java/org/swisspush/gateleen/queue/queuing/splitter/executors/QueueSplitExecutorFromStaticListTest.java index e815df2e0..3a162b08b 100644 --- a/gateleen-queue/src/test/java/org/swisspush/gateleen/queue/queuing/splitter/executors/QueueSplitExecutorFromListTest.java +++ b/gateleen-queue/src/test/java/org/swisspush/gateleen/queue/queuing/splitter/executors/QueueSplitExecutorFromStaticListTest.java @@ -10,13 +10,13 @@ import static org.junit.Assert.*; import static org.mockito.Mockito.mock; -public class QueueSplitExecutorFromListTest { +public class QueueSplitExecutorFromStaticListTest { @Test public void testMatchesWithStaticQueueName() { // Given - QueueSplitExecutorFromList executor = new QueueSplitExecutorFromList(new QueueSplitterConfiguration( + QueueSplitExecutorFromStaticList executor = new QueueSplitExecutorFromStaticList(new QueueSplitterConfiguration( Pattern.compile("queue-1"), "-", List.of("A", "B", "C", "D"), @@ -33,7 +33,7 @@ public void testMatchesWithStaticQueueName() { public void testMatchesWithWildCharQueueName() { // Given - QueueSplitExecutorFromList executor = new QueueSplitExecutorFromList(new QueueSplitterConfiguration( + QueueSplitExecutorFromStaticList executor = new QueueSplitExecutorFromStaticList(new QueueSplitterConfiguration( Pattern.compile("queue-[0-9]+"), "-", List.of("A", "B", "C", "D"), @@ -52,7 +52,7 @@ public void testMatchesWithWildCharQueueName() { public void testExecuteSplit() { // Given - QueueSplitExecutorFromList executor = new QueueSplitExecutorFromList(new QueueSplitterConfiguration( + QueueSplitExecutorFromStaticList executor = new QueueSplitExecutorFromStaticList(new QueueSplitterConfiguration( Pattern.compile("queue-1"), "-", List.of("A", "B", "C", "D"), @@ -77,7 +77,7 @@ public void testExecuteSplit() { public void testExecuteSplitForWrongQueue() { // Given - QueueSplitExecutorFromList executor = new QueueSplitExecutorFromList(new QueueSplitterConfiguration( + QueueSplitExecutorFromStaticList executor = new QueueSplitExecutorFromStaticList(new QueueSplitterConfiguration( Pattern.compile("queue-1"), "-", List.of("A", "B", "C", "D"), diff --git a/gateleen-runconfig/src/main/java/org/swisspush/gateleen/runconfig/RunConfig.java b/gateleen-runconfig/src/main/java/org/swisspush/gateleen/runconfig/RunConfig.java index 9bb039f7e..5f632b7e8 100755 --- a/gateleen-runconfig/src/main/java/org/swisspush/gateleen/runconfig/RunConfig.java +++ b/gateleen-runconfig/src/main/java/org/swisspush/gateleen/runconfig/RunConfig.java @@ -230,7 +230,7 @@ public static class RunConfigBuilder { private LoggingResourceManager loggingResourceManager; private ConfigurationResourceManager configurationResourceManager; private QueueCircuitBreakerConfigurationResourceManager queueCircuitBreakerConfigurationResourceManager; - private QueueSplitter queueSplitter = new NoOpQueueSplitter(); + private QueueSplitter queueSplitter; private EventBusHandler eventBusHandler; private KafkaHandler kafkaHandler; private CustomHttpResponseHandler customHttpResponseHandler; @@ -521,22 +521,22 @@ public static void deployModules(final Vertx vertx, Class verticleClass, Map { - if (event1.failed()) { - log.error("Could not load rest storage redis module", event1.cause()); - handler.handle(false); - return; - } - - // metrics module - vertx.deployVerticle("org.swisspush.metrics.MetricsModule", new DeploymentOptions().setConfig(RunConfig.buildMetricsConfig()), event2 -> { - if (event2.failed()) { - log.error("Could not load metrics module", event2.cause()); + // rest storage module + vertx.deployVerticle("org.swisspush.reststorage.RestStorageMod", new DeploymentOptions().setConfig(RunConfig.buildStorageConfig()).setInstances(4), event1 -> { + if (event1.failed()) { + log.error("Could not load rest storage redis module", event1.cause()); handler.handle(false); return; } - handler.handle(true); + + // metrics module + vertx.deployVerticle("org.swisspush.metrics.MetricsModule", new DeploymentOptions().setConfig(RunConfig.buildMetricsConfig()), event2 -> { + if (event2.failed()) { + log.error("Could not load metrics module", event2.cause()); + handler.handle(false); + return; + } + handler.handle(true); }); }); }); @@ -608,11 +608,25 @@ private void handleRequest(final RoutingContext ctx) { return; } if (PackingHandler.isPacked(request)) { - request.bodyHandler(new PackingHandler(request, new QueuingHandler(vertx, redisProvider, request, monitoringHandler, queueSplitter))); + request.bodyHandler(new PackingHandler( + request, + new QueuingHandler( + vertx, + redisProvider, + request, + monitoringHandler, + queueSplitter != null ? queueSplitter : new NoOpQueueSplitter() + ) + )); } else { if (QueuingHandler.isQueued(request)) { setISO8601Timestamps(request); - request.bodyHandler(new QueuingHandler(vertx, redisProvider, request, monitoringHandler, queueSplitter)); + request.bodyHandler(new QueuingHandler( + vertx, + redisProvider, + request, + monitoringHandler, + queueSplitter != null ? queueSplitter : new NoOpQueueSplitter())); } else { if (cacheHandler != null && cacheHandler.handle(request)) { return; From 7f30b4bd6532f559043d8bf05e3c8b28ff1b79be Mon Sep 17 00:00:00 2001 From: Giannandrea Castaldi Date: Tue, 6 Feb 2024 13:00:15 +0100 Subject: [PATCH 13/18] #550 added test NoOpQueueSplitterTest --- .../splitter/NoOpQueueSplitterTest.java | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 gateleen-queue/src/test/java/org/swisspush/gateleen/queue/queuing/splitter/NoOpQueueSplitterTest.java diff --git a/gateleen-queue/src/test/java/org/swisspush/gateleen/queue/queuing/splitter/NoOpQueueSplitterTest.java b/gateleen-queue/src/test/java/org/swisspush/gateleen/queue/queuing/splitter/NoOpQueueSplitterTest.java new file mode 100644 index 000000000..eff2942a7 --- /dev/null +++ b/gateleen-queue/src/test/java/org/swisspush/gateleen/queue/queuing/splitter/NoOpQueueSplitterTest.java @@ -0,0 +1,27 @@ +package org.swisspush.gateleen.queue.queuing.splitter; + +import io.vertx.core.http.HttpServerRequest; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.mock; + +public class NoOpQueueSplitterTest { + + @Test + public void splitKeepSameQueue() { + + // Given + NoOpQueueSplitter splitter = new NoOpQueueSplitter(); + HttpServerRequest request = mock(HttpServerRequest.class); + + // When + splitter.initialize().onComplete( event -> { + + // Then + assertEquals("my-queue-01", splitter.convertToSubQueue("my-queue-01", request)); + assertEquals("my-queue-02", splitter.convertToSubQueue("my-queue-02", request)); + assertEquals("my-queue-03", splitter.convertToSubQueue("my-queue-03", request)); + }); + } +} From ba3306b9ea37a0595b192e15857e211f0587e95a Mon Sep 17 00:00:00 2001 From: Giannandrea Castaldi Date: Wed, 7 Feb 2024 13:54:35 +0100 Subject: [PATCH 14/18] #550 fixes, refactorings and improvements in splitter --- .../playground/server/admin/v1/queueSplitters | 18 +++++-- .../queue/queuing/QueuingHandler.java | 9 +++- .../QueueSplitterConfigurationParser.java | 12 +++-- .../queuing/splitter/QueueSplitterImpl.java | 4 +- ...teleen_queue_splitter_configuration_schema | 51 +++++++++++++++++-- .../QueueSplitterConfigurationParserTest.java | 4 +- .../splitter/QueueSplitterImplTest.java | 25 +++------ ...resource_queuesplitter_configuration_valid | 18 ------- ...source_queuesplitter_configuration_valid_1 | 25 +++++++++ ...source_queuesplitter_configuration_valid_2 | 10 ++++ ...rce_queuesplitter_configuration_with_props | 4 +- .../gateleen/runconfig/RunConfig.java | 4 +- 12 files changed, 127 insertions(+), 57 deletions(-) delete mode 100644 gateleen-queue/src/test/resources/testresource_queuesplitter_configuration_valid create mode 100644 gateleen-queue/src/test/resources/testresource_queuesplitter_configuration_valid_1 create mode 100644 gateleen-queue/src/test/resources/testresource_queuesplitter_configuration_valid_2 diff --git a/gateleen-playground/src/main/resources/playground/server/admin/v1/queueSplitters b/gateleen-playground/src/main/resources/playground/server/admin/v1/queueSplitters index c96ee0218..6722e7d39 100644 --- a/gateleen-playground/src/main/resources/playground/server/admin/v1/queueSplitters +++ b/gateleen-playground/src/main/resources/playground/server/admin/v1/queueSplitters @@ -1,5 +1,6 @@ { "queue-static-split": { + "description": "Simple splitter with request header", "postfixFromStatic": [ "A", "B", @@ -8,16 +9,25 @@ ] }, "queue-header-[a-z]+": { + "description": "Simple splitter with request header", "postfixDelimiter": "+", - "postfixFromHeader": "x-rp-deviceid" + "postfixFromRequest": { + "header": "x-rp-deviceid" + } }, "queue-path-[a-z]+": { + "description": "Simple splitter with request url matching", "postfixDelimiter": "_", - "postfixFromUrl": ".*/path1/(.*)/.*" + "postfixFromRequest": { + "url": ".*/path1/(.*)/.*" + } }, "queue-header-and-path-[a-z]+": { + "description": "Simple splitter with request header and url matching", "postfixDelimiter": "_", - "postfixFromHeader": "x-rp-deviceid", - "postfixFromUrl": ".*/path1/(.*)/.*" + "postfixFromRequest": { + "header": "x-rp-deviceid", + "url": ".*/path1/(.*)/.*" + } } } \ No newline at end of file diff --git a/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/QueuingHandler.java b/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/QueuingHandler.java index b548a9c76..e95c0face 100755 --- a/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/QueuingHandler.java +++ b/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/QueuingHandler.java @@ -46,6 +46,7 @@ public QueuingHandler( ) { this(vertx, redisProvider, request, new QueueClient(vertx, monitoringHandler), new NoOpQueueSplitter()); } + public QueuingHandler( Vertx vertx, RedisProvider redisProvider, @@ -53,7 +54,13 @@ public QueuingHandler( MonitoringHandler monitoringHandler, QueueSplitter queueSplitter ) { - this(vertx, redisProvider, request, new QueueClient(vertx, monitoringHandler), queueSplitter); + this( + vertx, + redisProvider, + request, + new QueueClient(vertx, monitoringHandler), + queueSplitter == null ? new NoOpQueueSplitter() : queueSplitter + ); } public QueuingHandler( diff --git a/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterConfigurationParser.java b/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterConfigurationParser.java index d84b0672c..0861c9dfe 100644 --- a/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterConfigurationParser.java +++ b/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterConfigurationParser.java @@ -24,8 +24,9 @@ public class QueueSplitterConfigurationParser { private static final Logger log = LoggerFactory.getLogger(QueueSplitterConfigurationParser.class); public static final String POSTFIX_FROM_STATIC_KEY = "postfixFromStatic"; - public static final String POSTFIX_FROM_HEADER_KEY = "postfixFromHeader"; - public static final String POSTFIX_FROM_URL_KEY = "postfixFromUrl"; + public static final String POSTFIX_FROM_REQUEST_KEY = "postfixFromRequest"; + public static final String POSTFIX_FROM_HEADER_KEY = "header"; + public static final String POSTFIX_FROM_URL_KEY = "url"; public static final String POSTFIX_DELIMITER_KEY = "postfixDelimiter"; public static final String DEFAULT_POSTFIX_DELIMITER = "-"; @@ -60,9 +61,10 @@ static List parse(Buffer configurationResourceBuffer null )); } else { - String postfixFromHeader = queueConfig.getString(POSTFIX_FROM_HEADER_KEY); - String postfixFromUrl = queueConfig.getString(POSTFIX_FROM_URL_KEY); - if (postfixFromHeader != null || postfixFromUrl != null) { + JsonObject postfixFromRequest = queueConfig.getJsonObject(POSTFIX_FROM_REQUEST_KEY); + String postfixFromHeader = postfixFromRequest != null ? postfixFromRequest.getString(POSTFIX_FROM_HEADER_KEY) : null; + String postfixFromUrl = postfixFromRequest != null ? postfixFromRequest.getString(POSTFIX_FROM_URL_KEY) : null; + if (postfixFromRequest != null && (postfixFromHeader != null || postfixFromUrl != null)) { queueSplitterConfigurations.add(new QueueSplitterConfiguration( pattern, queueConfig.getString(POSTFIX_DELIMITER_KEY, DEFAULT_POSTFIX_DELIMITER), diff --git a/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterImpl.java b/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterImpl.java index 085cb3849..3e2f4779e 100644 --- a/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterImpl.java +++ b/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterImpl.java @@ -88,7 +88,6 @@ public String convertToSubQueue(final String queue, HttpServerRequest request) { public void resourceChanged(String resourceUri, Buffer resource) { if (configResourceUri() != null && configResourceUri().equals(resourceUri)) { log.info("Queue splitter configuration resource {} was updated. Going to initialize with new configuration", resourceUri); - System.out.println("Queue splitter configuration resource changed"); initializeQueueSplitterConfiguration(resource); } } @@ -96,8 +95,7 @@ public void resourceChanged(String resourceUri, Buffer resource) { @Override public void resourceRemoved(String resourceUri) { if (configResourceUri() != null && configResourceUri().equals(resourceUri)) { - log.info("Queue splitter configuration resource {} was removed. Going to close all kafka producers", resourceUri); - System.out.println("Queue splitter configuration resource removed"); + log.info("Queue splitter configuration resource {} was removed. Going to release all executors", resourceUri); queueSplitExecutors.clear(); initialized = false; } diff --git a/gateleen-queue/src/main/resources/gateleen_queue_splitter_configuration_schema b/gateleen-queue/src/main/resources/gateleen_queue_splitter_configuration_schema index ab8dbb66b..650534405 100644 --- a/gateleen-queue/src/main/resources/gateleen_queue_splitter_configuration_schema +++ b/gateleen-queue/src/main/resources/gateleen_queue_splitter_configuration_schema @@ -1,5 +1,6 @@ { "$schema": "http://json-schema.org/draft-04/schema#", + "description": "A list of queue splitter configurations", "type": "object", "additionalProperties": { "$ref": "#/definitions/QueueSplitter" @@ -9,10 +10,16 @@ "description": "A single queue splitter configuration", "type": "object", "properties": { + "description": { + "description": "description of the splitter", + "type": "string" + }, "postfixDelimiter": { + "description": "Separator between original queue and postfix added", "type": "string" }, "postfixFromStatic": { + "description": "List of postfixes to use to compose sub-queues", "type": "array", "items": { "type": "string" @@ -20,13 +27,49 @@ "minItems": 1, "uniqueItems": true }, - "postfixFromHeader": { + "postfixFromRequest": { + "$ref": "#/definitions/PostfixFromRequest" + } + }, + "additionalProperties": false, + "oneOf": [ + { + "required": [ + "postfixFromStatic" + ] + }, + { + "required": [ + "postfixFromRequest" + ] + } + ] + }, + "PostfixFromRequest": { + "description": "Postfix generated using request header and/or url", + "anyOf": [ + { + "required": [ + "header" + ] + }, + { + "required": [ + "url" + ] + } + ], + "properties": { + "header": { + "description": "Header to use as postfix", + "type": "string" + }, + "url": { + "description": "Regex to group postfixes from url", "type": "string" } }, - "postfixFromUrl": { - "type": "string" - } + "additionalProperties": false } } } \ No newline at end of file diff --git a/gateleen-queue/src/test/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterConfigurationParserTest.java b/gateleen-queue/src/test/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterConfigurationParserTest.java index e5dc3bcc0..7d635776f 100644 --- a/gateleen-queue/src/test/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterConfigurationParserTest.java +++ b/gateleen-queue/src/test/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterConfigurationParserTest.java @@ -21,7 +21,7 @@ public class QueueSplitterConfigurationParserTest { private final String CONFIGURATION_VALID = ResourcesUtils.loadResource( - "testresource_queuesplitter_configuration_valid", + "testresource_queuesplitter_configuration_valid_1", true ); @@ -147,7 +147,7 @@ public void parseWithValidAndProps(TestContext context) { context.assertEquals(Pattern.compile("my-queue-[0-9]+").pattern(), config_1.getQueue().pattern()); context.assertEquals("_", config_1.getPostfixDelimiter()); context.assertNull(config_1.getPostfixFromStatic()); - context.assertEquals("{x-rp-deviceid}", config_1.getPostfixFromHeader()); + context.assertEquals("x-rp-deviceid", config_1.getPostfixFromHeader()); context.assertNull(config_1.getPostfixFromUrl()); } @Test diff --git a/gateleen-queue/src/test/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterImplTest.java b/gateleen-queue/src/test/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterImplTest.java index 44e9fca04..9adeb3d58 100644 --- a/gateleen-queue/src/test/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterImplTest.java +++ b/gateleen-queue/src/test/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterImplTest.java @@ -1,9 +1,7 @@ package org.swisspush.gateleen.queue.queuing.splitter; -import io.vertx.core.Handler; import io.vertx.core.Vertx; import io.vertx.core.buffer.Buffer; -import io.vertx.core.eventbus.Message; import io.vertx.core.http.HttpServerRequest; import io.vertx.core.http.impl.headers.HeadersMultiMap; import io.vertx.core.json.JsonObject; @@ -11,7 +9,6 @@ import io.vertx.ext.unit.TestContext; import io.vertx.ext.unit.junit.VertxUnitRunner; import org.junit.Before; -import org.junit.Ignore; import org.junit.Test; import org.junit.runner.RunWith; import org.swisspush.gateleen.core.configuration.ConfigurationResourceManager; @@ -19,11 +16,7 @@ import org.swisspush.gateleen.core.storage.MockResourceStorage; import org.swisspush.gateleen.core.util.ResourcesUtils; -import java.util.concurrent.atomic.AtomicInteger; - -import static java.util.concurrent.TimeUnit.SECONDS; import static org.awaitility.Awaitility.await; -import static org.hamcrest.Matchers.equalTo; import static org.mockito.Mockito.*; import static org.swisspush.gateleen.core.configuration.ConfigurationResourceManager.CONFIG_RESOURCE_CHANGED_ADDRESS; @@ -36,7 +29,8 @@ public class QueueSplitterImplTest { private ConfigurationResourceManager configurationResourceManager; private QueueSplitterImpl queueSplitter; - private final String CONFIG_RESOURCE_VALID = ResourcesUtils.loadResource("testresource_queuesplitter_configuration_valid", true); + private final String CONFIG_RESOURCE_VALID_1 = ResourcesUtils.loadResource("testresource_queuesplitter_configuration_valid_1", true); + private final String CONFIG_RESOURCE_VALID_2 = ResourcesUtils.loadResource("testresource_queuesplitter_configuration_valid_2", true); private final String CONFIG_RESOURCE_INVALID = ResourcesUtils.loadResource("testresource_queuesplitter_configuration_invalid", true); @Before @@ -71,7 +65,7 @@ public void splitWithValidConfigResource(TestContext context) { // Given Async async = context.async(); - storage.putMockData(configResourceUri, CONFIG_RESOURCE_VALID); + storage.putMockData(configResourceUri, CONFIG_RESOURCE_VALID_1); context.assertFalse(queueSplitter.isInitialized()); // When @@ -98,10 +92,7 @@ public void splitWithPartiallyInvalidConfigResource(TestContext context) { queueSplitter.initialize().onComplete(event -> { // Then - context.assertTrue(queueSplitter.isInitialized()); - verifySplitStaticExecuted(context); - verifySplitWithHeaderNotExecuted(context); - verifySplitWithUrlNotExecuted(context); + context.assertFalse(queueSplitter.isInitialized()); async.complete(); }); } @@ -112,7 +103,7 @@ public void splitWithQueueNotMatchingAnyConfiguration(TestContext context) { // Given Async async = context.async(); - storage.putMockData(configResourceUri, CONFIG_RESOURCE_VALID); + storage.putMockData(configResourceUri, CONFIG_RESOURCE_VALID_1); context.assertFalse(queueSplitter.isInitialized()); // When @@ -131,7 +122,7 @@ public void configResourceRemovedTriggerRemoveAllExecutors(TestContext context) // Given Async async = context.async(); - storage.putMockData(configResourceUri, CONFIG_RESOURCE_VALID); + storage.putMockData(configResourceUri, CONFIG_RESOURCE_VALID_1); queueSplitter.initialize().onComplete(event -> { context.assertTrue(queueSplitter.isInitialized()); verifySplitStaticExecuted(context); @@ -169,7 +160,7 @@ public void configResourceChangedTriggerNewInitOfExecutors(TestContext context) // Given Async async = context.async(); - storage.putMockData(configResourceUri, CONFIG_RESOURCE_VALID); + storage.putMockData(configResourceUri, CONFIG_RESOURCE_VALID_1); queueSplitter.initialize().onComplete(event -> { context.assertTrue(queueSplitter.isInitialized()); verifySplitStaticExecuted(context); @@ -199,7 +190,7 @@ public void resourceRemoved(String resourceUri) { }, configResourceUri); // When - storage.putMockData(configResourceUri, CONFIG_RESOURCE_INVALID); + storage.putMockData(configResourceUri, CONFIG_RESOURCE_VALID_2); JsonObject object = new JsonObject(); object.put("requestUri", configResourceUri); object.put("type", "change"); diff --git a/gateleen-queue/src/test/resources/testresource_queuesplitter_configuration_valid b/gateleen-queue/src/test/resources/testresource_queuesplitter_configuration_valid deleted file mode 100644 index bd359d385..000000000 --- a/gateleen-queue/src/test/resources/testresource_queuesplitter_configuration_valid +++ /dev/null @@ -1,18 +0,0 @@ -{ - "my-queue-1" : { - "postfixFromStatic": [ - "A", - "B", - "C", - "D" - ] - }, - "my-queue-[0-9]+" : { - "postfixDelimiter": "+", - "postfixFromHeader": "x-rp-deviceid" - }, - "my-queue-[a-zA-Z]+" : { - "postfixDelimiter": "_", - "postfixFromUrl": ".*/path1/(.*)/path3/path4/.*" - } -} \ No newline at end of file diff --git a/gateleen-queue/src/test/resources/testresource_queuesplitter_configuration_valid_1 b/gateleen-queue/src/test/resources/testresource_queuesplitter_configuration_valid_1 new file mode 100644 index 000000000..7782e233b --- /dev/null +++ b/gateleen-queue/src/test/resources/testresource_queuesplitter_configuration_valid_1 @@ -0,0 +1,25 @@ +{ + "my-queue-1" : { + "description": "Simple splitter with static list", + "postfixFromStatic": [ + "A", + "B", + "C", + "D" + ] + }, + "my-queue-[0-9]+" : { + "description": "Simple splitter with request header", + "postfixFromRequest": { + "header": "x-rp-deviceid" + }, + "postfixDelimiter": "+" + }, + "my-queue-[a-zA-Z]+" : { + "description": "Simple splitter with request url matching", + "postfixFromRequest" : { + "url": ".*/path1/(.*)/path3/path4/.*" + }, + "postfixDelimiter": "_" + } +} \ No newline at end of file diff --git a/gateleen-queue/src/test/resources/testresource_queuesplitter_configuration_valid_2 b/gateleen-queue/src/test/resources/testresource_queuesplitter_configuration_valid_2 new file mode 100644 index 000000000..abf394389 --- /dev/null +++ b/gateleen-queue/src/test/resources/testresource_queuesplitter_configuration_valid_2 @@ -0,0 +1,10 @@ +{ + "my-queue-1" : { + "postfixFromStatic": [ + "A", + "B", + "C", + "D" + ] + } +} \ No newline at end of file diff --git a/gateleen-queue/src/test/resources/testresource_queuesplitter_configuration_with_props b/gateleen-queue/src/test/resources/testresource_queuesplitter_configuration_with_props index ae03103c8..3bde49078 100644 --- a/gateleen-queue/src/test/resources/testresource_queuesplitter_configuration_with_props +++ b/gateleen-queue/src/test/resources/testresource_queuesplitter_configuration_with_props @@ -1,6 +1,8 @@ { "my-queue-[0-9]+" : { "postfixDelimiter": "${queue.splitter.delimiter}", - "postfixFromHeader": "{x-rp-deviceid}" + "postfixFromRequest": { + "header": "x-rp-deviceid" + } } } \ No newline at end of file diff --git a/gateleen-runconfig/src/main/java/org/swisspush/gateleen/runconfig/RunConfig.java b/gateleen-runconfig/src/main/java/org/swisspush/gateleen/runconfig/RunConfig.java index 5f632b7e8..824add3af 100755 --- a/gateleen-runconfig/src/main/java/org/swisspush/gateleen/runconfig/RunConfig.java +++ b/gateleen-runconfig/src/main/java/org/swisspush/gateleen/runconfig/RunConfig.java @@ -615,7 +615,7 @@ private void handleRequest(final RoutingContext ctx) { redisProvider, request, monitoringHandler, - queueSplitter != null ? queueSplitter : new NoOpQueueSplitter() + queueSplitter ) )); } else { @@ -626,7 +626,7 @@ private void handleRequest(final RoutingContext ctx) { redisProvider, request, monitoringHandler, - queueSplitter != null ? queueSplitter : new NoOpQueueSplitter())); + queueSplitter)); } else { if (cacheHandler != null && cacheHandler.handle(request)) { return; From 4f6d4c974a1498bd1a9b8f0d49a7ac32e0d25648 Mon Sep 17 00:00:00 2001 From: Giannandrea Castaldi Date: Wed, 7 Feb 2024 18:31:34 +0100 Subject: [PATCH 15/18] #550 fixes, refactorings and improvements in splitter --- .../QueueSplitterConfigurationParser.java | 2 +- .../queuing/splitter/QueueSplitterImpl.java | 10 +-- ...teleen_queue_splitter_configuration_schema | 2 +- .../splitter/NoOpQueueSplitterTest.java | 16 +++-- ...ueueConfigurationSchemaValidationTest.java | 70 +++++++++++++++++++ .../QueueSplitterConfigurationParserTest.java | 2 +- .../splitter/QueueSplitterImplTest.java | 20 ++---- ...euesplitter_configuration_missing_postfix} | 0 ...tter_configuration_missing_postfix_request | 14 ++++ 9 files changed, 104 insertions(+), 32 deletions(-) create mode 100644 gateleen-queue/src/test/java/org/swisspush/gateleen/queue/queuing/splitter/QueueConfigurationSchemaValidationTest.java rename gateleen-queue/src/test/resources/{testresource_queuesplitter_configuration_invalid => testresource_queuesplitter_configuration_missing_postfix} (100%) create mode 100644 gateleen-queue/src/test/resources/testresource_queuesplitter_configuration_missing_postfix_request diff --git a/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterConfigurationParser.java b/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterConfigurationParser.java index 0861c9dfe..ad972d8a1 100644 --- a/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterConfigurationParser.java +++ b/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterConfigurationParser.java @@ -64,7 +64,7 @@ static List parse(Buffer configurationResourceBuffer JsonObject postfixFromRequest = queueConfig.getJsonObject(POSTFIX_FROM_REQUEST_KEY); String postfixFromHeader = postfixFromRequest != null ? postfixFromRequest.getString(POSTFIX_FROM_HEADER_KEY) : null; String postfixFromUrl = postfixFromRequest != null ? postfixFromRequest.getString(POSTFIX_FROM_URL_KEY) : null; - if (postfixFromRequest != null && (postfixFromHeader != null || postfixFromUrl != null)) { + if (postfixFromHeader != null || postfixFromUrl != null) { queueSplitterConfigurations.add(new QueueSplitterConfiguration( pattern, queueConfig.getString(POSTFIX_DELIMITER_KEY, DEFAULT_POSTFIX_DELIMITER), diff --git a/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterImpl.java b/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterImpl.java index 3e2f4779e..a7ebf5de9 100644 --- a/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterImpl.java +++ b/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterImpl.java @@ -9,8 +9,8 @@ import org.swisspush.gateleen.core.configuration.ConfigurationResourceConsumer; import org.swisspush.gateleen.core.configuration.ConfigurationResourceManager; import org.swisspush.gateleen.queue.queuing.splitter.executors.QueueSplitExecutor; -import org.swisspush.gateleen.queue.queuing.splitter.executors.QueueSplitExecutorFromStaticList; import org.swisspush.gateleen.queue.queuing.splitter.executors.QueueSplitExecutorFromRequest; +import org.swisspush.gateleen.queue.queuing.splitter.executors.QueueSplitExecutorFromStaticList; import java.util.*; import java.util.stream.Collectors; @@ -24,8 +24,6 @@ public class QueueSplitterImpl extends ConfigurationResourceConsumer implements private final Map properties; - private boolean initialized = false; - private List queueSplitExecutors = new ArrayList<>(); public QueueSplitterImpl( @@ -58,10 +56,6 @@ public Future initialize() { return promise.future(); } - public boolean isInitialized() { - return initialized; - } - private void initializeQueueSplitterConfiguration(Buffer configuration) { final List configurations = QueueSplitterConfigurationParser.parse(configuration, properties); queueSplitExecutors.clear(); @@ -72,7 +66,6 @@ private void initializeQueueSplitterConfiguration(Buffer configuration) { return new QueueSplitExecutorFromRequest(queueSplitterConfiguration); } }).collect(Collectors.toList()); - initialized = true; } /** @@ -97,7 +90,6 @@ public void resourceRemoved(String resourceUri) { if (configResourceUri() != null && configResourceUri().equals(resourceUri)) { log.info("Queue splitter configuration resource {} was removed. Going to release all executors", resourceUri); queueSplitExecutors.clear(); - initialized = false; } } } diff --git a/gateleen-queue/src/main/resources/gateleen_queue_splitter_configuration_schema b/gateleen-queue/src/main/resources/gateleen_queue_splitter_configuration_schema index 650534405..e799b9c34 100644 --- a/gateleen-queue/src/main/resources/gateleen_queue_splitter_configuration_schema +++ b/gateleen-queue/src/main/resources/gateleen_queue_splitter_configuration_schema @@ -1,6 +1,6 @@ { "$schema": "http://json-schema.org/draft-04/schema#", - "description": "A list of queue splitter configurations", + "description": "Queue splitter configurations", "type": "object", "additionalProperties": { "$ref": "#/definitions/QueueSplitter" diff --git a/gateleen-queue/src/test/java/org/swisspush/gateleen/queue/queuing/splitter/NoOpQueueSplitterTest.java b/gateleen-queue/src/test/java/org/swisspush/gateleen/queue/queuing/splitter/NoOpQueueSplitterTest.java index eff2942a7..db5a793a0 100644 --- a/gateleen-queue/src/test/java/org/swisspush/gateleen/queue/queuing/splitter/NoOpQueueSplitterTest.java +++ b/gateleen-queue/src/test/java/org/swisspush/gateleen/queue/queuing/splitter/NoOpQueueSplitterTest.java @@ -1,27 +1,31 @@ package org.swisspush.gateleen.queue.queuing.splitter; import io.vertx.core.http.HttpServerRequest; +import io.vertx.ext.unit.TestContext; +import io.vertx.ext.unit.junit.VertxUnitRunner; import org.junit.Test; +import org.junit.runner.RunWith; import static org.junit.Assert.assertEquals; import static org.mockito.Mockito.mock; +@RunWith(VertxUnitRunner.class) public class NoOpQueueSplitterTest { @Test - public void splitKeepSameQueue() { + public void splitKeepSameQueue(TestContext context) { // Given NoOpQueueSplitter splitter = new NoOpQueueSplitter(); HttpServerRequest request = mock(HttpServerRequest.class); // When - splitter.initialize().onComplete( event -> { + splitter.initialize().onComplete(event -> { - // Then - assertEquals("my-queue-01", splitter.convertToSubQueue("my-queue-01", request)); - assertEquals("my-queue-02", splitter.convertToSubQueue("my-queue-02", request)); - assertEquals("my-queue-03", splitter.convertToSubQueue("my-queue-03", request)); + // Then + context.assertEquals("my-queue-01", splitter.convertToSubQueue("my-queue-01", request)); + context.assertEquals("my-queue-02", splitter.convertToSubQueue("my-queue-02", request)); + context.assertEquals("my-queue-03", splitter.convertToSubQueue("my-queue-03", request)); }); } } diff --git a/gateleen-queue/src/test/java/org/swisspush/gateleen/queue/queuing/splitter/QueueConfigurationSchemaValidationTest.java b/gateleen-queue/src/test/java/org/swisspush/gateleen/queue/queuing/splitter/QueueConfigurationSchemaValidationTest.java new file mode 100644 index 000000000..bbf9ec28f --- /dev/null +++ b/gateleen-queue/src/test/java/org/swisspush/gateleen/queue/queuing/splitter/QueueConfigurationSchemaValidationTest.java @@ -0,0 +1,70 @@ +package org.swisspush.gateleen.queue.queuing.splitter; + +import io.vertx.core.buffer.Buffer; +import io.vertx.ext.unit.TestContext; +import io.vertx.ext.unit.junit.VertxUnitRunner; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.swisspush.gateleen.core.util.ResourcesUtils; +import org.swisspush.gateleen.core.validation.ValidationResult; +import org.swisspush.gateleen.core.validation.ValidationStatus; +import org.swisspush.gateleen.validation.Validator; + +@RunWith(VertxUnitRunner.class) +public class QueueConfigurationSchemaValidationTest { + + private final String CONFIG_SCHEMA = ResourcesUtils.loadResource("gateleen_queue_splitter_configuration_schema", true); + + private final Logger logger = LoggerFactory.getLogger(QueueConfigurationSchemaValidationTest.class); + + private final String CONFIG_RESOURCE_VALID = ResourcesUtils.loadResource("testresource_queuesplitter_configuration_valid_1", true); + + private final String CONFIG_RESOURCE_MISSING_POSTFIX = ResourcesUtils.loadResource("testresource_queuesplitter_configuration_missing_postfix", true); + private final String CONFIG_RESOURCE_MISSING_POSTFIX_REQUEST = ResourcesUtils.loadResource("testresource_queuesplitter_configuration_missing_postfix_request", true); + + @Test + public void testValidConfig(TestContext context) { + + // When + ValidationResult validationResult = validate(CONFIG_RESOURCE_VALID); + + // Then + context.assertNotNull(validationResult); + context.assertEquals(ValidationStatus.VALIDATED_POSITIV, validationResult.getValidationStatus()); + } + + @Test + public void testMissingPostfix(TestContext context) { + + // When + ValidationResult validationResult = validate(CONFIG_RESOURCE_MISSING_POSTFIX); + + // Then + context.assertNotNull(validationResult); + context.assertEquals(ValidationStatus.VALIDATED_NEGATIV, validationResult.getValidationStatus()); + context.assertEquals("$.my-queue-[0-9]+.postfixFromStatic: is missing but it is required", extractErrorMessage(validationResult)); + } + + @Test + public void testMissingPostfixRequest(TestContext context) { + + // When + ValidationResult validationResult = validate(CONFIG_RESOURCE_MISSING_POSTFIX_REQUEST); + + // Then + context.assertNotNull(validationResult); + context.assertEquals(ValidationStatus.VALIDATED_NEGATIV, validationResult.getValidationStatus()); + context.assertEquals("$.my-queue-[0-9]+.postfixFromRequest.header: is missing but it is required", extractErrorMessage(validationResult)); + } + + private ValidationResult validate(String loggingResource) { + return Validator.validateStatic(Buffer.buffer(loggingResource), CONFIG_SCHEMA, logger); + } + + private String extractErrorMessage(ValidationResult validationResult){ + return validationResult.getValidationDetails().getJsonObject(0).getString("message"); + } + +} diff --git a/gateleen-queue/src/test/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterConfigurationParserTest.java b/gateleen-queue/src/test/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterConfigurationParserTest.java index 7d635776f..facd53c4d 100644 --- a/gateleen-queue/src/test/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterConfigurationParserTest.java +++ b/gateleen-queue/src/test/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterConfigurationParserTest.java @@ -26,7 +26,7 @@ public class QueueSplitterConfigurationParserTest { ); private final String CONFIGURATION_INVALID = ResourcesUtils.loadResource( - "testresource_queuesplitter_configuration_invalid", + "testresource_queuesplitter_configuration_missing_postfix", true ); diff --git a/gateleen-queue/src/test/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterImplTest.java b/gateleen-queue/src/test/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterImplTest.java index 9adeb3d58..2fa996c49 100644 --- a/gateleen-queue/src/test/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterImplTest.java +++ b/gateleen-queue/src/test/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterImplTest.java @@ -16,8 +16,8 @@ import org.swisspush.gateleen.core.storage.MockResourceStorage; import org.swisspush.gateleen.core.util.ResourcesUtils; -import static org.awaitility.Awaitility.await; -import static org.mockito.Mockito.*; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; import static org.swisspush.gateleen.core.configuration.ConfigurationResourceManager.CONFIG_RESOURCE_CHANGED_ADDRESS; @RunWith(VertxUnitRunner.class) @@ -31,7 +31,7 @@ public class QueueSplitterImplTest { private final String CONFIG_RESOURCE_VALID_1 = ResourcesUtils.loadResource("testresource_queuesplitter_configuration_valid_1", true); private final String CONFIG_RESOURCE_VALID_2 = ResourcesUtils.loadResource("testresource_queuesplitter_configuration_valid_2", true); - private final String CONFIG_RESOURCE_INVALID = ResourcesUtils.loadResource("testresource_queuesplitter_configuration_invalid", true); + private final String CONFIG_RESOURCE_INVALID = ResourcesUtils.loadResource("testresource_queuesplitter_configuration_missing_postfix", true); @Before public void setUp() { @@ -46,13 +46,11 @@ public void splitWithMissingConfigResource(TestContext context) { // Given Async async = context.async(); - context.assertFalse(queueSplitter.isInitialized()); // When queueSplitter.initialize().onComplete(event -> { // Then - context.assertFalse(queueSplitter.isInitialized()); verifySplitStaticNotExecuted(context); verifySplitWithHeaderNotExecuted(context); verifySplitWithUrlNotExecuted(context); @@ -66,13 +64,11 @@ public void splitWithValidConfigResource(TestContext context) { // Given Async async = context.async(); storage.putMockData(configResourceUri, CONFIG_RESOURCE_VALID_1); - context.assertFalse(queueSplitter.isInitialized()); // When queueSplitter.initialize().onComplete(event -> { // Then - context.assertTrue(queueSplitter.isInitialized()); verifySplitStaticExecuted(context); verifySplitWithHeaderExecuted(context); verifySplitWithUrlExecuted(context); @@ -86,13 +82,14 @@ public void splitWithPartiallyInvalidConfigResource(TestContext context) { // Given Async async = context.async(); storage.putMockData(configResourceUri, CONFIG_RESOURCE_INVALID); - context.assertFalse(queueSplitter.isInitialized()); // When queueSplitter.initialize().onComplete(event -> { // Then - context.assertFalse(queueSplitter.isInitialized()); + verifySplitStaticNotExecuted(context); + verifySplitWithHeaderNotExecuted(context); + verifySplitWithUrlNotExecuted(context); async.complete(); }); } @@ -104,7 +101,6 @@ public void splitWithQueueNotMatchingAnyConfiguration(TestContext context) { // Given Async async = context.async(); storage.putMockData(configResourceUri, CONFIG_RESOURCE_VALID_1); - context.assertFalse(queueSplitter.isInitialized()); // When queueSplitter.initialize().onComplete(event -> { @@ -124,7 +120,6 @@ public void configResourceRemovedTriggerRemoveAllExecutors(TestContext context) Async async = context.async(); storage.putMockData(configResourceUri, CONFIG_RESOURCE_VALID_1); queueSplitter.initialize().onComplete(event -> { - context.assertTrue(queueSplitter.isInitialized()); verifySplitStaticExecuted(context); verifySplitWithHeaderExecuted(context); verifySplitWithUrlExecuted(context); @@ -139,7 +134,6 @@ public void resourceChanged(String resourceUri, Buffer resource) { public void resourceRemoved(String resourceUri) { // Then - context.assertFalse(queueSplitter.isInitialized()); verifySplitStaticNotExecuted(context); verifySplitWithHeaderNotExecuted(context); verifySplitWithUrlNotExecuted(context); @@ -162,7 +156,6 @@ public void configResourceChangedTriggerNewInitOfExecutors(TestContext context) Async async = context.async(); storage.putMockData(configResourceUri, CONFIG_RESOURCE_VALID_1); queueSplitter.initialize().onComplete(event -> { - context.assertTrue(queueSplitter.isInitialized()); verifySplitStaticExecuted(context); verifySplitWithHeaderExecuted(context); verifySplitWithUrlExecuted(context); @@ -176,7 +169,6 @@ public void resourceChanged(String resourceUri, Buffer resource) { // Then resourceChangedCalls++; if (resourceChangedCalls == 2) { - context.assertTrue(queueSplitter.isInitialized()); verifySplitStaticExecuted(context); verifySplitWithHeaderNotExecuted(context); verifySplitWithUrlNotExecuted(context); diff --git a/gateleen-queue/src/test/resources/testresource_queuesplitter_configuration_invalid b/gateleen-queue/src/test/resources/testresource_queuesplitter_configuration_missing_postfix similarity index 100% rename from gateleen-queue/src/test/resources/testresource_queuesplitter_configuration_invalid rename to gateleen-queue/src/test/resources/testresource_queuesplitter_configuration_missing_postfix diff --git a/gateleen-queue/src/test/resources/testresource_queuesplitter_configuration_missing_postfix_request b/gateleen-queue/src/test/resources/testresource_queuesplitter_configuration_missing_postfix_request new file mode 100644 index 000000000..a8a77721b --- /dev/null +++ b/gateleen-queue/src/test/resources/testresource_queuesplitter_configuration_missing_postfix_request @@ -0,0 +1,14 @@ +{ + "my-queue-1" : { + "postfixFromStatic": [ + "A", + "B", + "C", + "D" + ] + }, + "my-queue-[0-9]+" : { + "postfixDelimiter": "+", + "postfixFromRequest": {} + } +} \ No newline at end of file From d3939202dd8a9b148b85b0d431088265316b7cdc Mon Sep 17 00:00:00 2001 From: Giannandrea Castaldi Date: Thu, 8 Feb 2024 08:44:22 +0100 Subject: [PATCH 16/18] #550 added test to verify header matching is case insensitive --- .../QueueSplitExecutorFromRequestTest.java | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/gateleen-queue/src/test/java/org/swisspush/gateleen/queue/queuing/splitter/executors/QueueSplitExecutorFromRequestTest.java b/gateleen-queue/src/test/java/org/swisspush/gateleen/queue/queuing/splitter/executors/QueueSplitExecutorFromRequestTest.java index 9a44b5206..3aaa63a90 100644 --- a/gateleen-queue/src/test/java/org/swisspush/gateleen/queue/queuing/splitter/executors/QueueSplitExecutorFromRequestTest.java +++ b/gateleen-queue/src/test/java/org/swisspush/gateleen/queue/queuing/splitter/executors/QueueSplitExecutorFromRequestTest.java @@ -67,6 +67,24 @@ public void testExecuteSplitWithHeader() { assertEquals("queue-1-A1B2C3D4E5F6", executor.executeSplit("queue-1", request)); } + @Test + public void testExecuteSplitWithHeaderInDifferentCase() { + + // Given + QueueSplitExecutorFromRequest executor = new QueueSplitExecutorFromRequest(new QueueSplitterConfiguration( + Pattern.compile("queue-1"), + "-", + null, + "x-rp-deviceid", + null + )); + HttpServerRequest request = mock(HttpServerRequest.class); + when(request.headers()).thenReturn(new HeadersMultiMap().add("X-RP-DEVICEID", "A1B2C3D4E5F6")); + + // Then + assertEquals("queue-1-A1B2C3D4E5F6", executor.executeSplit("queue-1", request)); + } + @Test public void testExecuteSplitWithHeaderButMissingInRequest() { From 318c2ae9ad554bb9562607171e98040803266fe3 Mon Sep 17 00:00:00 2001 From: Giannandrea Castaldi Date: Thu, 8 Feb 2024 17:06:37 +0100 Subject: [PATCH 17/18] #550 extended readme --- .../playground/server/admin/v1/queueSplitters | 2 +- gateleen-queue/README_queue.md | 63 +++++++++++++++++-- 2 files changed, 60 insertions(+), 5 deletions(-) diff --git a/gateleen-playground/src/main/resources/playground/server/admin/v1/queueSplitters b/gateleen-playground/src/main/resources/playground/server/admin/v1/queueSplitters index 6722e7d39..5479df115 100644 --- a/gateleen-playground/src/main/resources/playground/server/admin/v1/queueSplitters +++ b/gateleen-playground/src/main/resources/playground/server/admin/v1/queueSplitters @@ -1,6 +1,6 @@ { "queue-static-split": { - "description": "Simple splitter with request header", + "description": "Simple static splitter", "postfixFromStatic": [ "A", "B", diff --git a/gateleen-queue/README_queue.md b/gateleen-queue/README_queue.md index b733b6924..806a10435 100644 --- a/gateleen-queue/README_queue.md +++ b/gateleen-queue/README_queue.md @@ -1,7 +1,7 @@ # gateleen-queue The gateleen-queue module provides queuing functionality and acts as a bridge to [vertx-redisques](https://github.com/swisspush/vertx-redisques). -### Queue retry configuration +## Queue retry configuration Normally, failed queue items remain in the queue until successfully processed or manually deleted. With the `x-queue-retry-xxx` request header, you are able to control this behaviour. Example values are: @@ -32,9 +32,9 @@ The state diagram below describes the possible transitions between these states: ``` +--------+ - +------->+ | open circuit + +------->+ | fail ratio reached success | | CLOSED +---------------------+ - +--------+ | | + +--------+ | open circuit | +---+----+ | ^ | | v @@ -469,4 +469,59 @@ Example: Also you have to enable the logging on the [QueueCircuitBreakerConfigurationResource](src/main/java/org/swisspush/gateleen/queue/queuing/circuitbreaker/configuration/QueueCircuitBreakerConfigurationResource.java) by calling ```java queueCircuitBreakerConfigurationResourceManager.enableResourceLogging(true); -``` \ No newline at end of file +``` + +## Queue Splitter +In case there are queues can have a large number of requests to process there is the option to configure for these queues the split of requests in sub-queues. +The split is implemented dispatching the incoming request in one of the sub-queues (so the split is always active). Before using this feature for a queue must be evaluated if the consistency of the data of the system is maintained processing the requests not in the cronological order (requests in sub-queues are processed in parallel). + +Queue splitters are configured together (for example in admin/v1/queueSplitters), each splitter configuration is composted of two parts: +* name: regex used to match the queue name of the incoming requests +* postfix rule: rule used to generate the postfix to append to the initial queue name + +There are two types of postfix rules: +* static +* based on request + +### Static postfix rule +In the rule are listed all postfixes to use splitting. Here an example of splitter configuration with static postfix rule: +```json +"queue-static-split": { + "description": "Simple static splitter", + "postfixFromStatic": [ + "A", + "B", + "C", + "D" + ] + } +``` +In this case the splitter is applied only if the queue in the request is 'queue-static-split' and the splitting is done distributing uniformly the requests in the sub-queues 'queue-static-split-A', 'queue-static-split-B', 'queue-static-split-C' and 'queue-static-split-D' (also the separator can be configured). + +### Postfix rule based on request +In the rule can be defined the header to use as postfix and/or the regex to extract parts of the url. Here two examples: +```json + "queue-header-[a-z]+": { + "description": "Simple splitter with request header", + "postfixDelimiter": "+", + "postfixFromRequest": { + "header": "x-rp-deviceid" + } + }, + "queue-path-[a-z]+": { + "description": "Simple splitter with request url matching", + "postfixDelimiter": "_", + "postfixFromRequest": { + "url": ".*/path1/(.*)/.*" + } + } +``` +In both cases the splitter is applied to all the queues matching the name regex. + +In first case is added the value of the header 'x-rp-deviceid'. If for example is entering a request with queue 'queue-header-test' and with header 'x-rp-deviceid' valued 'A1B2C3D4' the request is inserted in the sub-queue 'queue-header-test+A1B2C3D4'. +In second case are added the matching parts of the request url. If for example is entering a request with queue 'queue-path-test' and with the url ending with .../path1/path2/path3 the queue is inserted in the sub-queue 'queue-path-test_path2'. +In the playground are configured the three splitters above (playground/server/admin/v1/queueSplitters) and so you can try them. + +### Splitter implementation +The evaluation of splitting for a queue is defined in the interface [QueueSplitter](../gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitter.java) with two implementations: [QueueSplitterImpl](../gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterImpl.java) (to execute the splitters configured) and [NoOpQueueSplitter](../gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/NoOpQueueSplitter.java) (no splitter). +For each splitter configured is created either an instance of [QueueSplitExecutorFromStaticList](../gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/executors/QueueSplitExecutorFromStaticList.java) (for the case of static postfix rule) or an instance of [QueueSplitExecutorFromRequest](../gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/executors/QueueSplitExecutorFromRequest.java) (for the case of postfix rule based on request). \ No newline at end of file From c6597cdef3c6b5999559ea39d95b30c69898e7c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Weber?= Date: Fri, 9 Feb 2024 08:40:21 +0100 Subject: [PATCH 18/18] Update README_queue.md --- gateleen-queue/README_queue.md | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/gateleen-queue/README_queue.md b/gateleen-queue/README_queue.md index 806a10435..0b2617756 100644 --- a/gateleen-queue/README_queue.md +++ b/gateleen-queue/README_queue.md @@ -472,21 +472,22 @@ queueCircuitBreakerConfigurationResourceManager.enableResourceLogging(true); ``` ## Queue Splitter -In case there are queues can have a large number of requests to process there is the option to configure for these queues the split of requests in sub-queues. -The split is implemented dispatching the incoming request in one of the sub-queues (so the split is always active). Before using this feature for a queue must be evaluated if the consistency of the data of the system is maintained processing the requests not in the cronological order (requests in sub-queues are processed in parallel). +In case there are queues with a large number of queue items to process there is the option to configure these queues to split into sub-queues. +The split is implemented dispatching the incoming request in one of the sub-queues (so the split is always active). Don't use the queue splitting feature when queue items from a queue have to be in cronological order (requests in sub-queues are processed in parallel). -Queue splitters are configured together (for example in admin/v1/queueSplitters), each splitter configuration is composted of two parts: +Queue splitters are configured together (for example in admin/v1/queueSplitters), each splitter configuration is composted of three parts: * name: regex used to match the queue name of the incoming requests * postfix rule: rule used to generate the postfix to append to the initial queue name +* postfixDelimiter: Optional delimiter value to add between queue name and postfix. When not configured, _-_ is used There are two types of postfix rules: * static * based on request ### Static postfix rule -In the rule are listed all postfixes to use splitting. Here an example of splitter configuration with static postfix rule: +In the rule all postfixes for splitting are listed. Here is an example of splitter configuration with static postfix rule: ```json -"queue-static-split": { +"queue-static": { "description": "Simple static splitter", "postfixFromStatic": [ "A", @@ -496,10 +497,10 @@ In the rule are listed all postfixes to use splitting. Here an example of splitt ] } ``` -In this case the splitter is applied only if the queue in the request is 'queue-static-split' and the splitting is done distributing uniformly the requests in the sub-queues 'queue-static-split-A', 'queue-static-split-B', 'queue-static-split-C' and 'queue-static-split-D' (also the separator can be configured). +In this case the splitter is applied only if the queue in the request is 'queue-static' and the splitting is done distributing uniformly the requests in the sub-queues 'queue-static-A', 'queue-static-B', 'queue-static-C' and 'queue-static-D'. ### Postfix rule based on request -In the rule can be defined the header to use as postfix and/or the regex to extract parts of the url. Here two examples: +Request header to use as postfix and/or the regex to extract parts from the url can be defined in the _postfixFromRequest_ property. Here are two examples: ```json "queue-header-[a-z]+": { "description": "Simple splitter with request header", @@ -518,10 +519,9 @@ In the rule can be defined the header to use as postfix and/or the regex to extr ``` In both cases the splitter is applied to all the queues matching the name regex. -In first case is added the value of the header 'x-rp-deviceid'. If for example is entering a request with queue 'queue-header-test' and with header 'x-rp-deviceid' valued 'A1B2C3D4' the request is inserted in the sub-queue 'queue-header-test+A1B2C3D4'. -In second case are added the matching parts of the request url. If for example is entering a request with queue 'queue-path-test' and with the url ending with .../path1/path2/path3 the queue is inserted in the sub-queue 'queue-path-test_path2'. -In the playground are configured the three splitters above (playground/server/admin/v1/queueSplitters) and so you can try them. +In first case the value of the header 'x-rp-deviceid' is added. A queue with name 'queue-header-test' and with a request header 'x-rp-deviceid' valued 'A1B2C3D4' the request is splitted in the sub-queue 'queue-header-test+A1B2C3D4'. +For the second case the matching parts of the request url are added. A queue with name 'queue-path-test' and with the url ending with .../path1/path2/path3 the request is splitted in the sub-queue 'queue-path-test_path2'. ### Splitter implementation The evaluation of splitting for a queue is defined in the interface [QueueSplitter](../gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitter.java) with two implementations: [QueueSplitterImpl](../gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/QueueSplitterImpl.java) (to execute the splitters configured) and [NoOpQueueSplitter](../gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/NoOpQueueSplitter.java) (no splitter). -For each splitter configured is created either an instance of [QueueSplitExecutorFromStaticList](../gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/executors/QueueSplitExecutorFromStaticList.java) (for the case of static postfix rule) or an instance of [QueueSplitExecutorFromRequest](../gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/executors/QueueSplitExecutorFromRequest.java) (for the case of postfix rule based on request). \ No newline at end of file +For each splitter configured is created either an instance of [QueueSplitExecutorFromStaticList](../gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/executors/QueueSplitExecutorFromStaticList.java) (for the case of static postfix rule) or an instance of [QueueSplitExecutorFromRequest](../gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/splitter/executors/QueueSplitExecutorFromRequest.java) (for the case of postfix rule based on request).