diff --git a/gateleen-queue/README_queue.md b/gateleen-queue/README_queue.md index 0b2617756..7aaa04a08 100644 --- a/gateleen-queue/README_queue.md +++ b/gateleen-queue/README_queue.md @@ -303,6 +303,7 @@ The result will be a json object with the circuit information like the example b "status": "closed", "info": { "failRatio": 15, + "metric": "server-tests", "circuit": "/playground/server/tests/(.*)" } } @@ -323,6 +324,7 @@ The result will be a json object with the information of all circuits like the e "myCircuitHash": { "infos": { "failRatio": 15, + "metric": "server-tests", "circuit": "/playground/server/tests/(.*)" }, "status": "closed" diff --git a/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/circuitbreaker/api/QueueCircuitBreakerHttpRequestHandler.java b/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/circuitbreaker/api/QueueCircuitBreakerHttpRequestHandler.java index 12661461a..1033ee4c7 100644 --- a/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/circuitbreaker/api/QueueCircuitBreakerHttpRequestHandler.java +++ b/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/circuitbreaker/api/QueueCircuitBreakerHttpRequestHandler.java @@ -291,7 +291,7 @@ private void handleGetCircuitStatus(Message message) { private void handleCloseCircuit(Message message) { String circuitHash = message.body().getJsonObject(PAYLOAD).getString(CIRCUIT_HASH); - PatternAndCircuitHash patternAndCircuitHash = new PatternAndCircuitHash(null, circuitHash); + PatternAndCircuitHash patternAndCircuitHash = new PatternAndCircuitHash(null, circuitHash, null); storage.closeCircuit(patternAndCircuitHash).onComplete(event -> { if (event.failed()) { message.reply(new JsonObject().put(STATUS, ERROR).put(MESSAGE, event.cause().getMessage())); diff --git a/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/circuitbreaker/impl/RedisQueueCircuitBreakerStorage.java b/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/circuitbreaker/impl/RedisQueueCircuitBreakerStorage.java index 1746916f6..7833d4fd9 100644 --- a/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/circuitbreaker/impl/RedisQueueCircuitBreakerStorage.java +++ b/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/circuitbreaker/impl/RedisQueueCircuitBreakerStorage.java @@ -40,6 +40,7 @@ public class RedisQueueCircuitBreakerStorage implements QueueCircuitBreakerStora public static final String FIELD_STATE = "state"; public static final String FIELD_FAILRATIO = "failRatio"; public static final String FIELD_CIRCUIT = "circuit"; + public static final String FIELD_METRICNAME = "metric"; private final LuaScriptState openCircuitLuaScriptState; private final LuaScriptState closeCircuitLuaScriptState; @@ -99,7 +100,7 @@ public Future getQueueCircuitState(String circuitHash) { public Future getQueueCircuitInformation(String circuitHash) { Promise promise = Promise.promise(); redisProvider.redis().onSuccess(redisAPI -> redisAPI.hmget(Arrays.asList(buildInfosKey(circuitHash), FIELD_STATE, - FIELD_FAILRATIO, FIELD_CIRCUIT), event -> { + FIELD_FAILRATIO, FIELD_CIRCUIT, FIELD_METRICNAME), event -> { if (event.failed()) { promise.fail(event.cause()); } else { @@ -108,6 +109,7 @@ public Future getQueueCircuitInformation(String circuitHash) { QueueCircuitState.CLOSED); String failRatioStr = Objects.toString(event.result().get(1), null); String circuit = Objects.toString(event.result().get(2), null); + String metric = Objects.toString(event.result().get(3), null); JsonObject result = new JsonObject(); result.put("status", state.name().toLowerCase()); JsonObject info = new JsonObject(); @@ -117,6 +119,9 @@ public Future getQueueCircuitInformation(String circuitHash) { if (circuit != null) { info.put(FIELD_CIRCUIT, circuit); } + if (StringUtils.isNotEmptyTrimmed(metric)) { + info.put(FIELD_METRICNAME, metric); + } result.put("info", info); promise.complete(result); } catch (Exception e) { @@ -156,6 +161,7 @@ public Future updateStatistics(PatternAndCircuitHash pat List arguments = Arrays.asList( uniqueRequestID, patternAndCircuitHash.getPattern().pattern(), + patternAndCircuitHash.getMetricName() != null ? patternAndCircuitHash.getMetricName() : "", patternAndCircuitHash.getCircuitHash(), String.valueOf(timestamp), String.valueOf(errorThresholdPercentage), diff --git a/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/circuitbreaker/util/PatternAndCircuitHash.java b/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/circuitbreaker/util/PatternAndCircuitHash.java index 321d3ef74..12485fdaf 100644 --- a/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/circuitbreaker/util/PatternAndCircuitHash.java +++ b/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/circuitbreaker/util/PatternAndCircuitHash.java @@ -1,5 +1,7 @@ package org.swisspush.gateleen.queue.queuing.circuitbreaker.util; +import javax.annotation.Nullable; +import java.util.Objects; import java.util.regex.Pattern; /** @@ -11,10 +13,12 @@ public class PatternAndCircuitHash { private final Pattern pattern; private final String circuitHash; + private final String metricName; - public PatternAndCircuitHash(Pattern pattern, String circuitHash) { + public PatternAndCircuitHash(@Nullable Pattern pattern, String circuitHash, @Nullable String metricName) { this.pattern = pattern; this.circuitHash = circuitHash; + this.metricName = metricName; } public Pattern getPattern() { @@ -25,21 +29,27 @@ public String getCircuitHash() { return circuitHash; } + public String getMetricName() { + return metricName; + } + @Override public String toString() { - return "url pattern: " + pattern.pattern() + " circuit hash: " + circuitHash; + return "PatternAndCircuitHash{" + + "pattern=" + pattern + + ", circuitHash='" + circuitHash + '\'' + + ", metricName='" + metricName + '\'' + + '}'; } @Override public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; - PatternAndCircuitHash that = (PatternAndCircuitHash) o; - - if (!pattern.pattern().equals(that.pattern.pattern())) return false; - return circuitHash.equals(that.circuitHash); - + return Objects.equals(pattern != null ? pattern.pattern() : null, + that.pattern != null ? that.pattern.pattern() : null) && + circuitHash.equals(that.circuitHash); } @Override diff --git a/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/circuitbreaker/util/QueueCircuitBreakerRulePatternToCircuitMapping.java b/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/circuitbreaker/util/QueueCircuitBreakerRulePatternToCircuitMapping.java index 9ac19f372..6f98754f3 100644 --- a/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/circuitbreaker/util/QueueCircuitBreakerRulePatternToCircuitMapping.java +++ b/gateleen-queue/src/main/java/org/swisspush/gateleen/queue/queuing/circuitbreaker/util/QueueCircuitBreakerRulePatternToCircuitMapping.java @@ -66,7 +66,7 @@ private PatternAndCircuitHash getPatternAndCircuitHashFromRule(Rule rule){ try { Pattern pattern = Pattern.compile(rule.getUrlPattern()); String circuitHash = HashCodeGenerator.createHashCode(rule.getUrlPattern()); - return new PatternAndCircuitHash(pattern, circuitHash); + return new PatternAndCircuitHash(pattern, circuitHash, rule.getMetricName()); } catch (Exception e) { log.error("Could not compile the regex:{} to a pattern.", rule.getUrlPattern()); return null; diff --git a/gateleen-queue/src/main/resources/circuitbreaker_getAllCircuits.lua b/gateleen-queue/src/main/resources/circuitbreaker_getAllCircuits.lua index d1908c1d2..69c73d520 100644 --- a/gateleen-queue/src/main/resources/circuitbreaker_getAllCircuits.lua +++ b/gateleen-queue/src/main/resources/circuitbreaker_getAllCircuits.lua @@ -1,6 +1,7 @@ local stateField = "state" local failRatioField = "failRatio" local circuitField = "circuit" +local metricField = "metric" local allCircuitsKey = KEYS[1] @@ -8,7 +9,7 @@ local circuitInfoKeyPrefix = ARGV[1] local circuitInfoKeySuffix = ARGV[2] local function getCircuitInfos(circuit) - return redis.call('hmget',circuitInfoKeyPrefix..circuit..circuitInfoKeySuffix,stateField,circuitField,failRatioField) + return redis.call('hmget',circuitInfoKeyPrefix..circuit..circuitInfoKeySuffix,stateField,circuitField,metricField,failRatioField) end local function string_not_empty(s) @@ -29,7 +30,10 @@ for k, circuit in ipairs(allCircuits) do result[circuit].infos.circuit = fields[2] end if string_not_empty(fields[3]) then - result[circuit].infos.failRatio = tonumber(fields[3]) + result[circuit].infos.metric = fields[3] + end + if string_not_empty(fields[4]) then + result[circuit].infos.failRatio = tonumber(fields[4]) end end diff --git a/gateleen-queue/src/main/resources/circuitbreaker_update.lua b/gateleen-queue/src/main/resources/circuitbreaker_update.lua index 7d8314779..303ec2a61 100644 --- a/gateleen-queue/src/main/resources/circuitbreaker_update.lua +++ b/gateleen-queue/src/main/resources/circuitbreaker_update.lua @@ -1,6 +1,7 @@ local stateField = "state" local failRatioField = "failRatio" local circuitField = "circuit" +local metricField = "metric" local circuitInfoKey = KEYS[1] local circuitSuccessKey = KEYS[2] local circuitFailureKey = KEYS[3] @@ -10,12 +11,13 @@ local allCircuitsKey = KEYS[6] local requestID = ARGV[1] local circuit = ARGV[2] -local circuitHash = ARGV[3] -local requestTS = tonumber(ARGV[4]) -local errorThresholdPercentage = tonumber(ARGV[5]) -local entriesMaxAgeMS = tonumber(ARGV[6]) -local minQueueSampleCount = tonumber(ARGV[7]) -local maxQueueSampleCount = tonumber(ARGV[8]) +local metric = ARGV[3] +local circuitHash = ARGV[4] +local requestTS = tonumber(ARGV[5]) +local errorThresholdPercentage = tonumber(ARGV[6]) +local entriesMaxAgeMS = tonumber(ARGV[7]) +local minQueueSampleCount = tonumber(ARGV[8]) +local maxQueueSampleCount = tonumber(ARGV[9]) local return_value = "OK" local minScore = requestTS - entriesMaxAgeMS @@ -24,6 +26,8 @@ local minScore = requestTS - entriesMaxAgeMS redis.call('zadd',circuitKeyToUpdate,requestTS,requestID) -- write circuit pattern to infos redis.call('hsetnx',circuitInfoKey, circuitField, circuit) +-- write metric to infos +redis.call('hsetnx',circuitInfoKey, metricField, metric) -- add circuit to all circuits set redis.call('sadd',allCircuitsKey,circuitHash) diff --git a/gateleen-queue/src/test/java/org/swisspush/gateleen/queue/queuing/circuitbreaker/api/QueueCircuitBreakerHttpRequestHandlerTest.java b/gateleen-queue/src/test/java/org/swisspush/gateleen/queue/queuing/circuitbreaker/api/QueueCircuitBreakerHttpRequestHandlerTest.java index a33336bb1..32f6d6144 100644 --- a/gateleen-queue/src/test/java/org/swisspush/gateleen/queue/queuing/circuitbreaker/api/QueueCircuitBreakerHttpRequestHandlerTest.java +++ b/gateleen-queue/src/test/java/org/swisspush/gateleen/queue/queuing/circuitbreaker/api/QueueCircuitBreakerHttpRequestHandlerTest.java @@ -84,6 +84,7 @@ public void testGetQueueCircuitInformationSuccess(TestContext context) { JsonObject info = new JsonObject(); info.put("failRatio", 99); info.put("circuit", "/path/of/circuit"); + info.put("metric", "my-metric-1"); result.put("info", info); Mockito.when(queueCircuitBreakerStorage.getQueueCircuitInformation(anyString())) @@ -97,6 +98,34 @@ public void testGetQueueCircuitInformationSuccess(TestContext context) { context.assertEquals(QueueCircuitState.HALF_OPEN.name(), payload.getString(STATUS)); context.assertEquals(99, payload.getJsonObject("info").getInteger("failRatio")); context.assertEquals("/path/of/circuit", payload.getJsonObject("info").getString("circuit")); + context.assertEquals("my-metric-1", payload.getJsonObject("info").getString("metric")); + async.complete(); + }); + } + + @Test + public void testGetQueueCircuitInformationSuccessWithoutMetricName(TestContext context) { + Async async = context.async(); + + JsonObject result = new JsonObject(); + result.put("status", QueueCircuitState.HALF_OPEN.name()); + JsonObject info = new JsonObject(); + info.put("failRatio", 99); + info.put("circuit", "/path/of/circuit"); + result.put("info", info); + + Mockito.when(queueCircuitBreakerStorage.getQueueCircuitInformation(anyString())) + .thenReturn(Future.succeededFuture(result)); + + vertx.eventBus().request(HTTP_REQUEST_API_ADDRESS, QueueCircuitBreakerAPI.buildGetCircuitInformationOperation("someCircuit"), + (Handler>>) reply -> { + JsonObject replyBody = reply.result().body(); + context.assertEquals(OK, replyBody.getString(STATUS)); + JsonObject payload = replyBody.getJsonObject(VALUE); + context.assertEquals(QueueCircuitState.HALF_OPEN.name(), payload.getString(STATUS)); + context.assertEquals(99, payload.getJsonObject("info").getInteger("failRatio")); + context.assertEquals("/path/of/circuit", payload.getJsonObject("info").getString("circuit")); + context.assertNull(payload.getJsonObject("info").getString("metric")); async.complete(); }); } diff --git a/gateleen-queue/src/test/java/org/swisspush/gateleen/queue/queuing/circuitbreaker/impl/QueueCircuitBreakerImplTest.java b/gateleen-queue/src/test/java/org/swisspush/gateleen/queue/queuing/circuitbreaker/impl/QueueCircuitBreakerImplTest.java index d6dcb9916..6cb839e99 100644 --- a/gateleen-queue/src/test/java/org/swisspush/gateleen/queue/queuing/circuitbreaker/impl/QueueCircuitBreakerImplTest.java +++ b/gateleen-queue/src/test/java/org/swisspush/gateleen/queue/queuing/circuitbreaker/impl/QueueCircuitBreakerImplTest.java @@ -176,7 +176,7 @@ public void testHandleQueuedRequest(TestContext context) { HttpRequest req = new HttpRequest(HttpMethod.PUT, "/playground/circuitBreaker/test", MultiMap.caseInsensitiveMultiMap(), null); Mockito.when(ruleToCircuitMapping.getCircuitFromRequestUri(anyString())) - .thenReturn(new PatternAndCircuitHash(Pattern.compile("/someCircuit"), "someCircuitHash")); + .thenReturn(new PatternAndCircuitHash(Pattern.compile("/someCircuit"), "someCircuitHash", "my-metric-1")); Mockito.when(queueCircuitBreakerStorage.getQueueCircuitState(any(PatternAndCircuitHash.class))) .thenReturn(Future.succeededFuture(QueueCircuitState.CLOSED)); @@ -206,7 +206,7 @@ public void testHandleQueuedRequestCallsLockQueueWhenCircuitIsOpen(TestContext c HttpRequest req = new HttpRequest(HttpMethod.PUT, "/playground/circuitBreaker/test", MultiMap.caseInsensitiveMultiMap(), null); Mockito.when(ruleToCircuitMapping.getCircuitFromRequestUri(anyString())) - .thenReturn(new PatternAndCircuitHash(Pattern.compile("/someCircuit"), "someCircuitHash")); + .thenReturn(new PatternAndCircuitHash(Pattern.compile("/someCircuit"), "someCircuitHash", "my-metric-1")); Mockito.when(queueCircuitBreakerStorage.getQueueCircuitState(any(PatternAndCircuitHash.class))) .thenReturn(Future.succeededFuture(QueueCircuitState.OPEN)); @@ -229,7 +229,7 @@ public void testHandleQueuedRequestDoesNotCallLockQueueWhenCircuitIsNotOpen(Test HttpRequest req = new HttpRequest(HttpMethod.PUT, "/playground/circuitBreaker/test", MultiMap.caseInsensitiveMultiMap(), null); Mockito.when(ruleToCircuitMapping.getCircuitFromRequestUri(anyString())) - .thenReturn(new PatternAndCircuitHash(Pattern.compile("/someCircuit"), "someCircuitHash")); + .thenReturn(new PatternAndCircuitHash(Pattern.compile("/someCircuit"), "someCircuitHash", "my-metric-1")); Mockito.when(queueCircuitBreakerStorage.getQueueCircuitState(any(PatternAndCircuitHash.class))) .thenReturn(Future.succeededFuture(QueueCircuitState.CLOSED)); @@ -274,7 +274,7 @@ public void testUpdateStatistics(TestContext context) { HttpRequest req = new HttpRequest(HttpMethod.PUT, "/playground/circuitBreaker/test", MultiMap.caseInsensitiveMultiMap(), null); Mockito.when(ruleToCircuitMapping.getCircuitFromRequestUri(anyString())) - .thenReturn(new PatternAndCircuitHash(Pattern.compile("/someCircuit"), "someCircuitHash")); + .thenReturn(new PatternAndCircuitHash(Pattern.compile("/someCircuit"), "someCircuitHash", "my-metric-1")); Mockito.when(queueCircuitBreakerStorage.updateStatistics(any(PatternAndCircuitHash.class), anyString(), anyLong(), anyInt(), anyLong(), anyLong(), anyLong(), any(QueueResponseType.class))) @@ -311,7 +311,7 @@ public void testUpdateStatisticsTriggersQueueLock(TestContext context) { HttpRequest req = new HttpRequest(HttpMethod.PUT, "/playground/circuitBreaker/test", MultiMap.caseInsensitiveMultiMap(), null); Mockito.when(ruleToCircuitMapping.getCircuitFromRequestUri(anyString())) - .thenReturn(new PatternAndCircuitHash(Pattern.compile("/someCircuit"), "someCircuitHash")); + .thenReturn(new PatternAndCircuitHash(Pattern.compile("/someCircuit"), "someCircuitHash", "my-metric-1")); Mockito.when(queueCircuitBreakerStorage.updateStatistics(any(PatternAndCircuitHash.class), anyString(), anyLong(), anyInt(), anyLong(), anyLong(), anyLong(), any(QueueResponseType.class))) @@ -334,7 +334,7 @@ public void testQueueLock(TestContext context) { HttpRequest req = new HttpRequest(HttpMethod.PUT, "/playground/circuitBreaker/test", MultiMap.caseInsensitiveMultiMap(), null); Mockito.when(ruleToCircuitMapping.getCircuitFromRequestUri(anyString())) - .thenReturn(new PatternAndCircuitHash(Pattern.compile("/someCircuit"), "someCircuitHash")); + .thenReturn(new PatternAndCircuitHash(Pattern.compile("/someCircuit"), "someCircuitHash", "my-metric-1")); Mockito.when(queueCircuitBreakerStorage.lockQueue(anyString(), any(PatternAndCircuitHash.class))) .thenReturn(Future.succeededFuture()); @@ -360,7 +360,7 @@ public void testQueueLockFailingRedisques(TestContext context) { HttpRequest req = new HttpRequest(HttpMethod.PUT, "/playground/circuitBreaker/test", MultiMap.caseInsensitiveMultiMap(), null); Mockito.when(ruleToCircuitMapping.getCircuitFromRequestUri(anyString())) - .thenReturn(new PatternAndCircuitHash(Pattern.compile("/someCircuit"), "someCircuitHash")); + .thenReturn(new PatternAndCircuitHash(Pattern.compile("/someCircuit"), "someCircuitHash", "my-metric-1")); Mockito.when(queueCircuitBreakerStorage.lockQueue(anyString(), any(PatternAndCircuitHash.class))) .thenReturn(Future.succeededFuture()); @@ -387,7 +387,7 @@ public void testQueueLockFailingStorage(TestContext context) { HttpRequest req = new HttpRequest(HttpMethod.PUT, "/playground/circuitBreaker/test", MultiMap.caseInsensitiveMultiMap(), null); Mockito.when(ruleToCircuitMapping.getCircuitFromRequestUri(anyString())) - .thenReturn(new PatternAndCircuitHash(Pattern.compile("/someCircuit"), "someCircuitHash")); + .thenReturn(new PatternAndCircuitHash(Pattern.compile("/someCircuit"), "someCircuitHash", "my-metric-1")); Mockito.when(queueCircuitBreakerStorage.lockQueue(anyString(), any(PatternAndCircuitHash.class))) .thenReturn(Future.failedFuture("queue could not be locked")); @@ -544,7 +544,7 @@ public void testCloseCircuit(TestContext context) { HttpRequest req = new HttpRequest(HttpMethod.PUT, "/playground/circuitBreaker/test", MultiMap.caseInsensitiveMultiMap(), null); Mockito.when(ruleToCircuitMapping.getCircuitFromRequestUri(anyString())) - .thenReturn(new PatternAndCircuitHash(Pattern.compile("/someCircuit"), "someCircuitHash")); + .thenReturn(new PatternAndCircuitHash(Pattern.compile("/someCircuit"), "someCircuitHash", "my-metric-1")); Mockito.when(queueCircuitBreakerStorage.closeCircuit(any(PatternAndCircuitHash.class))) .thenReturn(Future.succeededFuture()); @@ -563,7 +563,7 @@ public void testCloseCircuitFailingStorage(TestContext context) { HttpRequest req = new HttpRequest(HttpMethod.PUT, "/playground/circuitBreaker/test", MultiMap.caseInsensitiveMultiMap(), null); Mockito.when(ruleToCircuitMapping.getCircuitFromRequestUri(anyString())) - .thenReturn(new PatternAndCircuitHash(Pattern.compile("/someCircuit"), "someCircuitHash")); + .thenReturn(new PatternAndCircuitHash(Pattern.compile("/someCircuit"), "someCircuitHash", "my-metric-1")); Mockito.when(queueCircuitBreakerStorage.closeCircuit(any(PatternAndCircuitHash.class))) .thenReturn(Future.failedFuture("unable to close circuit")); @@ -614,7 +614,7 @@ public void testReOpenCircuit(TestContext context) { HttpRequest req = new HttpRequest(HttpMethod.PUT, "/playground/circuitBreaker/test", MultiMap.caseInsensitiveMultiMap(), null); Mockito.when(ruleToCircuitMapping.getCircuitFromRequestUri(anyString())) - .thenReturn(new PatternAndCircuitHash(Pattern.compile("/someCircuit"), "someCircuitHash")); + .thenReturn(new PatternAndCircuitHash(Pattern.compile("/someCircuit"), "someCircuitHash", "my-metric-1")); Mockito.when(queueCircuitBreakerStorage.reOpenCircuit(any(PatternAndCircuitHash.class))) .thenReturn(Future.succeededFuture()); @@ -633,7 +633,7 @@ public void testReOpenCircuitFailingStorage(TestContext context) { HttpRequest req = new HttpRequest(HttpMethod.PUT, "/playground/circuitBreaker/test", MultiMap.caseInsensitiveMultiMap(), null); Mockito.when(ruleToCircuitMapping.getCircuitFromRequestUri(anyString())) - .thenReturn(new PatternAndCircuitHash(Pattern.compile("/someCircuit"), "someCircuitHash")); + .thenReturn(new PatternAndCircuitHash(Pattern.compile("/someCircuit"), "someCircuitHash", "my-metric-1")); Mockito.when(queueCircuitBreakerStorage.reOpenCircuit(any(PatternAndCircuitHash.class))) .thenReturn(Future.failedFuture("unable to re-open circuit")); diff --git a/gateleen-queue/src/test/java/org/swisspush/gateleen/queue/queuing/circuitbreaker/impl/RedisQueueCircuitBreakerStorageTest.java b/gateleen-queue/src/test/java/org/swisspush/gateleen/queue/queuing/circuitbreaker/impl/RedisQueueCircuitBreakerStorageTest.java index 22145d719..61efd30e2 100644 --- a/gateleen-queue/src/test/java/org/swisspush/gateleen/queue/queuing/circuitbreaker/impl/RedisQueueCircuitBreakerStorageTest.java +++ b/gateleen-queue/src/test/java/org/swisspush/gateleen/queue/queuing/circuitbreaker/impl/RedisQueueCircuitBreakerStorageTest.java @@ -18,7 +18,6 @@ import org.junit.BeforeClass; import org.junit.Test; import org.junit.runner.RunWith; -import org.swisspush.gateleen.core.exception.GateleenExceptionFactory; import org.swisspush.gateleen.queue.queuing.circuitbreaker.util.PatternAndCircuitHash; import org.swisspush.gateleen.queue.queuing.circuitbreaker.util.QueueCircuitState; import org.swisspush.gateleen.queue.queuing.circuitbreaker.util.QueueResponseType; @@ -68,7 +67,7 @@ public void setUp(){ @Test public void testGetQueueCircuitState(TestContext context){ Async async = context.async(); - PatternAndCircuitHash patternAndCircuitHash = buildPatternAndCircuitHash("/someCircuit", "someCircuitHash"); + PatternAndCircuitHash patternAndCircuitHash = buildPatternAndCircuitHash("/someCircuit", "someCircuitHash", "my-metric-1"); storage.getQueueCircuitState(patternAndCircuitHash).onComplete(event -> { context.assertEquals(CLOSED, event.result()); writeQueueCircuitStateToDatabase("someCircuitHash", HALF_OPEN); @@ -103,8 +102,9 @@ public void testGetQueueCircuitInformation(TestContext context){ context.assertTrue(result.containsKey("info")); context.assertFalse(result.getJsonObject("info").containsKey("failRatio")); context.assertFalse(result.getJsonObject("info").containsKey("circuit")); + context.assertFalse(result.getJsonObject("info").containsKey("metric")); - writeQueueCircuit(hash, HALF_OPEN, "/some/circuit/path", 99); + writeQueueCircuit(hash, HALF_OPEN, "/some/circuit/path", "my-metric-1", 99); storage.getQueueCircuitInformation(hash).onComplete(event1 -> { context.assertTrue(event1.succeeded()); @@ -113,6 +113,7 @@ public void testGetQueueCircuitInformation(TestContext context){ context.assertTrue(result1.containsKey("info")); context.assertEquals(99, result1.getJsonObject("info").getInteger("failRatio")); context.assertEquals("/some/circuit/path", result1.getJsonObject("info").getString("circuit")); + context.assertEquals("my-metric-1", result1.getJsonObject("info").getString("metric")); async.complete(); }); }); @@ -131,9 +132,9 @@ public void testGetAllCircuits(TestContext context){ context.assertFalse(jedis.exists(infosKey(hash3))); // prepare - writeQueueCircuit(hash1, HALF_OPEN, "/path/to/hash_1", 60); - writeQueueCircuit(hash2, CLOSED, "/path/to/hash_2", 20); - writeQueueCircuit(hash3, OPEN, "/path/to/hash_3", 99); + writeQueueCircuit(hash1, HALF_OPEN, "/path/to/hash_1", "metric-1", 60); + writeQueueCircuit(hash2, CLOSED, "/path/to/hash_2","metric-2", 20); + writeQueueCircuit(hash3, OPEN, "/path/to/hash_3","metric-3", 99); context.assertTrue(jedis.exists(infosKey(hash1))); context.assertTrue(jedis.exists(infosKey(hash2))); @@ -143,22 +144,25 @@ public void testGetAllCircuits(TestContext context){ context.assertEquals(HALF_OPEN.name().toLowerCase(), jedis.hget(infosKey(hash1), FIELD_STATE).toLowerCase()); context.assertEquals("/path/to/hash_1", jedis.hget(infosKey(hash1), FIELD_CIRCUIT)); + context.assertEquals("metric-1", jedis.hget(infosKey(hash1), FIELD_METRICNAME)); context.assertEquals("60", jedis.hget(infosKey(hash1), FIELD_FAILRATIO)); context.assertEquals(CLOSED.name().toLowerCase(), jedis.hget(infosKey(hash2), FIELD_STATE).toLowerCase()); context.assertEquals("/path/to/hash_2", jedis.hget(infosKey(hash2), FIELD_CIRCUIT)); + context.assertEquals("metric-2", jedis.hget(infosKey(hash2), FIELD_METRICNAME)); context.assertEquals("20", jedis.hget(infosKey(hash2), FIELD_FAILRATIO)); context.assertEquals(OPEN.name().toLowerCase(), jedis.hget(infosKey(hash3), FIELD_STATE).toLowerCase()); context.assertEquals("/path/to/hash_3", jedis.hget(infosKey(hash3), FIELD_CIRCUIT)); + context.assertEquals("metric-3", jedis.hget(infosKey(hash3), FIELD_METRICNAME)); context.assertEquals("99", jedis.hget(infosKey(hash3), FIELD_FAILRATIO)); storage.getAllCircuits().onComplete(event -> { context.assertTrue(event.succeeded()); JsonObject result = event.result(); - assertJsonObjectContents(context, result, hash1, HALF_OPEN, "/path/to/hash_1", 60); - assertJsonObjectContents(context, result, hash2, CLOSED, "/path/to/hash_2", 20); - assertJsonObjectContents(context, result, hash3, OPEN, "/path/to/hash_3", 99); + assertJsonObjectContents(context, result, hash1, HALF_OPEN, "/path/to/hash_1", "metric-1", 60); + assertJsonObjectContents(context, result, hash2, CLOSED, "/path/to/hash_2","metric-2", 20); + assertJsonObjectContents(context, result, hash3, OPEN, "/path/to/hash_3", "metric-3",99); async.complete(); }); } @@ -181,7 +185,7 @@ public void testUpdateStatistics(TestContext context){ Async async = context.async(); String circuitHash = "anotherCircuitHash"; - PatternAndCircuitHash patternAndCircuitHash = buildPatternAndCircuitHash("/anotherCircuit", circuitHash); + PatternAndCircuitHash patternAndCircuitHash = buildPatternAndCircuitHash("/anotherCircuit", circuitHash, "my-metric-1"); context.assertFalse(jedis.exists(key(circuitHash, QueueResponseType.SUCCESS))); context.assertFalse(jedis.exists(key(circuitHash, QueueResponseType.FAILURE))); @@ -222,7 +226,7 @@ public void testOpenCircuit(TestContext context){ Async async = context.async(); String circuitHash = "anotherCircuitHash"; - PatternAndCircuitHash patternAndCircuitHash = buildPatternAndCircuitHash("/anotherCircuit", circuitHash); + PatternAndCircuitHash patternAndCircuitHash = buildPatternAndCircuitHash("/anotherCircuit", circuitHash, "my-metric-1"); context.assertFalse(jedis.exists(key(circuitHash, QueueResponseType.SUCCESS))); context.assertFalse(jedis.exists(key(circuitHash, QueueResponseType.FAILURE))); @@ -244,11 +248,11 @@ public void testOpenCircuit(TestContext context){ storage.getQueueCircuitState(patternAndCircuitHash).onComplete(event1 -> { context.assertEquals(CLOSED, event1.result()); - assertStateAndErroPercentage(context, circuitHash, CLOSED, 66); + assertStateAndErrorPercentage(context, circuitHash, CLOSED, 66); context.assertFalse(jedis.exists(STORAGE_OPEN_CIRCUITS)); storage.updateStatistics(patternAndCircuitHash, "req_4", 4, errorThreshold, entriesMaxAgeMS, minQueueSampleCount, maxQueueSampleCount, QueueResponseType.FAILURE).onComplete(event2 -> { storage.getQueueCircuitState(patternAndCircuitHash).onComplete(event3 -> { - assertStateAndErroPercentage(context, circuitHash, OPEN, 75); + assertStateAndErrorPercentage(context, circuitHash, OPEN, 75); context.assertEquals(OPEN, event3.result()); context.assertTrue(jedis.exists(STORAGE_OPEN_CIRCUITS)); assertHashInOpenCircuitsSet(context, circuitHash, 1); @@ -268,7 +272,7 @@ public void testLockQueue(TestContext context){ context.assertFalse(jedis.exists(queuesKey(circuitHash))); - PatternAndCircuitHash patternAndCircuitHash = buildPatternAndCircuitHash("/anotherCircuit", circuitHash); + PatternAndCircuitHash patternAndCircuitHash = buildPatternAndCircuitHash("/anotherCircuit", circuitHash, "my-metric-1"); storage.lockQueue("someQueue", patternAndCircuitHash).onComplete(event -> { context.assertTrue(jedis.exists(queuesKey(circuitHash))); context.assertEquals(1L, jedis.zcard(queuesKey(circuitHash))); @@ -352,7 +356,7 @@ public void testCloseCircuit(TestContext context){ context.assertEquals(1L, jedis.scard(STORAGE_OPEN_CIRCUITS)); context.assertEquals(0L, jedis.llen(STORAGE_QUEUES_TO_UNLOCK)); - PatternAndCircuitHash patternAndCircuitHash = buildPatternAndCircuitHash("/anotherCircuit", circuitHash); + PatternAndCircuitHash patternAndCircuitHash = buildPatternAndCircuitHash("/anotherCircuit", circuitHash, "my-metric-2"); storage.closeCircuit(patternAndCircuitHash).onComplete(event -> { context.assertTrue(event.succeeded()); @@ -370,7 +374,7 @@ public void testCloseCircuit(TestContext context){ context.assertEquals("queue_3", jedis.lpop(STORAGE_QUEUES_TO_UNLOCK)); context.assertTrue(jedis.exists(infosKey(circuitHash))); - assertStateAndErroPercentage(context, circuitHash, CLOSED, 0); + assertStateAndErrorPercentage(context, circuitHash, CLOSED, 0); context.assertEquals(3L, jedis.scard(STORAGE_HALFOPEN_CIRCUITS)); context.assertEquals(4L, jedis.scard(STORAGE_ALL_CIRCUITS)); @@ -437,7 +441,7 @@ public void testCloseAndRemoveCircuit(TestContext context){ context.assertEquals(1L, jedis.scard(STORAGE_OPEN_CIRCUITS)); context.assertEquals(0L, jedis.llen(STORAGE_QUEUES_TO_UNLOCK)); - PatternAndCircuitHash patternAndCircuitHash = buildPatternAndCircuitHash("/anotherCircuit", circuitHash); + PatternAndCircuitHash patternAndCircuitHash = buildPatternAndCircuitHash("/anotherCircuit", circuitHash, "my-metric-2"); storage.closeAndRemoveCircuit(patternAndCircuitHash).onComplete(event -> { context.assertTrue(event.succeeded()); @@ -517,7 +521,7 @@ public void testCloseAllCircuits(TestContext context){ for (int index = 1; index <= 5; index++) { String hash = "hash_" + index; context.assertFalse(jedis.exists(queuesKey(hash))); - assertStateAndErroPercentage(context, hash, CLOSED, 0); + assertStateAndErrorPercentage(context, hash, CLOSED, 0); context.assertFalse(jedis.exists(key(hash, QueueResponseType.SUCCESS))); context.assertFalse(jedis.exists(key(hash, QueueResponseType.FAILURE))); context.assertFalse(jedis.exists(queuesKey(hash))); @@ -583,7 +587,7 @@ public void testReOpenCircuit(TestContext context){ context.assertEquals(4L, jedis.scard(STORAGE_HALFOPEN_CIRCUITS)); context.assertEquals(5L, jedis.scard(STORAGE_OPEN_CIRCUITS)); - PatternAndCircuitHash patternAndCircuitHash = buildPatternAndCircuitHash("/anotherCircuit", circuitHash); + PatternAndCircuitHash patternAndCircuitHash = buildPatternAndCircuitHash("/anotherCircuit", circuitHash, "my-metric-2"); storage.reOpenCircuit(patternAndCircuitHash).onComplete(event -> { context.assertTrue(event.succeeded()); @@ -591,7 +595,7 @@ public void testReOpenCircuit(TestContext context){ context.assertTrue(jedis.exists(STORAGE_HALFOPEN_CIRCUITS)); context.assertTrue(jedis.exists(STORAGE_OPEN_CIRCUITS)); - assertStateAndErroPercentage(context, circuitHash, OPEN, 50); + assertStateAndErrorPercentage(context, circuitHash, OPEN, 50); context.assertEquals(3L, jedis.scard(STORAGE_HALFOPEN_CIRCUITS)); Set halfOpenCircuits = jedis.smembers(STORAGE_HALFOPEN_CIRCUITS); @@ -812,8 +816,8 @@ private void buildCircuitEntry(String circuitHash, QueueCircuitState state, List } } - private PatternAndCircuitHash buildPatternAndCircuitHash(String pattern, String circuitHash){ - return new PatternAndCircuitHash(Pattern.compile(pattern), circuitHash); + private PatternAndCircuitHash buildPatternAndCircuitHash(String pattern, String circuitHash, String metricName){ + return new PatternAndCircuitHash(Pattern.compile(pattern), circuitHash, metricName); } private String key(String circuitHash, QueueResponseType queueResponseType){ @@ -828,9 +832,10 @@ private String infosKey(String circuitHash){ return STORAGE_PREFIX + circuitHash + STORAGE_INFOS_SUFFIX; } - private void writeQueueCircuit(String circuitHash, QueueCircuitState state, String circuit, int failPercentage){ + private void writeQueueCircuit(String circuitHash, QueueCircuitState state, String circuit, String metricName, int failPercentage){ writeQueueCircuitField(circuitHash, FIELD_STATE, state.name().toLowerCase()); writeQueueCircuitField(circuitHash, FIELD_CIRCUIT, circuit); + writeQueueCircuitField(circuitHash, FIELD_METRICNAME, metricName); writeQueueCircuitField(circuitHash, FIELD_FAILRATIO, String.valueOf(failPercentage)); jedis.sadd(STORAGE_ALL_CIRCUITS, circuitHash); } @@ -856,7 +861,7 @@ private void assertState(TestContext context, String circuitHash, QueueCircuitSt context.assertEquals(state.name().toLowerCase(), stateFromDb); } - private void assertStateAndErroPercentage(TestContext context, String circuitHash, QueueCircuitState state, int percentage){ + private void assertStateAndErrorPercentage(TestContext context, String circuitHash, QueueCircuitState state, int percentage){ assertState(context, circuitHash, state); String percentageAsString = jedis.hget(STORAGE_PREFIX + circuitHash + STORAGE_INFOS_SUFFIX, FIELD_FAILRATIO); context.assertEquals(percentage, Integer.valueOf(percentageAsString)); @@ -882,13 +887,14 @@ private void assertQueuesToUnlockItems(TestContext context, List items){ } } - private void assertJsonObjectContents(TestContext context, JsonObject result, String hash, QueueCircuitState status, String circuit, int failRatio){ + private void assertJsonObjectContents(TestContext context, JsonObject result, String hash, QueueCircuitState status, String circuit, String metricName, int failRatio){ context.assertTrue(result.containsKey(hash)); context.assertTrue(result.getJsonObject(hash).containsKey("status")); context.assertEquals(status.name().toLowerCase(), result.getJsonObject(hash).getString("status")); context.assertTrue(result.getJsonObject(hash).containsKey("infos")); context.assertTrue(result.getJsonObject(hash).getJsonObject("infos").containsKey("circuit")); context.assertEquals(circuit, result.getJsonObject(hash).getJsonObject("infos").getString("circuit")); + context.assertEquals(metricName, result.getJsonObject(hash).getJsonObject("infos").getString("metric")); context.assertTrue(result.getJsonObject(hash).getJsonObject("infos").containsKey("failRatio")); context.assertEquals(failRatio, result.getJsonObject(hash).getJsonObject("infos").getInteger("failRatio")); } diff --git a/gateleen-queue/src/test/java/org/swisspush/gateleen/queue/queuing/circuitbreaker/lua/QueueCircuitBreakerUpdateStatsLuaScriptTests.java b/gateleen-queue/src/test/java/org/swisspush/gateleen/queue/queuing/circuitbreaker/lua/QueueCircuitBreakerUpdateStatsLuaScriptTests.java index 291722e68..0216c2775 100644 --- a/gateleen-queue/src/test/java/org/swisspush/gateleen/queue/queuing/circuitbreaker/lua/QueueCircuitBreakerUpdateStatsLuaScriptTests.java +++ b/gateleen-queue/src/test/java/org/swisspush/gateleen/queue/queuing/circuitbreaker/lua/QueueCircuitBreakerUpdateStatsLuaScriptTests.java @@ -46,9 +46,9 @@ public void testCalculateErrorPercentage(){ assertThat(jedis.exists(allCircuitsKey), is(false)); // adding 3 failing requests - evalScriptUpdateQueueCircuitBreakerStats(update_fail, "req_1", "url_pattern", 0, 50, 10, 4, 10); - evalScriptUpdateQueueCircuitBreakerStats(update_fail, "req_2", "url_pattern", 1, 50, 10, 4, 10); - evalScriptUpdateQueueCircuitBreakerStats(update_fail, "req_3", "url_pattern", 2, 50, 10, 4, 10); + evalScriptUpdateQueueCircuitBreakerStats(update_fail, "req_1", "url_pattern", "metric-1", 0, 50, 10, 4, 10); + evalScriptUpdateQueueCircuitBreakerStats(update_fail, "req_2", "url_pattern", "metric-1",1, 50, 10, 4, 10); + evalScriptUpdateQueueCircuitBreakerStats(update_fail, "req_3", "url_pattern","metric-1", 2, 50, 10, 4, 10); // asserts assertThat(jedis.exists(circuitInfoKey), is(true)); @@ -256,6 +256,13 @@ private void assertSizeSizeNotExceedingLimit(String setKey, long maxSetSize){ private Object evalScriptUpdateQueueCircuitBreakerStats(String circuitKeyToUpdate, String uniqueRequestID, String circuit, long timestamp, int errorThresholdPercentage, long entriesMaxAgeMS, long minQueueSampleCount, long maxQueueSampleCount) { + return evalScriptUpdateQueueCircuitBreakerStats(circuitKeyToUpdate, uniqueRequestID, circuit, null, timestamp, errorThresholdPercentage, + entriesMaxAgeMS, minQueueSampleCount, maxQueueSampleCount); + } + + private Object evalScriptUpdateQueueCircuitBreakerStats(String circuitKeyToUpdate, String uniqueRequestID, + String circuit, String metricName, long timestamp, int errorThresholdPercentage, + long entriesMaxAgeMS, long minQueueSampleCount, long maxQueueSampleCount) { String script = readScript(QueueCircuitBreakerLuaScripts.UPDATE_CIRCUIT.getFilename()); List keys = Arrays.asList( @@ -270,6 +277,7 @@ private Object evalScriptUpdateQueueCircuitBreakerStats(String circuitKeyToUpdat List arguments = Arrays.asList( uniqueRequestID, circuit, + metricName != null ? metricName : "", circuit+"Hash", String.valueOf(timestamp), String.valueOf(errorThresholdPercentage),