From 116447a325824204425f54f3f9475cacec520251 Mon Sep 17 00:00:00 2001 From: Ankush Jain Date: Thu, 29 Jun 2023 10:42:18 +0530 Subject: [PATCH 1/2] fetch sample msgs from api collection ids --- .../com/akto/store/SampleMessageStore.java | 6 +++-- .../java/com/akto/testing/TestExecutor.java | 23 +++++++++++++++---- .../akto/store/TestSampleMessageStore.java | 4 ++-- 3 files changed, 24 insertions(+), 9 deletions(-) diff --git a/apps/testing/src/main/java/com/akto/store/SampleMessageStore.java b/apps/testing/src/main/java/com/akto/store/SampleMessageStore.java index 30c6263dee..fe46f029a6 100644 --- a/apps/testing/src/main/java/com/akto/store/SampleMessageStore.java +++ b/apps/testing/src/main/java/com/akto/store/SampleMessageStore.java @@ -18,6 +18,7 @@ import com.akto.testing.NucleiExecutor; import com.mongodb.BasicDBObject; import com.mongodb.client.model.Filters; +import org.bson.conversions.Bson; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -76,8 +77,9 @@ public static List fetchTestRoles() { } - public static Map> fetchSampleMessages() { - List sampleDataList = SampleDataDao.instance.findAll(new BasicDBObject(), 0, 10_000, null); + public static Map> fetchSampleMessages(List apiCollectionIds) { + Bson filterQ = Filters.in("_id.apiCollectionId", apiCollectionIds); + List sampleDataList = SampleDataDao.instance.findAll(filterQ, 0, 10_000, null); System.out.println("SampleDataSize " + sampleDataList.size()); Map> tempSampleDataMap = new HashMap<>(); for (SampleData sampleData: sampleDataList) { diff --git a/apps/testing/src/main/java/com/akto/testing/TestExecutor.java b/apps/testing/src/main/java/com/akto/testing/TestExecutor.java index 472f62334d..e9a3deeb76 100644 --- a/apps/testing/src/main/java/com/akto/testing/TestExecutor.java +++ b/apps/testing/src/main/java/com/akto/testing/TestExecutor.java @@ -95,6 +95,15 @@ public void workflowInit (TestingRun testingRun, ObjectId summaryId) { ); } + private List extractApiCollectionIds(List apiInfoKeyList) { + List ret = new ArrayList<>(); + for(ApiInfo.ApiInfoKey apiInfoKey: apiInfoKeyList) { + ret.add(apiInfoKey.getApiCollectionId()); + } + + return ret; + } + public void apiWiseInit(TestingRun testingRun, ObjectId summaryId) { int accountId = Context.accountId.get(); int now = Context.now(); @@ -102,7 +111,15 @@ public void apiWiseInit(TestingRun testingRun, ObjectId summaryId) { TestingEndpoints testingEndpoints = testingRun.getTestingEndpoints(); Map singleTypeInfoMap = SampleMessageStore.buildSingleTypeInfoMap(testingEndpoints); - Map> sampleMessages = SampleMessageStore.fetchSampleMessages(); + + List apiInfoKeyList = testingEndpoints.returnApis(); + if (apiInfoKeyList == null || apiInfoKeyList.isEmpty()) return; + loggerMaker.infoAndAddToDb("APIs found: " + apiInfoKeyList.size(), LogDb.TESTING); + + + List apiCollectionIds = extractApiCollectionIds(apiInfoKeyList); + + Map> sampleMessages = SampleMessageStore.fetchSampleMessages(apiCollectionIds); List testRoles = SampleMessageStore.fetchTestRoles(); AuthMechanism authMechanism = AuthMechanismsDao.instance.findOne(new BasicDBObject()); @@ -125,10 +142,6 @@ public void apiWiseInit(TestingRun testingRun, ObjectId summaryId) { return; } - List apiInfoKeyList = testingEndpoints.returnApis(); - if (apiInfoKeyList == null || apiInfoKeyList.isEmpty()) return; - loggerMaker.infoAndAddToDb("APIs found: " + apiInfoKeyList.size(), LogDb.TESTING); - Map> sampleDataMapForStatusCodeAnalyser = new HashMap<>(); Set apiInfoKeySet = new HashSet<>(apiInfoKeyList); for (ApiInfo.ApiInfoKey apiInfoKey: sampleMessages.keySet()) { diff --git a/apps/testing/src/test/java/com/akto/store/TestSampleMessageStore.java b/apps/testing/src/test/java/com/akto/store/TestSampleMessageStore.java index 5fd0d267c8..4b66a9dd9e 100644 --- a/apps/testing/src/test/java/com/akto/store/TestSampleMessageStore.java +++ b/apps/testing/src/test/java/com/akto/store/TestSampleMessageStore.java @@ -25,7 +25,7 @@ public void testFetchSampleMessages() { SampleData sampleData3 = new SampleData(new Key(0, "url3", URLMethods.Method.GET,0,0,0), Collections.emptyList()); SampleDataDao.instance.insertMany(Arrays.asList(sampleData1, sampleData2, sampleData3)); - Map> sampleDataMap = SampleMessageStore.fetchSampleMessages(); + Map> sampleDataMap = SampleMessageStore.fetchSampleMessages(Collections.singletonList(0)); assertEquals(sampleDataMap.size(), 2); List messages = sampleDataMap.get(new ApiInfo.ApiInfoKey(0, "url2", URLMethods.Method.GET)); @@ -35,7 +35,7 @@ public void testFetchSampleMessages() { sampleData2 = new SampleData(new Key(0, "url2", URLMethods.Method.GET,0,0,0), Arrays.asList("m1", "m2", "m3")); SampleDataDao.instance.insertMany(Arrays.asList(sampleData1, sampleData2)); - sampleDataMap = SampleMessageStore.fetchSampleMessages(); + sampleDataMap = SampleMessageStore.fetchSampleMessages(Collections.singletonList(0)); assertEquals(sampleDataMap.size(), 1); messages = sampleDataMap.get(new ApiInfo.ApiInfoKey(0, "url2", URLMethods.Method.GET)); assertEquals(messages.size(), 3); From 30c4ca0caf86b7ba11b9628fd59ee12209f58713 Mon Sep 17 00:00:00 2001 From: Avneesh Hota Date: Thu, 29 Jun 2023 16:16:25 +0530 Subject: [PATCH 2/2] used set instead of list and added test cases for fetch sample data --- .../java/com/akto/store/SampleMessageStore.java | 2 +- .../main/java/com/akto/testing/TestExecutor.java | 6 +++--- .../com/akto/store/TestSampleMessageStore.java | 16 ++++++++++++---- 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/apps/testing/src/main/java/com/akto/store/SampleMessageStore.java b/apps/testing/src/main/java/com/akto/store/SampleMessageStore.java index fe46f029a6..859845a319 100644 --- a/apps/testing/src/main/java/com/akto/store/SampleMessageStore.java +++ b/apps/testing/src/main/java/com/akto/store/SampleMessageStore.java @@ -77,7 +77,7 @@ public static List fetchTestRoles() { } - public static Map> fetchSampleMessages(List apiCollectionIds) { + public static Map> fetchSampleMessages(Set apiCollectionIds) { Bson filterQ = Filters.in("_id.apiCollectionId", apiCollectionIds); List sampleDataList = SampleDataDao.instance.findAll(filterQ, 0, 10_000, null); System.out.println("SampleDataSize " + sampleDataList.size()); diff --git a/apps/testing/src/main/java/com/akto/testing/TestExecutor.java b/apps/testing/src/main/java/com/akto/testing/TestExecutor.java index e9a3deeb76..d36d3dac8f 100644 --- a/apps/testing/src/main/java/com/akto/testing/TestExecutor.java +++ b/apps/testing/src/main/java/com/akto/testing/TestExecutor.java @@ -95,8 +95,8 @@ public void workflowInit (TestingRun testingRun, ObjectId summaryId) { ); } - private List extractApiCollectionIds(List apiInfoKeyList) { - List ret = new ArrayList<>(); + private Set extractApiCollectionIds(List apiInfoKeyList) { + Set ret = new HashSet<>(); for(ApiInfo.ApiInfoKey apiInfoKey: apiInfoKeyList) { ret.add(apiInfoKey.getApiCollectionId()); } @@ -117,7 +117,7 @@ public void apiWiseInit(TestingRun testingRun, ObjectId summaryId) { loggerMaker.infoAndAddToDb("APIs found: " + apiInfoKeyList.size(), LogDb.TESTING); - List apiCollectionIds = extractApiCollectionIds(apiInfoKeyList); + Set apiCollectionIds = extractApiCollectionIds(apiInfoKeyList); Map> sampleMessages = SampleMessageStore.fetchSampleMessages(apiCollectionIds); List testRoles = SampleMessageStore.fetchTestRoles(); diff --git a/apps/testing/src/test/java/com/akto/store/TestSampleMessageStore.java b/apps/testing/src/test/java/com/akto/store/TestSampleMessageStore.java index 4b66a9dd9e..6cbfa5768c 100644 --- a/apps/testing/src/test/java/com/akto/store/TestSampleMessageStore.java +++ b/apps/testing/src/test/java/com/akto/store/TestSampleMessageStore.java @@ -23,19 +23,27 @@ public void testFetchSampleMessages() { SampleData sampleData1 = new SampleData(new Key(0, "url1", URLMethods.Method.GET,0,0,0), null); SampleData sampleData2 = new SampleData(new Key(0, "url2", URLMethods.Method.GET,0,0,0), Arrays.asList("m1", "m2")); SampleData sampleData3 = new SampleData(new Key(0, "url3", URLMethods.Method.GET,0,0,0), Collections.emptyList()); - SampleDataDao.instance.insertMany(Arrays.asList(sampleData1, sampleData2, sampleData3)); + SampleData sampleData4 = new SampleData(new Key(1, "url1", URLMethods.Method.GET,0,0,0), Arrays.asList("m3", "m4", "m5")); + SampleDataDao.instance.insertMany(Arrays.asList(sampleData1, sampleData2, sampleData3, sampleData4)); - Map> sampleDataMap = SampleMessageStore.fetchSampleMessages(Collections.singletonList(0)); + Set apiCollectionIds = new HashSet<>(); + apiCollectionIds.add(0); + apiCollectionIds.add(1); - assertEquals(sampleDataMap.size(), 2); + Map> sampleDataMap = SampleMessageStore.fetchSampleMessages(apiCollectionIds); + + assertEquals(sampleDataMap.size(), 3); List messages = sampleDataMap.get(new ApiInfo.ApiInfoKey(0, "url2", URLMethods.Method.GET)); assertEquals(messages.size(), 2); + messages = sampleDataMap.get(new ApiInfo.ApiInfoKey(1, "url1", URLMethods.Method.GET)); + assertEquals(messages.size(), 3); + SampleDataDao.instance.getMCollection().drop(); sampleData2 = new SampleData(new Key(0, "url2", URLMethods.Method.GET,0,0,0), Arrays.asList("m1", "m2", "m3")); SampleDataDao.instance.insertMany(Arrays.asList(sampleData1, sampleData2)); - sampleDataMap = SampleMessageStore.fetchSampleMessages(Collections.singletonList(0)); + sampleDataMap = SampleMessageStore.fetchSampleMessages(apiCollectionIds); assertEquals(sampleDataMap.size(), 1); messages = sampleDataMap.get(new ApiInfo.ApiInfoKey(0, "url2", URLMethods.Method.GET)); assertEquals(messages.size(), 3);