From 1c778f4a2c9ae1d158473f99fea75702a8dd991c Mon Sep 17 00:00:00 2001 From: Ark2307 Date: Wed, 25 Dec 2024 16:21:13 +0530 Subject: [PATCH 1/6] Insertion of data completed --- .../src/main/java/com/akto/utils/Utils.java | 116 +++++++++++++++++- .../com/akto/utils/jobs/CleanInventory.java | 54 +++++++- .../java/com/akto/utils/jobs/JobUtils.java | 9 ++ .../akto/dao/AccountsContextDaoWithRbac.java | 23 +++- .../main/java/com/akto/dto/traffic/Key.java | 18 +++ 5 files changed, 210 insertions(+), 10 deletions(-) diff --git a/apps/dashboard/src/main/java/com/akto/utils/Utils.java b/apps/dashboard/src/main/java/com/akto/utils/Utils.java index fd3de0a99c..cea8ddb9a0 100644 --- a/apps/dashboard/src/main/java/com/akto/utils/Utils.java +++ b/apps/dashboard/src/main/java/com/akto/utils/Utils.java @@ -12,15 +12,20 @@ import com.akto.dao.SensitiveSampleDataDao; import com.akto.dao.SingleTypeInfoDao; import com.akto.dto.AccountSettings; +import com.akto.dto.ApiInfo; import com.akto.dto.HttpResponseParams; import com.akto.dto.OriginalHttpRequest; import com.akto.dto.OriginalHttpResponse; +import com.akto.dto.SensitiveSampleData; +import com.akto.dto.ApiInfo.ApiInfoKey; import com.akto.dto.dependency_flow.DependencyFlow; import com.akto.dto.third_party_access.Credential; import com.akto.dto.third_party_access.PostmanCredential; import com.akto.dto.third_party_access.ThirdPartyAccess; import com.akto.dto.traffic.Key; +import com.akto.dto.traffic.SampleData; import com.akto.dto.type.SingleTypeInfo; +import com.akto.dto.type.URLMethods; import com.akto.dto.upload.FileUploadError; import com.akto.listener.KafkaListener; import com.akto.listener.RuntimeListener; @@ -34,10 +39,17 @@ import com.fasterxml.jackson.databind.node.ArrayNode; import com.google.gson.Gson; import com.mongodb.BasicDBObject; +import com.mongodb.client.model.BulkWriteOptions; import com.mongodb.client.model.Filters; +import com.mongodb.client.model.UpdateManyModel; +import com.mongodb.client.model.Updates; +import com.mongodb.client.model.WriteModel; + import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; +import org.bson.conversions.Bson; +import java.lang.reflect.Method; import java.net.MalformedURLException; import java.net.URISyntaxException; import java.net.URL; @@ -52,6 +64,7 @@ public class Utils { private static final LoggerMaker loggerMaker = new LoggerMaker(Utils.class); private final static ObjectMapper mapper = new ObjectMapper(); + private static String id = "_id."; public static Map getAuthMap(JsonNode auth, Map variableMap) { Map result = new HashMap<>(); @@ -600,8 +613,6 @@ public static float getRiskScoreValueFromSeverityScore(float severityScore){ public static void deleteApis(List toBeDeleted) { - String id = "_id."; - AccountsContextDaoWithRbac.deleteApisPerDao(toBeDeleted, SingleTypeInfoDao.instance, ""); AccountsContextDaoWithRbac.deleteApisPerDao(toBeDeleted, ApiInfoDao.instance, id); AccountsContextDaoWithRbac.deleteApisPerDao(toBeDeleted, SampleDataDao.instance, id); @@ -612,6 +623,107 @@ public static void deleteApis(List toBeDeleted) { } + private static Key copy(Key oldKey) { + Key newKey = new Key(); + newKey.setApiCollectionId(oldKey.getApiCollectionId()); + newKey.setUrl(oldKey.getUrl()); + newKey.setMethod(oldKey.getMethod()); + newKey.setResponseCode(oldKey.getResponseCode()); + newKey.setBucketStartEpoch(oldKey.getBucketStartEpoch()); + newKey.setBucketEndEpoch(oldKey.getBucketEndEpoch()); + return newKey; + } + + public static void moveApisFromSampleData(Map sampleDataToBeMovedMap){ + + Map mapApiInfoKeyToKey = new HashMap<>(); + + // insert new sample data + Bson sampleDataFilter = AccountsContextDaoWithRbac.generateCommonFilter(sampleDataToBeMovedMap, SampleDataDao.instance, id); + List sampleDataList = SampleDataDao.instance.findAll(sampleDataFilter); + Iterator sampleDataIterator = sampleDataList.iterator(); + while(sampleDataIterator.hasNext()){ + SampleData sampleData = sampleDataIterator.next(); + Key key = sampleData.getId(); + Key oldKey = copy(key); + int newCollId = sampleDataToBeMovedMap.getOrDefault(key, key.getApiCollectionId()); + ApiInfoKey apiInfoKey = new ApiInfoKey(key.getApiCollectionId(), key.getUrl(), key.getMethod()); + mapApiInfoKeyToKey.put(apiInfoKey, oldKey); + if(key.getApiCollectionId() != newCollId){ + key.setApiCollectionId(newCollId); + sampleData.setId(key); + }else{ + sampleDataIterator.remove(); + } + } + if(!sampleDataList.isEmpty()){ + SampleDataDao.instance.insertMany(sampleDataList); + loggerMaker.infoAndAddToDb("Inserted " + sampleDataList.size() + " new sample data into database."); + } + + // insert new api info data + Bson apiInfoFilter = AccountsContextDaoWithRbac.generateCommonFilter(sampleDataToBeMovedMap, ApiInfoDao.instance, id); + List apiInfos = ApiInfoDao.instance.findAll(apiInfoFilter); + Iterator apiInfoIterator = apiInfos.iterator(); + while(apiInfoIterator.hasNext()){ + ApiInfo apiInfo = apiInfoIterator.next(); + ApiInfoKey apiInfoKey = apiInfo.getId(); + Key mappedKey = mapApiInfoKeyToKey.get(apiInfoKey); + int newCollId = sampleDataToBeMovedMap.getOrDefault(mappedKey, mappedKey.getApiCollectionId()); + if(mappedKey.getApiCollectionId() != newCollId){ + apiInfoKey.setApiCollectionId(newCollId); + apiInfo.setId(apiInfoKey); + }else{ + apiInfoIterator.remove(); + } + } + + if(!apiInfos.isEmpty()){ + ApiInfoDao.instance.insertMany(apiInfos); + loggerMaker.infoAndAddToDb("Inserted " + apiInfos.size() + " new api infos into database."); + } + + // insert new sensitive sample data + Bson sensitiveDataFilter = AccountsContextDaoWithRbac.generateCommonFilter(sampleDataToBeMovedMap, SensitiveSampleDataDao.instance, id); + List sensitiveSampleDataList = SensitiveSampleDataDao.instance.findAll(sensitiveDataFilter); + Iterator sensitiveSampleDataIterator = sensitiveSampleDataList.iterator(); + while(sensitiveSampleDataIterator.hasNext()){ + SensitiveSampleData sampleData = sensitiveSampleDataIterator.next(); + SingleTypeInfo.ParamId key = sampleData.getId(); + ApiInfoKey apiInfoKey = new ApiInfoKey(key.getApiCollectionId(), key.getUrl(), URLMethods.Method.valueOf(key.getMethod())); + Key mappedKey = mapApiInfoKeyToKey.get(apiInfoKey); + int newCollId = sampleDataToBeMovedMap.getOrDefault(mappedKey, mappedKey.getApiCollectionId()); + + if(key.getApiCollectionId() != newCollId){ + key.setApiCollectionId(newCollId); + sampleData.setId(key); + }else{ + sensitiveSampleDataIterator.remove(); + } + } + if(!sensitiveSampleDataList.isEmpty()){ + SensitiveSampleDataDao.instance.insertMany(sensitiveSampleDataList); + loggerMaker.infoAndAddToDb("Inserted " + sensitiveSampleDataList.size() + " new sensitive sample data into database."); + } + + // update single type info data + ArrayList> bulkUpdatesForSti = new ArrayList<>(); + for(Key key: sampleDataToBeMovedMap.keySet()){ + Bson filterQ = Filters.and( + Filters.eq(SingleTypeInfo._API_COLLECTION_ID, key.getApiCollectionId()), + Filters.eq(SingleTypeInfo._URL, key.getUrl()), + Filters.eq(SingleTypeInfo._METHOD, key.getMethod()) + ); + bulkUpdatesForSti.add(new UpdateManyModel<>(filterQ, Updates.set(SingleTypeInfo._API_COLLECTION_ID, sampleDataToBeMovedMap.get(key)))); + } + + if(!bulkUpdatesForSti.isEmpty()){ + loggerMaker.infoAndAddToDb("Updated " + bulkUpdatesForSti.size() + " stis into database."); + SingleTypeInfoDao.instance.getMCollection().bulkWrite(bulkUpdatesForSti, new BulkWriteOptions().ordered(false)); + } + + } + public static List getUniqueValuesOfList(List input){ if(input == null || input.isEmpty()){ return new ArrayList<>(); diff --git a/apps/dashboard/src/main/java/com/akto/utils/jobs/CleanInventory.java b/apps/dashboard/src/main/java/com/akto/utils/jobs/CleanInventory.java index 231939d22c..98998cfa2d 100644 --- a/apps/dashboard/src/main/java/com/akto/utils/jobs/CleanInventory.java +++ b/apps/dashboard/src/main/java/com/akto/utils/jobs/CleanInventory.java @@ -1,8 +1,5 @@ package com.akto.utils.jobs; -import java.io.BufferedWriter; -import java.io.File; -import java.io.FileWriter; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; @@ -19,12 +16,12 @@ import java.util.regex.Pattern; import java.util.stream.Collectors; -import com.akto.dto.CodeAnalysisRepo; import com.mongodb.client.model.Updates; import org.bson.conversions.Bson; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.akto.dao.ApiCollectionsDao; import com.akto.dao.ApiInfoDao; import com.akto.dao.SampleDataDao; import com.akto.dao.SensitiveSampleDataDao; @@ -52,9 +49,11 @@ import com.akto.util.Pair; import com.mongodb.BasicDBObject; import com.mongodb.client.model.Filters; +import com.mongodb.client.model.FindOneAndUpdateOptions; import com.mongodb.client.model.Sorts; import static com.akto.utils.Utils.deleteApis; +import static com.akto.utils.Utils.moveApisFromSampleData; import static com.akto.runtime.utils.Utils.createRegexPatternFromList; public class CleanInventory { @@ -139,10 +138,20 @@ public static void cleanFilteredSampleDataFromAdvancedFilters(List collectionWiseDeletionCountMap = new HashMap<>(); + Map moveHostCollectionWiseCountMap = new HashMap<>(); + + FindOneAndUpdateOptions updateOptions = new FindOneAndUpdateOptions(); + updateOptions.upsert(true); + + Bson updatesForCollection = Updates.combine( + Updates.setOnInsert("startTs", Context.now()), + Updates.setOnInsert("urls", new HashSet<>()) + ); Map filterMap = FilterYamlTemplateDao.instance.fetchFilterConfig(false, yamlTemplates, true); Pattern pattern = createRegexPatternFromList(redundantUrlList); do { + Map sampleDataToBeMovedIntoCollection = new HashMap<>(); sampleDataList = SampleDataDao.instance.findAll(filters, skip, limit, sort); skip += limit; List toBeDeleted = new ArrayList<>(); @@ -169,6 +178,7 @@ public static void cleanFilteredSampleDataFromAdvancedFilters(List currentHostArr = param.getRequestParams().getHeaders().get("host"); + String hostStringFromParam = ""; + if(currentHostArr != null && !currentHostArr.isEmpty()){ + hostStringFromParam = currentHostArr.get(0); + } + boolean hostNameModified = JobUtils.hasHostModified(hostStringFromParam, apiCollection); + if(hostNameModified){ + int collectionId = hostStringFromParam.hashCode(); + // creating new ApiCollection from this id + Bson currentUpdate = Updates.combine(updatesForCollection, Updates.setOnInsert("_id", collectionId)); + ApiCollectionsDao.instance.getMCollection().findOneAndUpdate(Filters.eq(ApiCollection.HOST_NAME, hostStringFromParam), currentUpdate, updateOptions); + sampleDataToBeMovedIntoCollection.put(sampleData.getId(), collectionId); + } break; }else if(filterType.equals(FILTER_TYPE.ALLOWED)){ // filter passed and not modified @@ -205,6 +230,8 @@ public static void cleanFilteredSampleDataFromAdvancedFilters(List iterator: moveHostCollectionWiseCountMap.entrySet()){ + int collId = iterator.getKey(); + int deletionCount = iterator.getValue(); + String name = apiCollectionMap.get(collId).getDisplayName(); + + if(saveLogsToDB){ + loggerMaker.infoAndAddToDb("Total apis moved from collection: " + name + " are: " + deletionCount, LogDb.DASHBOARD); + } + } + // writer.flush(); // writer.close(); } diff --git a/apps/dashboard/src/main/java/com/akto/utils/jobs/JobUtils.java b/apps/dashboard/src/main/java/com/akto/utils/jobs/JobUtils.java index d12010285e..cf57667439 100644 --- a/apps/dashboard/src/main/java/com/akto/utils/jobs/JobUtils.java +++ b/apps/dashboard/src/main/java/com/akto/utils/jobs/JobUtils.java @@ -1,5 +1,6 @@ package com.akto.utils.jobs; +import com.akto.dto.ApiCollection; import com.akto.util.DashboardMode; public class JobUtils { @@ -19,4 +20,12 @@ public static boolean getRunJobFunctionsAnyway() { return true; } } + + public static boolean hasHostModified(String hostNameFromParam, ApiCollection collection){ + if(collection.getHostName() == null){ + return true; + } + + return collection.getHostName().equals(hostNameFromParam); + } } \ No newline at end of file diff --git a/libs/dao/src/main/java/com/akto/dao/AccountsContextDaoWithRbac.java b/libs/dao/src/main/java/com/akto/dao/AccountsContextDaoWithRbac.java index 23d56217ca..be08e91302 100644 --- a/libs/dao/src/main/java/com/akto/dao/AccountsContextDaoWithRbac.java +++ b/libs/dao/src/main/java/com/akto/dao/AccountsContextDaoWithRbac.java @@ -2,6 +2,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.Map; import com.akto.dto.rbac.UsersCollectionsList; import com.akto.dto.traffic.Key; @@ -29,14 +30,30 @@ public static void deleteApisPerDao(List toBeDeleted, AccountsContextDa for(Key key: toBeDeleted) { stiList.add(new DeleteManyModel<>(Filters.and( - Filters.eq(prefix + "apiCollectionId", key.getApiCollectionId()), - Filters.eq(prefix + "method", key.getMethod()), - Filters.eq(prefix + "url", key.getUrl()) + Filters.eq(prefix + SingleTypeInfo._API_COLLECTION_ID, key.getApiCollectionId()), + Filters.eq(prefix + SingleTypeInfo._METHOD, key.getMethod()), + Filters.eq(prefix + SingleTypeInfo._URL, key.getUrl()) ))); } dao.bulkWrite(stiList, new BulkWriteOptions().ordered(false)); } + public static Bson generateCommonFilter(Map toBeInserted, AccountsContextDaoWithRbac dao, String prefix){ + List filterList = new ArrayList<>(); + for(Key key: toBeInserted.keySet()){ + Bson filter = Filters.and( + Filters.eq(prefix + SingleTypeInfo._API_COLLECTION_ID, key.getApiCollectionId()), + Filters.eq(prefix + SingleTypeInfo._METHOD, key.getMethod()), + Filters.eq(prefix + SingleTypeInfo._URL, key.getUrl()) + ); + + filterList.add(filter); + } + + Bson finalFilter = Filters.or(filterList); + return finalFilter; + } + abstract public String getFilterKeyString(); protected Bson addRbacFilter(Bson originalQuery) { diff --git a/libs/dao/src/main/java/com/akto/dto/traffic/Key.java b/libs/dao/src/main/java/com/akto/dto/traffic/Key.java index 5bef451824..112a2439a7 100644 --- a/libs/dao/src/main/java/com/akto/dto/traffic/Key.java +++ b/libs/dao/src/main/java/com/akto/dto/traffic/Key.java @@ -1,5 +1,7 @@ package com.akto.dto.traffic; +import java.util.Objects; + import com.akto.dto.type.URLMethods.Method; public class Key { @@ -74,5 +76,21 @@ public String toString() { return apiCollectionId + " " + url + " " + method + " " + responseCode; } + @Override + public int hashCode() { + return Objects.hash(url, method, apiCollectionId, responseCode); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Key key = (Key) o; + return apiCollectionId == key.apiCollectionId + && responseCode == key.responseCode + && Objects.equals(url, key.url) + && method == key.method; + } + } From 1f16f30d4c3d608f7ea1be47ec46f531a370bb5a Mon Sep 17 00:00:00 2001 From: Ark2307 Date: Mon, 30 Dec 2024 12:46:11 +0530 Subject: [PATCH 2/6] Fixing filter and modifying data --- .../src/main/java/com/akto/utils/Utils.java | 27 ++++++++++++++++--- .../com/akto/utils/jobs/CleanInventory.java | 2 +- 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/apps/dashboard/src/main/java/com/akto/utils/Utils.java b/apps/dashboard/src/main/java/com/akto/utils/Utils.java index cea8ddb9a0..77e8e6a3be 100644 --- a/apps/dashboard/src/main/java/com/akto/utils/Utils.java +++ b/apps/dashboard/src/main/java/com/akto/utils/Utils.java @@ -49,7 +49,6 @@ import org.apache.commons.lang3.tuple.Pair; import org.bson.conversions.Bson; -import java.lang.reflect.Method; import java.net.MalformedURLException; import java.net.URISyntaxException; import java.net.URL; @@ -634,10 +633,16 @@ private static Key copy(Key oldKey) { return newKey; } + private static List getModifiedCollectionIds(List oldList, int newCollId, int oldCollId){ + oldList.remove(oldCollId); + oldList.add(newCollId); + return oldList; + } + public static void moveApisFromSampleData(Map sampleDataToBeMovedMap){ Map mapApiInfoKeyToKey = new HashMap<>(); - + List toBeDeleted = new ArrayList<>(); // insert new sample data Bson sampleDataFilter = AccountsContextDaoWithRbac.generateCommonFilter(sampleDataToBeMovedMap, SampleDataDao.instance, id); List sampleDataList = SampleDataDao.instance.findAll(sampleDataFilter); @@ -652,6 +657,8 @@ public static void moveApisFromSampleData(Map sampleDataToBeMovedMa if(key.getApiCollectionId() != newCollId){ key.setApiCollectionId(newCollId); sampleData.setId(key); + sampleData.setCollectionIds(getModifiedCollectionIds(sampleData.getCollectionIds(), newCollId, oldKey.getApiCollectionId())); + toBeDeleted.add(oldKey); }else{ sampleDataIterator.remove(); } @@ -659,8 +666,10 @@ public static void moveApisFromSampleData(Map sampleDataToBeMovedMa if(!sampleDataList.isEmpty()){ SampleDataDao.instance.insertMany(sampleDataList); loggerMaker.infoAndAddToDb("Inserted " + sampleDataList.size() + " new sample data into database."); + AccountsContextDaoWithRbac.deleteApisPerDao(toBeDeleted, SampleDataDao.instance, id); } + toBeDeleted.clear(); // insert new api info data Bson apiInfoFilter = AccountsContextDaoWithRbac.generateCommonFilter(sampleDataToBeMovedMap, ApiInfoDao.instance, id); List apiInfos = ApiInfoDao.instance.findAll(apiInfoFilter); @@ -673,6 +682,8 @@ public static void moveApisFromSampleData(Map sampleDataToBeMovedMa if(mappedKey.getApiCollectionId() != newCollId){ apiInfoKey.setApiCollectionId(newCollId); apiInfo.setId(apiInfoKey); + apiInfo.setCollectionIds(getModifiedCollectionIds(apiInfo.getCollectionIds(), newCollId, mappedKey.getApiCollectionId())); + toBeDeleted.add(mappedKey); }else{ apiInfoIterator.remove(); } @@ -681,8 +692,10 @@ public static void moveApisFromSampleData(Map sampleDataToBeMovedMa if(!apiInfos.isEmpty()){ ApiInfoDao.instance.insertMany(apiInfos); loggerMaker.infoAndAddToDb("Inserted " + apiInfos.size() + " new api infos into database."); + AccountsContextDaoWithRbac.deleteApisPerDao(toBeDeleted, ApiInfoDao.instance, id); } + toBeDeleted.clear(); // insert new sensitive sample data Bson sensitiveDataFilter = AccountsContextDaoWithRbac.generateCommonFilter(sampleDataToBeMovedMap, SensitiveSampleDataDao.instance, id); List sensitiveSampleDataList = SensitiveSampleDataDao.instance.findAll(sensitiveDataFilter); @@ -697,6 +710,8 @@ public static void moveApisFromSampleData(Map sampleDataToBeMovedMa if(key.getApiCollectionId() != newCollId){ key.setApiCollectionId(newCollId); sampleData.setId(key); + sampleData.setCollectionIds(getModifiedCollectionIds(sampleData.getCollectionIds(), newCollId, mappedKey.getApiCollectionId())); + toBeDeleted.add(mappedKey); }else{ sensitiveSampleDataIterator.remove(); } @@ -704,6 +719,7 @@ public static void moveApisFromSampleData(Map sampleDataToBeMovedMa if(!sensitiveSampleDataList.isEmpty()){ SensitiveSampleDataDao.instance.insertMany(sensitiveSampleDataList); loggerMaker.infoAndAddToDb("Inserted " + sensitiveSampleDataList.size() + " new sensitive sample data into database."); + AccountsContextDaoWithRbac.deleteApisPerDao(toBeDeleted, SensitiveSampleDataDao.instance, id); } // update single type info data @@ -714,7 +730,12 @@ public static void moveApisFromSampleData(Map sampleDataToBeMovedMa Filters.eq(SingleTypeInfo._URL, key.getUrl()), Filters.eq(SingleTypeInfo._METHOD, key.getMethod()) ); - bulkUpdatesForSti.add(new UpdateManyModel<>(filterQ, Updates.set(SingleTypeInfo._API_COLLECTION_ID, sampleDataToBeMovedMap.get(key)))); + bulkUpdatesForSti.add(new UpdateManyModel<>(filterQ, + Updates.combine( + Updates.set(SingleTypeInfo._API_COLLECTION_ID, sampleDataToBeMovedMap.get(key)), + Updates.pull(SingleTypeInfo._COLLECTION_IDS, key.getApiCollectionId()), + Updates.push(SingleTypeInfo._COLLECTION_IDS, sampleDataToBeMovedMap.get(key)) + ))); } if(!bulkUpdatesForSti.isEmpty()){ diff --git a/apps/dashboard/src/main/java/com/akto/utils/jobs/CleanInventory.java b/apps/dashboard/src/main/java/com/akto/utils/jobs/CleanInventory.java index 98998cfa2d..6441324ac8 100644 --- a/apps/dashboard/src/main/java/com/akto/utils/jobs/CleanInventory.java +++ b/apps/dashboard/src/main/java/com/akto/utils/jobs/CleanInventory.java @@ -218,7 +218,7 @@ public static void cleanFilteredSampleDataFromAdvancedFilters(List Date: Thu, 2 Jan 2025 14:38:50 +0530 Subject: [PATCH 3/6] Moving sample data --- apps/dashboard/src/main/java/com/akto/utils/Utils.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/apps/dashboard/src/main/java/com/akto/utils/Utils.java b/apps/dashboard/src/main/java/com/akto/utils/Utils.java index 77e8e6a3be..ddb26cf16b 100644 --- a/apps/dashboard/src/main/java/com/akto/utils/Utils.java +++ b/apps/dashboard/src/main/java/com/akto/utils/Utils.java @@ -634,7 +634,7 @@ private static Key copy(Key oldKey) { } private static List getModifiedCollectionIds(List oldList, int newCollId, int oldCollId){ - oldList.remove(oldCollId); + oldList.removeIf(num -> num == oldCollId); oldList.add(newCollId); return oldList; } @@ -733,8 +733,7 @@ public static void moveApisFromSampleData(Map sampleDataToBeMovedMa bulkUpdatesForSti.add(new UpdateManyModel<>(filterQ, Updates.combine( Updates.set(SingleTypeInfo._API_COLLECTION_ID, sampleDataToBeMovedMap.get(key)), - Updates.pull(SingleTypeInfo._COLLECTION_IDS, key.getApiCollectionId()), - Updates.push(SingleTypeInfo._COLLECTION_IDS, sampleDataToBeMovedMap.get(key)) + Updates.addToSet(SingleTypeInfo._COLLECTION_IDS, sampleDataToBeMovedMap.get(key)) ))); } From 059195db522edf76895c6944de156ee8a51a3a87 Mon Sep 17 00:00:00 2001 From: Ark2307 Date: Fri, 3 Jan 2025 15:52:24 +0530 Subject: [PATCH 4/6] Handling edge cases for modifying data --- .../src/main/java/com/akto/utils/Utils.java | 40 +++++++++++++++---- .../main/java/com/akto/dao/ApiInfoDao.java | 27 +++++++++++++ .../main/java/com/akto/dao/SampleDataDao.java | 13 ++++++ .../com/akto/dao/SensitiveSampleDataDao.java | 13 ++++++ .../com/akto/dto/SensitiveSampleData.java | 2 + 5 files changed, 87 insertions(+), 8 deletions(-) diff --git a/apps/dashboard/src/main/java/com/akto/utils/Utils.java b/apps/dashboard/src/main/java/com/akto/utils/Utils.java index ddb26cf16b..a7c978cd44 100644 --- a/apps/dashboard/src/main/java/com/akto/utils/Utils.java +++ b/apps/dashboard/src/main/java/com/akto/utils/Utils.java @@ -34,6 +34,7 @@ import com.akto.parsers.HttpCallParser; import com.akto.runtime.APICatalogSync; import com.akto.testing.ApiExecutor; +import com.akto.util.Constants; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ArrayNode; @@ -42,6 +43,8 @@ import com.mongodb.client.model.BulkWriteOptions; import com.mongodb.client.model.Filters; import com.mongodb.client.model.UpdateManyModel; +import com.mongodb.client.model.UpdateOneModel; +import com.mongodb.client.model.UpdateOptions; import com.mongodb.client.model.Updates; import com.mongodb.client.model.WriteModel; @@ -647,6 +650,10 @@ public static void moveApisFromSampleData(Map sampleDataToBeMovedMa Bson sampleDataFilter = AccountsContextDaoWithRbac.generateCommonFilter(sampleDataToBeMovedMap, SampleDataDao.instance, id); List sampleDataList = SampleDataDao.instance.findAll(sampleDataFilter); Iterator sampleDataIterator = sampleDataList.iterator(); + UpdateOptions upsertOptions = new UpdateOptions().upsert(true); + + ArrayList> bulkUpdatesForSampleData = new ArrayList<>(); + while(sampleDataIterator.hasNext()){ SampleData sampleData = sampleDataIterator.next(); Key key = sampleData.getId(); @@ -658,18 +665,24 @@ public static void moveApisFromSampleData(Map sampleDataToBeMovedMa key.setApiCollectionId(newCollId); sampleData.setId(key); sampleData.setCollectionIds(getModifiedCollectionIds(sampleData.getCollectionIds(), newCollId, oldKey.getApiCollectionId())); + Bson update = SampleDataDao.instance.getUpdateFromSampleData(sampleData); + bulkUpdatesForSampleData.add( + new UpdateOneModel<>(Filters.eq(Constants.ID, sampleData.getId()), update, upsertOptions) + ); toBeDeleted.add(oldKey); }else{ sampleDataIterator.remove(); } } - if(!sampleDataList.isEmpty()){ - SampleDataDao.instance.insertMany(sampleDataList); + + if(!bulkUpdatesForSampleData.isEmpty()){ + SampleDataDao.instance.getMCollection().bulkWrite(bulkUpdatesForSampleData); loggerMaker.infoAndAddToDb("Inserted " + sampleDataList.size() + " new sample data into database."); AccountsContextDaoWithRbac.deleteApisPerDao(toBeDeleted, SampleDataDao.instance, id); } - toBeDeleted.clear(); + + ArrayList> bulkUpdatesForApiInfo = new ArrayList<>(); // insert new api info data Bson apiInfoFilter = AccountsContextDaoWithRbac.generateCommonFilter(sampleDataToBeMovedMap, ApiInfoDao.instance, id); List apiInfos = ApiInfoDao.instance.findAll(apiInfoFilter); @@ -684,18 +697,24 @@ public static void moveApisFromSampleData(Map sampleDataToBeMovedMa apiInfo.setId(apiInfoKey); apiInfo.setCollectionIds(getModifiedCollectionIds(apiInfo.getCollectionIds(), newCollId, mappedKey.getApiCollectionId())); toBeDeleted.add(mappedKey); + Bson update = ApiInfoDao.instance.getUpdateFromApiInfo(apiInfo); + bulkUpdatesForSampleData.add( + new UpdateOneModel<>(Filters.eq(Constants.ID, apiInfoKey), update, upsertOptions) + ); + }else{ apiInfoIterator.remove(); } } - if(!apiInfos.isEmpty()){ - ApiInfoDao.instance.insertMany(apiInfos); + if(!bulkUpdatesForApiInfo.isEmpty()){ + ApiInfoDao.instance.getMCollection().bulkWrite(bulkUpdatesForApiInfo); loggerMaker.infoAndAddToDb("Inserted " + apiInfos.size() + " new api infos into database."); AccountsContextDaoWithRbac.deleteApisPerDao(toBeDeleted, ApiInfoDao.instance, id); } - toBeDeleted.clear(); + + ArrayList> bulkUpdatesForSensitiveSampleData = new ArrayList<>(); // insert new sensitive sample data Bson sensitiveDataFilter = AccountsContextDaoWithRbac.generateCommonFilter(sampleDataToBeMovedMap, SensitiveSampleDataDao.instance, id); List sensitiveSampleDataList = SensitiveSampleDataDao.instance.findAll(sensitiveDataFilter); @@ -711,16 +730,21 @@ public static void moveApisFromSampleData(Map sampleDataToBeMovedMa key.setApiCollectionId(newCollId); sampleData.setId(key); sampleData.setCollectionIds(getModifiedCollectionIds(sampleData.getCollectionIds(), newCollId, mappedKey.getApiCollectionId())); + Bson update = SensitiveSampleDataDao.instance.getUpdateFromSampleData(sampleData); + bulkUpdatesForSampleData.add( + new UpdateOneModel<>(Filters.eq(Constants.ID, sampleData.getId()), update, upsertOptions) + ); toBeDeleted.add(mappedKey); }else{ sensitiveSampleDataIterator.remove(); } } - if(!sensitiveSampleDataList.isEmpty()){ - SensitiveSampleDataDao.instance.insertMany(sensitiveSampleDataList); + if(!bulkUpdatesForSensitiveSampleData.isEmpty()){ + SensitiveSampleDataDao.instance.getMCollection().bulkWrite(bulkUpdatesForSensitiveSampleData); loggerMaker.infoAndAddToDb("Inserted " + sensitiveSampleDataList.size() + " new sensitive sample data into database."); AccountsContextDaoWithRbac.deleteApisPerDao(toBeDeleted, SensitiveSampleDataDao.instance, id); } + toBeDeleted.clear(); // update single type info data ArrayList> bulkUpdatesForSti = new ArrayList<>(); diff --git a/libs/dao/src/main/java/com/akto/dao/ApiInfoDao.java b/libs/dao/src/main/java/com/akto/dao/ApiInfoDao.java index d84d000ad5..701a67929f 100644 --- a/libs/dao/src/main/java/com/akto/dao/ApiInfoDao.java +++ b/libs/dao/src/main/java/com/akto/dao/ApiInfoDao.java @@ -263,6 +263,33 @@ public Pair fetchApiInfoStats(Bson collectionFilter, int star return new Pair<>(apiStatsStart, apiStatsEnd); } + public Bson getUpdateFromApiInfo(ApiInfo apiInfo){ + + Bson update = Updates.combine( + Updates.setOnInsert(Constants.ID, apiInfo.getId()), + Updates.addEachToSet(ApiInfo.ALL_AUTH_TYPES_FOUND, Arrays.asList(apiInfo.getAllAuthTypesFound().toArray())), + Updates.set(ApiInfo.LAST_SEEN, apiInfo.getLastSeen()), + Updates.set(ApiInfo.LAST_TESTED, apiInfo.getLastTested()), + Updates.set(ApiInfo.IS_SENSITIVE, apiInfo.getIsSensitive()), + Updates.set(ApiInfo.SEVERITY_SCORE, apiInfo.getSeverityScore()), + Updates.set(ApiInfo.RISK_SCORE, apiInfo.getRiskScore()), + Updates.set(ApiInfo.LAST_CALCULATED_TIME, apiInfo.getLastCalculatedTime()), + Updates.set(ApiInfo.API_TYPE, apiInfo.getApiType()), + Updates.set(ApiInfo.RESPONSE_CODES, apiInfo.getResponseCodes()), + Updates.addEachToSet(ApiInfo.COLLECTION_IDS, apiInfo.getCollectionIds()), + Updates.set(ApiInfo.DISCOVERED_TIMESTAMP, apiInfo.getDiscoveredTimestamp()) + ); + + Set apiAccessTypes = apiInfo.getApiAccessTypes(); + if (apiAccessTypes.isEmpty()) { + update = Updates.combine(update,Updates.setOnInsert(ApiInfo.API_ACCESS_TYPES, new HashSet<>())); + } else { + update = Updates.combine(update,Updates.addEachToSet(ApiInfo.API_ACCESS_TYPES, Arrays.asList(apiAccessTypes.toArray()))); + } + + return update; + } + @Override public String getCollName() { return "api_info"; diff --git a/libs/dao/src/main/java/com/akto/dao/SampleDataDao.java b/libs/dao/src/main/java/com/akto/dao/SampleDataDao.java index f75617f4fa..c88ce14701 100644 --- a/libs/dao/src/main/java/com/akto/dao/SampleDataDao.java +++ b/libs/dao/src/main/java/com/akto/dao/SampleDataDao.java @@ -6,11 +6,14 @@ import com.akto.dto.testing.TestingEndpoints; import com.akto.dto.traffic.SampleData; import com.akto.dto.type.URLMethods; +import com.akto.util.Constants; import com.akto.dto.type.SingleTypeInfo; import com.mongodb.client.MongoCursor; import com.mongodb.client.model.Filters; import com.mongodb.client.model.Projections; import com.mongodb.client.model.Sorts; +import com.mongodb.client.model.Updates; + import org.bson.conversions.Bson; import java.util.ArrayList; @@ -161,6 +164,16 @@ public List fetchSampleDataPaginated(String lastFetchedUrl, return sampleDataList; } + public Bson getUpdateFromSampleData(SampleData sampleData){ + Bson update = Updates.combine( + Updates.setOnInsert(Constants.ID, sampleData.getId()), + Updates.setOnInsert(SingleTypeInfo._COLLECTION_IDS, sampleData.getCollectionIds()), + Updates.addEachToSet(SampleData.SAMPLES, sampleData.getSamples()) + ); + + return update; + } + @Override public String getFilterKeyString() { diff --git a/libs/dao/src/main/java/com/akto/dao/SensitiveSampleDataDao.java b/libs/dao/src/main/java/com/akto/dao/SensitiveSampleDataDao.java index 7aaa331d11..7ed41e96c8 100644 --- a/libs/dao/src/main/java/com/akto/dao/SensitiveSampleDataDao.java +++ b/libs/dao/src/main/java/com/akto/dao/SensitiveSampleDataDao.java @@ -5,10 +5,13 @@ import com.akto.dto.ApiInfo; import com.akto.dto.SensitiveSampleData; import com.akto.dto.testing.TestingEndpoints; +import com.akto.dto.traffic.SampleData; import com.akto.dto.type.SingleTypeInfo; +import com.akto.util.Constants; import com.mongodb.client.MongoCursor; import com.mongodb.client.model.Filters; import com.mongodb.client.model.Indexes; +import com.mongodb.client.model.Updates; import java.util.HashMap; import java.util.Map; @@ -78,6 +81,16 @@ public void createIndicesIfAbsent() { new String[] { SingleTypeInfo._COLLECTION_IDS }, true); } + public Bson getUpdateFromSampleData(SensitiveSampleData sampleData){ + Bson update = Updates.combine( + Updates.setOnInsert(Constants.ID, sampleData.getId()), + Updates.setOnInsert(SingleTypeInfo._COLLECTION_IDS, sampleData.getCollectionIds()), + Updates.addEachToSet(SampleData.SAMPLES, sampleData.getSampleData()) + ); + + return update; + } + @Override public String getFilterKeyString() { return TestingEndpoints.getFilterPrefix(ApiCollectionUsers.CollectionType.Id_ApiCollectionId) + ApiInfo.ApiInfoKey.API_COLLECTION_ID; diff --git a/libs/dao/src/main/java/com/akto/dto/SensitiveSampleData.java b/libs/dao/src/main/java/com/akto/dto/SensitiveSampleData.java index 42db8d3dfc..a16606cebc 100644 --- a/libs/dao/src/main/java/com/akto/dto/SensitiveSampleData.java +++ b/libs/dao/src/main/java/com/akto/dto/SensitiveSampleData.java @@ -25,6 +25,8 @@ public SensitiveSampleData(SingleTypeInfo.ParamId id, List sampleData) { } } + + public SingleTypeInfo.ParamId getId() { return id; } From 2d83cf370e1de92b8823ffdd1ef0163d232f7dfd Mon Sep 17 00:00:00 2001 From: Ark2307 Date: Sun, 5 Jan 2025 14:07:29 +0530 Subject: [PATCH 5/6] Solving multiple apis cases from different collection --- .../src/main/java/com/akto/utils/Utils.java | 25 ++++++++++++++----- .../com/akto/utils/jobs/CleanInventory.java | 6 ++++- 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/apps/dashboard/src/main/java/com/akto/utils/Utils.java b/apps/dashboard/src/main/java/com/akto/utils/Utils.java index a7c978cd44..03364af7df 100644 --- a/apps/dashboard/src/main/java/com/akto/utils/Utils.java +++ b/apps/dashboard/src/main/java/com/akto/utils/Utils.java @@ -41,6 +41,7 @@ import com.google.gson.Gson; import com.mongodb.BasicDBObject; import com.mongodb.client.model.BulkWriteOptions; +import com.mongodb.client.model.DeleteManyModel; import com.mongodb.client.model.Filters; import com.mongodb.client.model.UpdateManyModel; import com.mongodb.client.model.UpdateOneModel; @@ -642,7 +643,7 @@ private static List getModifiedCollectionIds(List oldList, int return oldList; } - public static void moveApisFromSampleData(Map sampleDataToBeMovedMap){ + public static void moveApisFromSampleData(Map sampleDataToBeMovedMap, Set alreadySeenApis){ Map mapApiInfoKeyToKey = new HashMap<>(); List toBeDeleted = new ArrayList<>(); @@ -748,17 +749,24 @@ public static void moveApisFromSampleData(Map sampleDataToBeMovedMa // update single type info data ArrayList> bulkUpdatesForSti = new ArrayList<>(); + ArrayList> deleteListForStis = new ArrayList<>(); for(Key key: sampleDataToBeMovedMap.keySet()){ + String currentApiInfoKey = key.getUrl() + "?#?" + key.getMethod(); Bson filterQ = Filters.and( Filters.eq(SingleTypeInfo._API_COLLECTION_ID, key.getApiCollectionId()), Filters.eq(SingleTypeInfo._URL, key.getUrl()), Filters.eq(SingleTypeInfo._METHOD, key.getMethod()) ); - bulkUpdatesForSti.add(new UpdateManyModel<>(filterQ, - Updates.combine( - Updates.set(SingleTypeInfo._API_COLLECTION_ID, sampleDataToBeMovedMap.get(key)), - Updates.addToSet(SingleTypeInfo._COLLECTION_IDS, sampleDataToBeMovedMap.get(key)) - ))); + if(alreadySeenApis.contains(currentApiInfoKey)){ + bulkUpdatesForSti.add(new UpdateManyModel<>(filterQ, + Updates.combine( + Updates.set(SingleTypeInfo._API_COLLECTION_ID, sampleDataToBeMovedMap.get(key)), + Updates.addToSet(SingleTypeInfo._COLLECTION_IDS, sampleDataToBeMovedMap.get(key)) + ))); + alreadySeenApis.remove(currentApiInfoKey); + }else{ + deleteListForStis.add(new DeleteManyModel<>(filterQ)); + } } if(!bulkUpdatesForSti.isEmpty()){ @@ -766,6 +774,11 @@ public static void moveApisFromSampleData(Map sampleDataToBeMovedMa SingleTypeInfoDao.instance.getMCollection().bulkWrite(bulkUpdatesForSti, new BulkWriteOptions().ordered(false)); } + if(!deleteListForStis.isEmpty()){ + loggerMaker.infoAndAddToDb("Deleting " + deleteListForStis.size() + " stis into database."); + SingleTypeInfoDao.instance.getMCollection().bulkWrite(deleteListForStis, new BulkWriteOptions().ordered(false)); + } + } public static List getUniqueValuesOfList(List input){ diff --git a/apps/dashboard/src/main/java/com/akto/utils/jobs/CleanInventory.java b/apps/dashboard/src/main/java/com/akto/utils/jobs/CleanInventory.java index 6441324ac8..016a189f7d 100644 --- a/apps/dashboard/src/main/java/com/akto/utils/jobs/CleanInventory.java +++ b/apps/dashboard/src/main/java/com/akto/utils/jobs/CleanInventory.java @@ -148,6 +148,8 @@ public static void cleanFilteredSampleDataFromAdvancedFilters(List()) ); + Set alreadySeenApis = new HashSet<>(); + Map filterMap = FilterYamlTemplateDao.instance.fetchFilterConfig(false, yamlTemplates, true); Pattern pattern = createRegexPatternFromList(redundantUrlList); do { @@ -232,6 +234,8 @@ public static void cleanFilteredSampleDataFromAdvancedFilters(List Date: Sun, 5 Jan 2025 14:40:14 +0530 Subject: [PATCH 6/6] Handling for saas accounts --- .../java/com/akto/utils/jobs/CleanInventory.java | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/apps/dashboard/src/main/java/com/akto/utils/jobs/CleanInventory.java b/apps/dashboard/src/main/java/com/akto/utils/jobs/CleanInventory.java index 016a189f7d..28cf70c7bf 100644 --- a/apps/dashboard/src/main/java/com/akto/utils/jobs/CleanInventory.java +++ b/apps/dashboard/src/main/java/com/akto/utils/jobs/CleanInventory.java @@ -290,8 +290,18 @@ public static void cleanFilteredSampleDataFromAdvancedFilters(List accounts = new HashSet<>(Arrays.asList(accountStringList)); + String currentAccount = String.valueOf(Context.accountId.get()); + shouldMove = accounts.contains(currentAccount); + }else if(System.getenv("MOVE_REDUNDANT_APIS") != null && System.getenv("MOVE_REDUNDANT_APIS").equals("true")){ + shouldMove =true; + } + if(shouldMove){ if(!sampleDataToBeMovedIntoCollection.isEmpty()){ moveApisFromSampleData(sampleDataToBeMovedIntoCollection, alreadySeenApis); sampleDataToBeMovedIntoCollection.clear();