From 16091e4895ee131adfcc0e21fe3c2c4b52315419 Mon Sep 17 00:00:00 2001 From: notshivansh Date: Fri, 3 Jan 2025 15:32:23 +0530 Subject: [PATCH 1/3] add clean up api --- .../java/com/akto/action/CleanAction.java | 96 +++++++++++++++++++ apps/dashboard/src/main/resources/struts.xml | 22 +++++ 2 files changed, 118 insertions(+) create mode 100644 apps/dashboard/src/main/java/com/akto/action/CleanAction.java diff --git a/apps/dashboard/src/main/java/com/akto/action/CleanAction.java b/apps/dashboard/src/main/java/com/akto/action/CleanAction.java new file mode 100644 index 0000000000..0650d95850 --- /dev/null +++ b/apps/dashboard/src/main/java/com/akto/action/CleanAction.java @@ -0,0 +1,96 @@ +package com.akto.action; + +import java.util.ArrayList; +import java.util.List; + +import org.bson.conversions.Bson; + +import com.akto.DaoInit; +import com.akto.dao.ApiInfoDao; +import com.akto.dao.SingleTypeInfoDao; +import com.akto.dao.context.Context; +import com.akto.dto.ApiInfo; +import com.akto.dto.ApiInfo.ApiInfoKey; +import com.akto.dto.type.SingleTypeInfo; +import com.akto.log.LoggerMaker; +import com.akto.log.LoggerMaker.LogDb; +import com.mongodb.ConnectionString; +import com.mongodb.client.model.Filters; +import com.mongodb.client.model.Projections; +import com.mongodb.client.result.DeleteResult; +import com.opensymphony.xwork2.Action; + +public class CleanAction extends UserAction { + + private static final LoggerMaker loggerMaker = new LoggerMaker(CleanAction.class, LogDb.DASHBOARD); + + /* + * delete api info if corresponding sti not found. + */ + + List apiCollectionIds; + boolean runActually; + + public String deleteExtraApiInfo() { + + List apiInfoKeys = new ArrayList<>(); + int count = 0; + for (int apiCollectionId : apiCollectionIds) { + List apiInfos = ApiInfoDao.instance.findAll(Filters.eq("_id.apiCollectionId", apiCollectionId), + Projections.include("_id")); + + if(apiInfos == null) { + loggerMaker.infoAndAddToDb("No API Info found for API Collection Id: " + apiCollectionId); + continue; + } + + loggerMaker.infoAndAddToDb("Checking ApiInfos count: " + apiInfos.size()); + for (ApiInfo apiInfo : apiInfos) { + ApiInfoKey key = apiInfo.getId(); + + SingleTypeInfo sti = SingleTypeInfoDao.instance.findOne(SingleTypeInfoDao + .filterForSTIUsingURL(key.getApiCollectionId(), key.getUrl(), key.getMethod())); + + if (sti != null) { + continue; + } + count++; + + loggerMaker.infoAndAddToDb("No STI found for API Info: " + key.toString()); + + if (runActually) { + apiInfoKeys.add(key); + } + } + } + loggerMaker.infoAndAddToDb("Total API Info to delete: " + count); + + if (runActually && apiInfoKeys.size() > 0) { + List filters = new ArrayList<>(); + for(ApiInfoKey key : apiInfoKeys) { + filters.add(ApiInfoDao.getFilter(key)); + } + loggerMaker.infoAndAddToDb("deleteExtraApiInfo Actually deleting : " + count); + DeleteResult res = ApiInfoDao.instance.deleteAll(Filters.eq("_id", Filters.or(filters))); + loggerMaker.infoAndAddToDb("deleteExtraApiInfo Actually deleted : " + res.getDeletedCount()); + } + + return Action.SUCCESS.toUpperCase(); + } + + public List getApiCollectionIds() { + return apiCollectionIds; + } + + public void setApiCollectionIds(List apiCollectionIds) { + this.apiCollectionIds = apiCollectionIds; + } + + public boolean getRunActually() { + return runActually; + } + + public void setRunActually(boolean runActually) { + this.runActually = runActually; + } +} diff --git a/apps/dashboard/src/main/resources/struts.xml b/apps/dashboard/src/main/resources/struts.xml index a5d6881002..65a52bcaf7 100644 --- a/apps/dashboard/src/main/resources/struts.xml +++ b/apps/dashboard/src/main/resources/struts.xml @@ -7501,6 +7501,28 @@ + + + + + API_COLLECTIONS + READ + + + + 403 + false + ^actionErrors.* + + + + + 422 + false + ^actionErrors.* + + + From 1078f986cb783cce792c45bbb87c278facc62568 Mon Sep 17 00:00:00 2001 From: notshivansh Date: Fri, 3 Jan 2025 15:44:09 +0530 Subject: [PATCH 2/3] fix api --- apps/dashboard/src/main/java/com/akto/action/CleanAction.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/dashboard/src/main/java/com/akto/action/CleanAction.java b/apps/dashboard/src/main/java/com/akto/action/CleanAction.java index 0650d95850..9fc9311ef1 100644 --- a/apps/dashboard/src/main/java/com/akto/action/CleanAction.java +++ b/apps/dashboard/src/main/java/com/akto/action/CleanAction.java @@ -71,7 +71,7 @@ public String deleteExtraApiInfo() { filters.add(ApiInfoDao.getFilter(key)); } loggerMaker.infoAndAddToDb("deleteExtraApiInfo Actually deleting : " + count); - DeleteResult res = ApiInfoDao.instance.deleteAll(Filters.eq("_id", Filters.or(filters))); + DeleteResult res = ApiInfoDao.instance.deleteAll(Filters.or(filters)); loggerMaker.infoAndAddToDb("deleteExtraApiInfo Actually deleted : " + res.getDeletedCount()); } From 367d4b6ce39ecf11eaf60e208fc92e56d40536c8 Mon Sep 17 00:00:00 2001 From: notshivansh Date: Fri, 3 Jan 2025 16:38:15 +0530 Subject: [PATCH 3/3] add batching --- .../java/com/akto/action/CleanAction.java | 65 ++++++++++++------- 1 file changed, 41 insertions(+), 24 deletions(-) diff --git a/apps/dashboard/src/main/java/com/akto/action/CleanAction.java b/apps/dashboard/src/main/java/com/akto/action/CleanAction.java index 9fc9311ef1..d01e21b9ad 100644 --- a/apps/dashboard/src/main/java/com/akto/action/CleanAction.java +++ b/apps/dashboard/src/main/java/com/akto/action/CleanAction.java @@ -1,20 +1,19 @@ package com.akto.action; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; import org.bson.conversions.Bson; -import com.akto.DaoInit; import com.akto.dao.ApiInfoDao; import com.akto.dao.SingleTypeInfoDao; -import com.akto.dao.context.Context; import com.akto.dto.ApiInfo; import com.akto.dto.ApiInfo.ApiInfoKey; import com.akto.dto.type.SingleTypeInfo; +import com.akto.dto.type.URLMethods.Method; import com.akto.log.LoggerMaker; import com.akto.log.LoggerMaker.LogDb; -import com.mongodb.ConnectionString; import com.mongodb.client.model.Filters; import com.mongodb.client.model.Projections; import com.mongodb.client.result.DeleteResult; @@ -33,8 +32,7 @@ public class CleanAction extends UserAction { public String deleteExtraApiInfo() { - List apiInfoKeys = new ArrayList<>(); - int count = 0; + List deleteFilters = new ArrayList<>(); for (int apiCollectionId : apiCollectionIds) { List apiInfos = ApiInfoDao.instance.findAll(Filters.eq("_id.apiCollectionId", apiCollectionId), Projections.include("_id")); @@ -45,39 +43,58 @@ public String deleteExtraApiInfo() { } loggerMaker.infoAndAddToDb("Checking ApiInfos count: " + apiInfos.size()); + List filters = new ArrayList<>(); for (ApiInfo apiInfo : apiInfos) { ApiInfoKey key = apiInfo.getId(); - SingleTypeInfo sti = SingleTypeInfoDao.instance.findOne(SingleTypeInfoDao - .filterForSTIUsingURL(key.getApiCollectionId(), key.getUrl(), key.getMethod())); + filters.add(key); - if (sti != null) { - continue; - } - count++; - - loggerMaker.infoAndAddToDb("No STI found for API Info: " + key.toString()); - - if (runActually) { - apiInfoKeys.add(key); + if (filters.size() >= 100) { + deleteFilters.addAll(checkSTIs(filters, runActually)); + filters.clear(); } } + if (!filters.isEmpty()) { + deleteFilters.addAll(checkSTIs(filters, runActually)); + } } - loggerMaker.infoAndAddToDb("Total API Info to delete: " + count); + loggerMaker.infoAndAddToDb("Total API Info to delete: " + deleteFilters.size()); - if (runActually && apiInfoKeys.size() > 0) { - List filters = new ArrayList<>(); - for(ApiInfoKey key : apiInfoKeys) { - filters.add(ApiInfoDao.getFilter(key)); - } - loggerMaker.infoAndAddToDb("deleteExtraApiInfo Actually deleting : " + count); - DeleteResult res = ApiInfoDao.instance.deleteAll(Filters.or(filters)); + if (runActually && deleteFilters.size() > 0) { + loggerMaker.infoAndAddToDb("deleteExtraApiInfo Actually deleting : " + deleteFilters.size()); + DeleteResult res = ApiInfoDao.instance.deleteAll(Filters.or(deleteFilters)); loggerMaker.infoAndAddToDb("deleteExtraApiInfo Actually deleted : " + res.getDeletedCount()); } return Action.SUCCESS.toUpperCase(); } + private static List checkSTIs(List filters, boolean runActually) { + List deleteFilters = new ArrayList<>(); + List filters2 = new ArrayList<>(); + for(ApiInfoKey key : filters) { + filters2.add(SingleTypeInfoDao.filterForSTIUsingURL(key.getApiCollectionId(), key.getUrl(), key.getMethod())); + } + List sti = SingleTypeInfoDao.instance.findAll(Filters.or(filters2)); + HashSet stiSet = new HashSet<>(); + if (sti != null && !sti.isEmpty()) { + for (SingleTypeInfo st : sti) { + stiSet.add(new ApiInfoKey(st.getApiCollectionId(), st.getUrl(), Method.valueOf(st.getMethod()))); + } + } + for(ApiInfoKey key : filters) { + if(stiSet.contains(key)) { + continue; + } + loggerMaker.infoAndAddToDb("STI not found for STI: " + key.toString()); + if (runActually) { + deleteFilters.add(ApiInfoDao.getFilter(key)); + } + } + + return deleteFilters; + } + public List getApiCollectionIds() { return apiCollectionIds; }