From d4f1244a97a31321b821c2b2846f85ad8bfdc301 Mon Sep 17 00:00:00 2001 From: Oren Saldanha Date: Thu, 9 May 2024 13:38:39 +0530 Subject: [PATCH 01/11] risk score based api groups initial changes --- .../akto/listener/InitializerListener.java | 34 +++++++- libs/dao/src/main/java/com/akto/DaoInit.java | 4 +- .../com/akto/dto/BackwardCompatibility.java | 14 ++- .../testing/RiskScoreTestingEndpoints.java | 87 +++++++++++++++++++ .../akto/dto/testing/TestingEndpoints.java | 2 +- 5 files changed, 134 insertions(+), 7 deletions(-) create mode 100644 libs/dao/src/main/java/com/akto/dto/testing/RiskScoreTestingEndpoints.java diff --git a/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java b/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java index 91f0fef5a7..7fda650ae0 100644 --- a/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java +++ b/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java @@ -1115,7 +1115,7 @@ public static void moveAuthMechanismDataToRole(BackwardCompatibility backwardCom } } - public static void createApiGroup(int id, String name, String regex) { + public static void createLoginApiGroup(int id, String name, String regex) { ApiCollection loginGroup = new ApiCollection(id, name, Context.now(), new HashSet<>(), null, 0, false, false); loginGroup.setType(ApiCollection.Type.API_GROUP); ArrayList loginConditions = new ArrayList<>(); @@ -1130,9 +1130,9 @@ public static void createApiGroup(int id, String name, String regex) { public static void createLoginSignupGroups(BackwardCompatibility backwardCompatibility) { if (backwardCompatibility.getLoginSignupGroups() == 0) { - createApiGroup(111_111_128, "Login APIs", "^((https?):\\/\\/)?(www\\.)?.*?(login|signin|sign-in|authenticate|session)(.*?)(\\?.*|\\/?|#.*?)?$"); - createApiGroup(111_111_129, "Signup APIs", "^((https?):\\/\\/)?(www\\.)?.*?(register|signup|sign-up|users\\/create|account\\/create|account_create|create_account)(.*?)(\\?.*|\\/?|#.*?)?$"); - createApiGroup(111_111_130, "Password Reset APIs", "^((https?):\\/\\/)?(www\\.)?.*?(password-reset|reset-password|forgot-password|user\\/reset|account\\/recover|api\\/password_reset|password\\/reset|account\\/reset-password-request|password_reset_request|account_recovery)(.*?)(\\?.*|\\/?|#.*?)?$"); + createLoginApiGroup(111_111_128, "Login APIs", "^((https?):\\/\\/)?(www\\.)?.*?(login|signin|sign-in|authenticate|session)(.*?)(\\?.*|\\/?|#.*?)?$"); + createLoginApiGroup(111_111_129, "Signup APIs", "^((https?):\\/\\/)?(www\\.)?.*?(register|signup|sign-up|users\\/create|account\\/create|account_create|create_account)(.*?)(\\?.*|\\/?|#.*?)?$"); + createLoginApiGroup(111_111_130, "Password Reset APIs", "^((https?):\\/\\/)?(www\\.)?.*?(password-reset|reset-password|forgot-password|user\\/reset|account\\/recover|api\\/password_reset|password\\/reset|account\\/reset-password-request|password_reset_request|account_recovery)(.*?)(\\?.*|\\/?|#.*?)?$"); BackwardCompatibilityDao.instance.updateOne( Filters.eq("_id", backwardCompatibility.getId()), @@ -1142,6 +1142,31 @@ public static void createLoginSignupGroups(BackwardCompatibility backwardCompati } } + public static void createRiskScoreApiGroup(int id, String name, RiskScoreTestingEndpoints.RiskScoreGroupType riskScoreGroupType) { + // ApiCollection riskScoreGroup = new ApiCollection(id, name, Context.now(), new HashSet<>(), null, 0, false, false); + // riskScoreGroup.setType(ApiCollection.Type.API_GROUP); + // ArrayList riskScoreConditions = new ArrayList<>(); + // riskScoreConditions.add(new RiskScoreTestingEndpoints(riskScoreGroupType)); + // riskScoreGroup.setConditions(riskScoreConditions); + + // ApiCollectionsDao.instance.insertOne(riskScoreGroup); + + ApiCollection apiCollection = ApiCollectionsDao.instance.findByName(name); + + } + + public static void createRiskScoreGroups(BackwardCompatibility backwardCompatibility) { + if (backwardCompatibility.getRiskScoreGroups() == 0) { + + createRiskScoreApiGroup(111_111_148, "Low Risk APIs", RiskScoreTestingEndpoints.RiskScoreGroupType.LOW); + + // BackwardCompatibilityDao.instance.updateOne( + // Filters.eq("_id", backwardCompatibility.getId()), + // Updates.set(BackwardCompatibility.RISK_SCORE_GROUPS, Context.now()) + // ); + } + } + public static void dropWorkflowTestResultCollection(BackwardCompatibility backwardCompatibility) { if (backwardCompatibility.getDropWorkflowTestResult() == 0) { WorkflowTestResultsDao.instance.getMCollection().drop(); @@ -1975,6 +2000,7 @@ public static void setBackwardCompatibilities(BackwardCompatibility backwardComp dropAuthMechanismData(backwardCompatibility); moveAuthMechanismDataToRole(backwardCompatibility); createLoginSignupGroups(backwardCompatibility); + createRiskScoreGroups(backwardCompatibility); deleteAccessListFromApiToken(backwardCompatibility); deleteNullSubCategoryIssues(backwardCompatibility); enableNewMerging(backwardCompatibility); diff --git a/libs/dao/src/main/java/com/akto/DaoInit.java b/libs/dao/src/main/java/com/akto/DaoInit.java index bf845565fe..0e58ff6bc7 100644 --- a/libs/dao/src/main/java/com/akto/DaoInit.java +++ b/libs/dao/src/main/java/com/akto/DaoInit.java @@ -243,6 +243,7 @@ public static CodecRegistry createCodecRegistry(){ ClassModel codeAnalysisApiLocationClassModel = ClassModel.builder(CodeAnalysisApiLocation.class).enableDiscriminator(true).build(); ClassModel codeAnalysisApiInfoClassModel = ClassModel.builder(CodeAnalysisApiInfo.class).enableDiscriminator(true).build(); ClassModel codeAnalysisApiInfoKeyClassModel = ClassModel.builder(CodeAnalysisApiInfo.CodeAnalysisApiInfoKey.class).enableDiscriminator(true).build(); + ClassModel riskScoreTestingEndpointsClassModel = ClassModel.builder(RiskScoreTestingEndpoints.class).enableDiscriminator(true).build(); CodecRegistry pojoCodecRegistry = fromProviders(PojoCodecProvider.builder().register( configClassModel, signupInfoClassModel, apiAuthClassModel, attempResultModel, urlTemplateModel, @@ -271,7 +272,8 @@ public static CodecRegistry createCodecRegistry(){ UsageMetricClassModel, UsageMetricInfoClassModel, UsageSyncClassModel, OrganizationClassModel, yamlNodeDetails, multiExecTestResultClassModel, workflowTestClassModel, dependencyNodeClassModel, paramInfoClassModel, nodeClassModel, connectionClassModel, edgeClassModel, replaceDetailClassModel, modifyHostDetailClassModel, fileUploadClassModel - ,fileUploadLogClassModel, codeAnalysisCollectionClassModel, codeAnalysisApiLocationClassModel, codeAnalysisApiInfoClassModel, codeAnalysisApiInfoKeyClassModel).automatic(true).build()); + ,fileUploadLogClassModel, codeAnalysisCollectionClassModel, codeAnalysisApiLocationClassModel, codeAnalysisApiInfoClassModel, codeAnalysisApiInfoKeyClassModel, + riskScoreTestingEndpointsClassModel).automatic(true).build()); final CodecRegistry customEnumCodecs = CodecRegistries.fromCodecs( new EnumCodec<>(Conditions.Operator.class), diff --git a/libs/dao/src/main/java/com/akto/dto/BackwardCompatibility.java b/libs/dao/src/main/java/com/akto/dto/BackwardCompatibility.java index 142c0534a0..5f31ff039a 100644 --- a/libs/dao/src/main/java/com/akto/dto/BackwardCompatibility.java +++ b/libs/dao/src/main/java/com/akto/dto/BackwardCompatibility.java @@ -66,13 +66,16 @@ public class BackwardCompatibility { public static final String VULNERABLE_API_UPDATION_VERSION_V1 = "vulnerableApiUpdationVersionV1"; private int vulnerableApiUpdationVersionV1; + public static final String RISK_SCORE_GROUPS = "riskScoreGroups"; + private int riskScoreGroups; + public BackwardCompatibility(int id, int dropFilterSampleData, int resetSingleTypeInfoCount, int dropWorkflowTestResult, int readyForNewTestingFramework,int addAktoDataTypes, boolean deploymentStatusUpdated, int authMechanismData, boolean mirroringLambdaTriggered, int deleteAccessListFromApiToken, int deleteNullSubCategoryIssues, int enableNewMerging, int aktoDefaultNewUI, int initializeOrganizationAccountBelongsTo, int orgsInBilling, int computeIntegratedConnections, int deleteLastCronRunInfo, int moveAuthMechanismToRole, - int loginSignupGroups, int vulnerableApiUpdationVersionV1) { + int loginSignupGroups, int vulnerableApiUpdationVersionV1, int riskScoreGroups) { this.id = id; this.dropFilterSampleData = dropFilterSampleData; this.resetSingleTypeInfoCount = resetSingleTypeInfoCount; @@ -94,6 +97,7 @@ public BackwardCompatibility(int id, int dropFilterSampleData, int resetSingleTy this.moveAuthMechanismToRole = moveAuthMechanismToRole; this.loginSignupGroups = loginSignupGroups; this.vulnerableApiUpdationVersionV1 = vulnerableApiUpdationVersionV1; + this.riskScoreGroups = riskScoreGroups; } public BackwardCompatibility() { @@ -282,4 +286,12 @@ public int getVulnerableApiUpdationVersionV1() { public void setVulnerableApiUpdationVersionV1(int vulnerableApiUpdationVersionV1) { this.vulnerableApiUpdationVersionV1 = vulnerableApiUpdationVersionV1; } + + public int getRiskScoreGroups() { + return riskScoreGroups; + } + + public void setRiskScoreGroups(int riskScoreGroups) { + this.riskScoreGroups = riskScoreGroups; + } } diff --git a/libs/dao/src/main/java/com/akto/dto/testing/RiskScoreTestingEndpoints.java b/libs/dao/src/main/java/com/akto/dto/testing/RiskScoreTestingEndpoints.java new file mode 100644 index 0000000000..9e9622a53b --- /dev/null +++ b/libs/dao/src/main/java/com/akto/dto/testing/RiskScoreTestingEndpoints.java @@ -0,0 +1,87 @@ +package com.akto.dto.testing; + +import com.akto.dao.MCollection; +import com.akto.dto.ApiCollectionUsers; +import com.akto.dto.ApiInfo; +import com.akto.dto.type.SingleTypeInfo; +import com.mongodb.client.model.Filters; +import org.apache.commons.lang3.NotImplementedException; +import org.bson.codecs.pojo.annotations.BsonIgnore; +import org.bson.conversions.Bson; + +import java.util.List; +import java.util.regex.Pattern; + +public class RiskScoreTestingEndpoints extends TestingEndpoints { + + private RiskScoreGroupType riskScoreGroupType; + private int riskScoreGroupLowerBound; + private int riskScoreGroupUpperBound; + + public enum RiskScoreGroupType { + LOW, MEDIUM, HIGH + } + + public RiskScoreTestingEndpoints() { + super(Type.RISK_SCORE); + } + + public RiskScoreTestingEndpoints(RiskScoreGroupType riskScoreGroupType) { + super(Type.RISK_SCORE); + this.riskScoreGroupType = riskScoreGroupType; + + switch (riskScoreGroupType) { + case LOW: + riskScoreGroupLowerBound = 0; + riskScoreGroupUpperBound = 3; + break; + case MEDIUM: + riskScoreGroupLowerBound = 3; + riskScoreGroupUpperBound = 4; + break; + case HIGH: + riskScoreGroupLowerBound = 4; + riskScoreGroupUpperBound = 5; + break; + } + } + + @Override + public List returnApis() { + throw new NotImplementedException(); + } + + @Override + public boolean containsApi(ApiInfo.ApiInfoKey key) { + throw new NotImplementedException(); + } + + @Override + public Bson createFilters(ApiCollectionUsers.CollectionType type) { + return MCollection.noMatchFilter; + } + + public RiskScoreGroupType getRiskScoreGroupType() { + return riskScoreGroupType; + } + + public void setRiskScoreGroupType(RiskScoreGroupType riskScoreGroupType) { + this.riskScoreGroupType = riskScoreGroupType; + } + + public int getRiskScoreGroupLowerBound() { + return riskScoreGroupLowerBound; + } + + public void setRiskScoreGroupLowerBound(int riskScoreGroupLowerBound) { + this.riskScoreGroupLowerBound = riskScoreGroupLowerBound; + } + + public int getRiskScoreGroupUpperBound() { + return riskScoreGroupUpperBound; + } + + public void setRiskScoreGroupUpperBound(int riskScoreGroupUpperBound) { + this.riskScoreGroupUpperBound = riskScoreGroupUpperBound; + } +} diff --git a/libs/dao/src/main/java/com/akto/dto/testing/TestingEndpoints.java b/libs/dao/src/main/java/com/akto/dto/testing/TestingEndpoints.java index c4cc38dcb3..70b4f954a2 100644 --- a/libs/dao/src/main/java/com/akto/dto/testing/TestingEndpoints.java +++ b/libs/dao/src/main/java/com/akto/dto/testing/TestingEndpoints.java @@ -38,7 +38,7 @@ public enum Operator { public enum Type { - CUSTOM, COLLECTION_WISE, WORKFLOW, LOGICAL_GROUP, METHOD, ALL, REGEX + CUSTOM, COLLECTION_WISE, WORKFLOW, LOGICAL_GROUP, METHOD, ALL, REGEX, RISK_SCORE } public Type getType() { From 5f2d979de242c12b288829f7a39c538a3f8e2ffe Mon Sep 17 00:00:00 2001 From: Oren Saldanha Date: Sat, 11 May 2024 14:50:17 +0530 Subject: [PATCH 02/11] added risk score based api groups --- .../com/akto/action/ApiCollectionsAction.java | 2 +- .../akto/listener/InitializerListener.java | 85 +++++++++++--- .../akto/utils/RiskScoreOfCollections.java | 25 ++++- .../utils/RiskScoreTestingEndpointsUtils.java | 95 ++++++++++++++++ .../java/com/akto/dto/ApiCollectionUsers.java | 11 +- .../testing/RiskScoreTestingEndpoints.java | 105 +++++++++++++++--- .../akto/dto/testing/TestingEndpoints.java | 9 ++ 7 files changed, 298 insertions(+), 34 deletions(-) create mode 100644 apps/dashboard/src/main/java/com/akto/utils/RiskScoreTestingEndpointsUtils.java diff --git a/apps/dashboard/src/main/java/com/akto/action/ApiCollectionsAction.java b/apps/dashboard/src/main/java/com/akto/action/ApiCollectionsAction.java index 9b4453d408..765bd4a328 100644 --- a/apps/dashboard/src/main/java/com/akto/action/ApiCollectionsAction.java +++ b/apps/dashboard/src/main/java/com/akto/action/ApiCollectionsAction.java @@ -281,7 +281,7 @@ public String removeApisFromCustomCollection(){ CustomTestingEndpoints condition = new CustomTestingEndpoints(apiList, CustomTestingEndpoints.Operator.OR); apiCollection.removeFromConditions(condition); ApiCollectionUsers.updateApiCollection(apiCollection.getConditions(), apiCollection.getId()); - ApiCollectionUsers.removeFromCollectionsForCollectionId(apiCollection.getConditions(), apiCollection.getId()); + ApiCollectionUsers.removeFromCollectionsForCollectionId(apiCollection.getConditions(), apiCollection.getId(), false); fetchAllCollections(); diff --git a/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java b/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java index 7fda650ae0..8343e759ad 100644 --- a/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java +++ b/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java @@ -1121,7 +1121,7 @@ public static void createLoginApiGroup(int id, String name, String regex) { ArrayList loginConditions = new ArrayList<>(); loginConditions.add(new RegexTestingEndpoints(TestingEndpoints.Operator.OR, regex)); loginGroup.setConditions(loginConditions); - ApiCollectionUsers.removeFromCollectionsForCollectionId(loginGroup.getConditions(), id); + ApiCollectionUsers.removeFromCollectionsForCollectionId(loginGroup.getConditions(), id, false); ApiCollectionsDao.instance.insertOne(loginGroup); ApiCollectionUsers.addToCollectionsForCollectionId(loginGroup.getConditions(), loginGroup.getId()); @@ -1143,27 +1143,67 @@ public static void createLoginSignupGroups(BackwardCompatibility backwardCompati } public static void createRiskScoreApiGroup(int id, String name, RiskScoreTestingEndpoints.RiskScoreGroupType riskScoreGroupType) { - // ApiCollection riskScoreGroup = new ApiCollection(id, name, Context.now(), new HashSet<>(), null, 0, false, false); - // riskScoreGroup.setType(ApiCollection.Type.API_GROUP); - // ArrayList riskScoreConditions = new ArrayList<>(); - // riskScoreConditions.add(new RiskScoreTestingEndpoints(riskScoreGroupType)); - // riskScoreGroup.setConditions(riskScoreConditions); + loggerMaker.infoAndAddToDb("Creating risk score group: " + name, LogDb.DASHBOARD); + + ApiCollection riskScoreGroup = ApiCollectionsDao.instance.findByName(name); + + if (riskScoreGroup == null) { + riskScoreGroup = new ApiCollection(id, name, Context.now(), new HashSet<>(), null, 0, false, false); + + List riskScoreConditions = new ArrayList<>(); + RiskScoreTestingEndpoints riskScoreTestingEndpoints = new RiskScoreTestingEndpoints(riskScoreGroupType); + riskScoreConditions.add(riskScoreTestingEndpoints); - // ApiCollectionsDao.instance.insertOne(riskScoreGroup); + riskScoreGroup.setConditions(riskScoreConditions); + riskScoreGroup.setType(ApiCollection.Type.API_GROUP); - ApiCollection apiCollection = ApiCollectionsDao.instance.findByName(name); + ApiCollectionsDao.instance.insertOne(riskScoreGroup); + } + List riskScoreConditions = riskScoreGroup.getConditions(); + RiskScoreTestingEndpoints riskScoreTestingEndpoints = (RiskScoreTestingEndpoints) riskScoreConditions.get(0); + + // Add APIs to the risk score group + List riskScoreGroupApisList = riskScoreTestingEndpoints.fetchRiskScoreGroupApis(); + + + // Skip API's wwhich have already been added to the risk score group + Iterator riskScoreGroupApisListIt = riskScoreGroupApisList.iterator(); + while (riskScoreGroupApisListIt.hasNext()) { + ApiInfo riskScoreGroupApi = riskScoreGroupApisListIt.next(); + List collectionIds = riskScoreGroupApi.getCollectionIds(); + + if (collectionIds.contains(id)) { + riskScoreGroupApisListIt.remove(); + } + } + + loggerMaker.infoAndAddToDb("Adding " + riskScoreGroupApisList.size() + " API's to API group - " + name , LogDb.DASHBOARD); + for (int start = 0; start < riskScoreGroupApisList.size(); start += RiskScoreTestingEndpoints.BATCH_SIZE) { + int end = Math.min(start + RiskScoreTestingEndpoints.BATCH_SIZE, riskScoreGroupApisList.size()); + + List batch = riskScoreGroupApisList.subList(start, end); + + riskScoreTestingEndpoints.setFilterRiskScoreGroupApis(batch); + ApiCollectionUsers.addToCollectionsForCollectionId(riskScoreConditions, id); + } } public static void createRiskScoreGroups(BackwardCompatibility backwardCompatibility) { if (backwardCompatibility.getRiskScoreGroups() == 0) { + try { + createRiskScoreApiGroup(111_111_148, "Low Risk APIs", RiskScoreTestingEndpoints.RiskScoreGroupType.LOW); + createRiskScoreApiGroup(111_111_149, "Medium Risk APIs", RiskScoreTestingEndpoints.RiskScoreGroupType.MEDIUM); + createRiskScoreApiGroup(111_111_150, "High Risk APIs", RiskScoreTestingEndpoints.RiskScoreGroupType.HIGH); - createRiskScoreApiGroup(111_111_148, "Low Risk APIs", RiskScoreTestingEndpoints.RiskScoreGroupType.LOW); - - // BackwardCompatibilityDao.instance.updateOne( - // Filters.eq("_id", backwardCompatibility.getId()), - // Updates.set(BackwardCompatibility.RISK_SCORE_GROUPS, Context.now()) - // ); + BackwardCompatibilityDao.instance.updateOne( + Filters.eq("_id", backwardCompatibility.getId()), + Updates.set(BackwardCompatibility.RISK_SCORE_GROUPS, Context.now()) + ); + } + catch (Exception e) { + loggerMaker.errorAndAddToDb(e, "Error while creating risk score groups: " + e.getMessage(), LogDb.DASHBOARD); + } } } @@ -1555,7 +1595,22 @@ public void updateCustomCollections() { List apiCollections = ApiCollectionsDao.instance.findAll(new BasicDBObject()); for (ApiCollection apiCollection : apiCollections) { if (ApiCollection.Type.API_GROUP.equals(apiCollection.getType())) { - ApiCollectionUsers.computeCollectionsForCollectionId(apiCollection.getConditions(), apiCollection.getId()); + List conditions = apiCollection.getConditions(); + + // Don't update API groups that are delta update based + boolean isDeltaUpdateBasedApiGroup = false; + for (TestingEndpoints testingEndpoints : conditions) { + if (TestingEndpoints.checkDeltaUpdateBased(testingEndpoints.getType())) { + isDeltaUpdateBasedApiGroup = true; + break; + } + } + + if (isDeltaUpdateBasedApiGroup) { + continue; + } + + ApiCollectionUsers.computeCollectionsForCollectionId(conditions, apiCollection.getId()); } } } diff --git a/apps/dashboard/src/main/java/com/akto/utils/RiskScoreOfCollections.java b/apps/dashboard/src/main/java/com/akto/utils/RiskScoreOfCollections.java index e370cc07cf..bc240e571b 100644 --- a/apps/dashboard/src/main/java/com/akto/utils/RiskScoreOfCollections.java +++ b/apps/dashboard/src/main/java/com/akto/utils/RiskScoreOfCollections.java @@ -17,10 +17,14 @@ import com.akto.dto.AccountSettings; import com.akto.dto.ApiInfo; import com.akto.dto.AktoDataType; +import com.akto.dto.ApiCollectionUsers; import com.akto.dto.ApiInfo.ApiInfoKey; import com.akto.dto.CustomDataType; import com.akto.dto.test_run_findings.TestingIssuesId; import com.akto.dto.test_run_findings.TestingRunIssues; +import com.akto.dto.testing.RiskScoreTestingEndpoints; +import com.akto.dto.testing.TestingEndpoints; +import com.akto.dto.testing.RiskScoreTestingEndpoints.RiskScoreGroupType; import com.akto.dto.type.SingleTypeInfo; import com.akto.log.LoggerMaker; import com.akto.log.LoggerMaker.LogDb; @@ -140,7 +144,9 @@ public void updateSeverityScoreInApiInfo(int timeStampFilter){ return ; } - Map severityScoreMap = getSeverityScoreMap(updatedIssues); + Map severityScoreMap = getSeverityScoreMap(updatedIssues); + + RiskScoreTestingEndpointsUtils riskScoreTestingEndpointsUtils = new RiskScoreTestingEndpointsUtils(); // after getting the severityScoreMap, we write that in DB if(severityScoreMap != null){ @@ -149,6 +155,11 @@ public void updateSeverityScoreInApiInfo(int timeStampFilter){ ApiInfo apiInfo = ApiInfoDao.instance.findOne(filter); boolean isSensitive = apiInfo != null ? apiInfo.getIsSensitive() : false; float riskScore = ApiInfoDao.getRiskScore(apiInfo, isSensitive, Utils.getRiskScoreValueFromSeverityScore(severityScore)); + + if (apiInfo.getRiskScore() != riskScore) { + riskScoreTestingEndpointsUtils.updateApiRiskScoreGroup(apiInfo, riskScore); + } + Bson update = Updates.combine( Updates.set(ApiInfo.SEVERITY_SCORE, severityScore), @@ -161,7 +172,8 @@ public void updateSeverityScoreInApiInfo(int timeStampFilter){ if (bulkUpdatesForApiInfo.size() > 0) { ApiInfoDao.instance.getMCollection().bulkWrite(bulkUpdatesForApiInfo, new BulkWriteOptions().ordered(false)); } - + + riskScoreTestingEndpointsUtils.syncRiskScoreGroupApis(); } private static void writeUpdatesForSensitiveInfoInApiInfo(List updatedDataTypes, int timeStampFilter){ @@ -258,6 +270,9 @@ public void calculateRiskScoreForAllApis() { Filters.lte(ApiInfo.LAST_CALCULATED_TIME, timeStamp) ); Bson projection = Projections.include("_id", ApiInfo.API_ACCESS_TYPES, ApiInfo.LAST_SEEN, ApiInfo.SEVERITY_SCORE, ApiInfo.IS_SENSITIVE); + + RiskScoreTestingEndpointsUtils riskScoreTestingEndpointsUtils = new RiskScoreTestingEndpointsUtils(); + while(count < 100){ List apiInfos = ApiInfoDao.instance.findAll(filter,0, limit, Sorts.descending(ApiInfo.LAST_CALCULATED_TIME), projection); for(ApiInfo apiInfo: apiInfos){ @@ -269,6 +284,10 @@ public void calculateRiskScoreForAllApis() { Bson filterQ = ApiInfoDao.getFilter(apiInfo.getId()); bulkUpdates.add(new UpdateManyModel<>(filterQ, update, new UpdateOptions().upsert(false))); + + if (apiInfo.getRiskScore() != riskScore) { + riskScoreTestingEndpointsUtils.updateApiRiskScoreGroup(apiInfo, riskScore); + } } if(bulkUpdates.size() > 0){ ApiInfoDao.instance.bulkWrite(bulkUpdates, new BulkWriteOptions().ordered(false)); @@ -279,5 +298,7 @@ public void calculateRiskScoreForAllApis() { break; } } + + riskScoreTestingEndpointsUtils.syncRiskScoreGroupApis(); } } diff --git a/apps/dashboard/src/main/java/com/akto/utils/RiskScoreTestingEndpointsUtils.java b/apps/dashboard/src/main/java/com/akto/utils/RiskScoreTestingEndpointsUtils.java new file mode 100644 index 0000000000..625fad1608 --- /dev/null +++ b/apps/dashboard/src/main/java/com/akto/utils/RiskScoreTestingEndpointsUtils.java @@ -0,0 +1,95 @@ +package com.akto.utils; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +import com.akto.dao.context.Context; +import com.akto.dto.ApiCollectionUsers; +import com.akto.dto.ApiInfo; +import com.akto.dto.testing.RiskScoreTestingEndpoints; +import com.akto.dto.testing.TestingEndpoints; +import com.akto.log.LoggerMaker; +import com.akto.log.LoggerMaker.LogDb; + +public class RiskScoreTestingEndpointsUtils { + private static final LoggerMaker loggerMaker = new LoggerMaker(RiskScoreTestingEndpointsUtils.class); + + private Map> removeApisFromRiskScoreGroupMap = new HashMap>() {{ + put(RiskScoreTestingEndpoints.RiskScoreGroupType.LOW, new ArrayList<>()); + put(RiskScoreTestingEndpoints.RiskScoreGroupType.MEDIUM, new ArrayList<>()); + put(RiskScoreTestingEndpoints.RiskScoreGroupType.HIGH, new ArrayList<>()); + }}; + + + private Map> addApisToRiskScoreGroupMap = new HashMap>() {{ + put(RiskScoreTestingEndpoints.RiskScoreGroupType.LOW, new ArrayList<>()); + put(RiskScoreTestingEndpoints.RiskScoreGroupType.MEDIUM, new ArrayList<>()); + put(RiskScoreTestingEndpoints.RiskScoreGroupType.HIGH, new ArrayList<>()); + }}; + + private static final ScheduledExecutorService executorService = Executors.newSingleThreadScheduledExecutor(); + + public RiskScoreTestingEndpointsUtils() { + } + + public void updateApiRiskScoreGroup(ApiInfo apiInfo, float updatedRiskScore) { + float oldRiskScore = apiInfo.getRiskScore(); + + RiskScoreTestingEndpoints.RiskScoreGroupType removeRiskScoreGroupType = RiskScoreTestingEndpoints.calculateRiskScoreGroup(oldRiskScore); + removeApisFromRiskScoreGroupMap.get(removeRiskScoreGroupType).add(apiInfo); + + RiskScoreTestingEndpoints.RiskScoreGroupType addRiskScoreGroupType = RiskScoreTestingEndpoints.calculateRiskScoreGroup(updatedRiskScore); + addApisToRiskScoreGroupMap.get(addRiskScoreGroupType).add(apiInfo); + } + + private void updateRiskScoreApiGroups() { + try { + for(RiskScoreTestingEndpoints.RiskScoreGroupType riskScoreGroupType: RiskScoreTestingEndpoints.RiskScoreGroupType.values()) { + RiskScoreTestingEndpoints riskScoreTestingEndpoints = new RiskScoreTestingEndpoints(riskScoreGroupType); + + List testingEndpoints = new ArrayList<>(); + testingEndpoints.add(riskScoreTestingEndpoints); + int apiCollectionId = RiskScoreTestingEndpoints.getApiCollectionId(riskScoreGroupType); + + // Remove APIs from the original risk score group + List removeApisFromRiskScoreGroupList = removeApisFromRiskScoreGroupMap.get(riskScoreGroupType); + for (int start = 0; start < removeApisFromRiskScoreGroupList.size(); start += RiskScoreTestingEndpoints.BATCH_SIZE) { + int end = Math.min(start + RiskScoreTestingEndpoints.BATCH_SIZE, removeApisFromRiskScoreGroupList.size()); + + List batch = removeApisFromRiskScoreGroupList.subList(start, end); + + riskScoreTestingEndpoints.setFilterRiskScoreGroupApis(batch); + ApiCollectionUsers.removeFromCollectionsForCollectionId(testingEndpoints, apiCollectionId, true); + } + + // Add APIs to the new risk score group + List addApisToRiskScoreGroupList = addApisToRiskScoreGroupMap.get(riskScoreGroupType); + for (int start = 0; start < addApisToRiskScoreGroupList.size(); start += RiskScoreTestingEndpoints.BATCH_SIZE) { + int end = Math.min(start + RiskScoreTestingEndpoints.BATCH_SIZE, addApisToRiskScoreGroupList.size()); + + List batch = addApisToRiskScoreGroupList.subList(start, end); + + riskScoreTestingEndpoints.setFilterRiskScoreGroupApis(batch); + ApiCollectionUsers.addToCollectionsForCollectionId(testingEndpoints, apiCollectionId); + } + } + } catch (Exception e) { + loggerMaker.errorAndAddToDb("Error updating risk score group APIs - " + e.getMessage(), LogDb.DASHBOARD); + } + } + + public void syncRiskScoreGroupApis() { + int accountId = Context.accountId.get(); + executorService.schedule( new Runnable() { + public void run() { + Context.accountId.set(accountId); + updateRiskScoreApiGroups(); + } + }, 0, TimeUnit.SECONDS); + } +} \ No newline at end of file diff --git a/libs/dao/src/main/java/com/akto/dto/ApiCollectionUsers.java b/libs/dao/src/main/java/com/akto/dto/ApiCollectionUsers.java index 03111bea85..da45ddde97 100644 --- a/libs/dao/src/main/java/com/akto/dto/ApiCollectionUsers.java +++ b/libs/dao/src/main/java/com/akto/dto/ApiCollectionUsers.java @@ -119,15 +119,20 @@ public static void addToCollectionsForCollectionId(List condit operationForCollectionId(conditions, apiCollectionId, update, matchFilter, false); } - public static void removeFromCollectionsForCollectionId(List conditions, int apiCollectionId) { + public static void removeFromCollectionsForCollectionId(List conditions, int apiCollectionId, boolean matchExactCondition) { Bson update = Updates.pull(SingleTypeInfo._COLLECTION_IDS, apiCollectionId); Bson matchFilter = Filters.in(SingleTypeInfo._COLLECTION_IDS, apiCollectionId); - operationForCollectionId(conditions, apiCollectionId, update, matchFilter, true); + + if (matchExactCondition) { + operationForCollectionId(conditions, apiCollectionId, update, matchFilter, false); + } else { + operationForCollectionId(conditions, apiCollectionId, update, matchFilter, true); + } } public static void computeCollectionsForCollectionId(List conditions, int apiCollectionId) { addToCollectionsForCollectionId(conditions, apiCollectionId); - removeFromCollectionsForCollectionId(conditions, apiCollectionId); + removeFromCollectionsForCollectionId(conditions, apiCollectionId, false); updateApiCollection(conditions, apiCollectionId); } diff --git a/libs/dao/src/main/java/com/akto/dto/testing/RiskScoreTestingEndpoints.java b/libs/dao/src/main/java/com/akto/dto/testing/RiskScoreTestingEndpoints.java index 9e9622a53b..17c9ad56c1 100644 --- a/libs/dao/src/main/java/com/akto/dto/testing/RiskScoreTestingEndpoints.java +++ b/libs/dao/src/main/java/com/akto/dto/testing/RiskScoreTestingEndpoints.java @@ -1,22 +1,35 @@ package com.akto.dto.testing; +import com.akto.dao.ApiInfoDao; import com.akto.dao.MCollection; import com.akto.dto.ApiCollectionUsers; +import com.akto.dto.ApiCollectionUsers.CollectionType; import com.akto.dto.ApiInfo; +import com.akto.dto.ApiInfo.ApiInfoKey; import com.akto.dto.type.SingleTypeInfo; import com.mongodb.client.model.Filters; +import com.mongodb.client.model.Projections; + import org.apache.commons.lang3.NotImplementedException; import org.bson.codecs.pojo.annotations.BsonIgnore; import org.bson.conversions.Bson; +import java.util.ArrayList; +import java.util.HashSet; import java.util.List; +import java.util.Set; import java.util.regex.Pattern; public class RiskScoreTestingEndpoints extends TestingEndpoints { private RiskScoreGroupType riskScoreGroupType; - private int riskScoreGroupLowerBound; - private int riskScoreGroupUpperBound; + private float riskScoreGroupLowerBound; + private float riskScoreGroupUpperBound; + + @BsonIgnore + private List filterRiskScoreGroupApis; + + public static int BATCH_SIZE = 100; public enum RiskScoreGroupType { LOW, MEDIUM, HIGH @@ -24,6 +37,7 @@ public enum RiskScoreGroupType { public RiskScoreTestingEndpoints() { super(Type.RISK_SCORE); + this.filterRiskScoreGroupApis = new ArrayList(); } public RiskScoreTestingEndpoints(RiskScoreGroupType riskScoreGroupType) { @@ -32,18 +46,20 @@ public RiskScoreTestingEndpoints(RiskScoreGroupType riskScoreGroupType) { switch (riskScoreGroupType) { case LOW: - riskScoreGroupLowerBound = 0; - riskScoreGroupUpperBound = 3; + riskScoreGroupLowerBound = 0f; + riskScoreGroupUpperBound = 3f; break; case MEDIUM: - riskScoreGroupLowerBound = 3; - riskScoreGroupUpperBound = 4; + riskScoreGroupLowerBound = 3f; + riskScoreGroupUpperBound = 4f; break; case HIGH: - riskScoreGroupLowerBound = 4; - riskScoreGroupUpperBound = 5; + riskScoreGroupLowerBound = 4f; + riskScoreGroupUpperBound = 5f; break; } + + this.filterRiskScoreGroupApis = new ArrayList(); } @Override @@ -56,11 +72,66 @@ public boolean containsApi(ApiInfo.ApiInfoKey key) { throw new NotImplementedException(); } + private static Bson createApiFilters(CollectionType type, ApiInfo api) { + + String prefix = getFilterPrefix(type); + ApiInfoKey apiInfoKey = api.getId(); + + return Filters.and( + Filters.eq(prefix + SingleTypeInfo._URL, apiInfoKey.getUrl()), + Filters.eq(prefix + SingleTypeInfo._METHOD, apiInfoKey.getMethod().toString()), + Filters.in(SingleTypeInfo._COLLECTION_IDS, apiInfoKey.getApiCollectionId())); + + } + @Override - public Bson createFilters(ApiCollectionUsers.CollectionType type) { + public Bson createFilters(CollectionType type) { + Set apisSet = new HashSet<>(filterRiskScoreGroupApis); + List apiFilters = new ArrayList<>(); + if (apisSet != null && !apisSet.isEmpty()) { + for (ApiInfo api : apisSet) { + apiFilters.add(createApiFilters(type, api)); + } + return Filters.or(apiFilters); + } + return MCollection.noMatchFilter; } + public List fetchRiskScoreGroupApis() { + List riskScoreGroupApis = ApiInfoDao.instance.findAll( + Filters.and( + Filters.gte("riskScore", riskScoreGroupLowerBound), + Filters.lt("riskScore", riskScoreGroupUpperBound)) + ); + return riskScoreGroupApis; + } + + public static RiskScoreGroupType calculateRiskScoreGroup(float riskScore) { + if (riskScore >= 0f && riskScore < 3f) { + return RiskScoreGroupType.LOW; + } else if (riskScore >= 3f && riskScore < 4f) { + return RiskScoreGroupType.MEDIUM; + } else if (riskScore >= 4f && riskScore <= 5f) { + return RiskScoreGroupType.HIGH; + } + + throw new IllegalArgumentException("Risk score is out of expected range: " + riskScore); + } + + public static int getApiCollectionId(RiskScoreGroupType riskScoreGroupType) { + switch (riskScoreGroupType) { + case LOW: + return 111_111_148; + case MEDIUM: + return 111_111_149; + case HIGH: + return 111_111_150; + default: + return -1; + } + } + public RiskScoreGroupType getRiskScoreGroupType() { return riskScoreGroupType; } @@ -69,19 +140,27 @@ public void setRiskScoreGroupType(RiskScoreGroupType riskScoreGroupType) { this.riskScoreGroupType = riskScoreGroupType; } - public int getRiskScoreGroupLowerBound() { + public float getRiskScoreGroupLowerBound() { return riskScoreGroupLowerBound; } - public void setRiskScoreGroupLowerBound(int riskScoreGroupLowerBound) { + public void setRiskScoreGroupLowerBound(float riskScoreGroupLowerBound) { this.riskScoreGroupLowerBound = riskScoreGroupLowerBound; } - public int getRiskScoreGroupUpperBound() { + public float getRiskScoreGroupUpperBound() { return riskScoreGroupUpperBound; } - public void setRiskScoreGroupUpperBound(int riskScoreGroupUpperBound) { + public void setRiskScoreGroupUpperBound(float riskScoreGroupUpperBound) { this.riskScoreGroupUpperBound = riskScoreGroupUpperBound; } + + public List getFilterRiskScoreGroupApis() { + return filterRiskScoreGroupApis; + } + + public void setFilterRiskScoreGroupApis(List filterRiskScoreGroupApis) { + this.filterRiskScoreGroupApis = filterRiskScoreGroupApis; + } } diff --git a/libs/dao/src/main/java/com/akto/dto/testing/TestingEndpoints.java b/libs/dao/src/main/java/com/akto/dto/testing/TestingEndpoints.java index 70b4f954a2..d6e419dbb3 100644 --- a/libs/dao/src/main/java/com/akto/dto/testing/TestingEndpoints.java +++ b/libs/dao/src/main/java/com/akto/dto/testing/TestingEndpoints.java @@ -119,4 +119,13 @@ public Map returnFiltersMap() { } return filtersMap; } + + public static Boolean checkDeltaUpdateBased(Type type) { + switch (type) { + case RISK_SCORE: + return true; + default: + return false; + } + } } From 31379666d6065fcac23c2300f197a0d977591aa3 Mon Sep 17 00:00:00 2001 From: Oren Saldanha Date: Mon, 13 May 2024 15:43:06 +0530 Subject: [PATCH 03/11] update containsapi --- .../java/com/akto/dto/testing/RiskScoreTestingEndpoints.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/dao/src/main/java/com/akto/dto/testing/RiskScoreTestingEndpoints.java b/libs/dao/src/main/java/com/akto/dto/testing/RiskScoreTestingEndpoints.java index 17c9ad56c1..98cc850e1e 100644 --- a/libs/dao/src/main/java/com/akto/dto/testing/RiskScoreTestingEndpoints.java +++ b/libs/dao/src/main/java/com/akto/dto/testing/RiskScoreTestingEndpoints.java @@ -69,7 +69,7 @@ public List returnApis() { @Override public boolean containsApi(ApiInfo.ApiInfoKey key) { - throw new NotImplementedException(); + return false; } private static Bson createApiFilters(CollectionType type, ApiInfo api) { From 70cf0931130f81fbac3cdb6e328598038491ed64 Mon Sep 17 00:00:00 2001 From: Oren Saldanha Date: Tue, 14 May 2024 12:41:57 +0530 Subject: [PATCH 04/11] add logging and update logic --- .../java/com/akto/utils/RiskScoreOfCollections.java | 10 +++++++++- .../com/akto/utils/RiskScoreTestingEndpointsUtils.java | 3 +++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/apps/dashboard/src/main/java/com/akto/utils/RiskScoreOfCollections.java b/apps/dashboard/src/main/java/com/akto/utils/RiskScoreOfCollections.java index bc240e571b..0ea810467d 100644 --- a/apps/dashboard/src/main/java/com/akto/utils/RiskScoreOfCollections.java +++ b/apps/dashboard/src/main/java/com/akto/utils/RiskScoreOfCollections.java @@ -285,7 +285,15 @@ public void calculateRiskScoreForAllApis() { bulkUpdates.add(new UpdateManyModel<>(filterQ, update, new UpdateOptions().upsert(false))); - if (apiInfo.getRiskScore() != riskScore) { + List collectionIds = apiInfo.getCollectionIds(); + float oldRiskScore = apiInfo.getRiskScore(); + RiskScoreTestingEndpoints.RiskScoreGroupType oldRiskScoreGroupType = RiskScoreTestingEndpoints.calculateRiskScoreGroup(oldRiskScore); + int oldRiskScoreGroupCollectionId = RiskScoreTestingEndpoints.getApiCollectionId(oldRiskScoreGroupType); + + if (!collectionIds.contains(oldRiskScoreGroupCollectionId) && oldRiskScore == riskScore) { + // handle case when api has risk score 0 and the new risk score is also 0 + riskScoreTestingEndpointsUtils.updateApiRiskScoreGroup(apiInfo, riskScore); + } else if (oldRiskScore != riskScore) { riskScoreTestingEndpointsUtils.updateApiRiskScoreGroup(apiInfo, riskScore); } } diff --git a/apps/dashboard/src/main/java/com/akto/utils/RiskScoreTestingEndpointsUtils.java b/apps/dashboard/src/main/java/com/akto/utils/RiskScoreTestingEndpointsUtils.java index 625fad1608..5f43cbe5fe 100644 --- a/apps/dashboard/src/main/java/com/akto/utils/RiskScoreTestingEndpointsUtils.java +++ b/apps/dashboard/src/main/java/com/akto/utils/RiskScoreTestingEndpointsUtils.java @@ -48,6 +48,7 @@ public void updateApiRiskScoreGroup(ApiInfo apiInfo, float updatedRiskScore) { } private void updateRiskScoreApiGroups() { + loggerMaker.infoAndAddToDb("Updating risk score API groups", LogDb.DASHBOARD); try { for(RiskScoreTestingEndpoints.RiskScoreGroupType riskScoreGroupType: RiskScoreTestingEndpoints.RiskScoreGroupType.values()) { RiskScoreTestingEndpoints riskScoreTestingEndpoints = new RiskScoreTestingEndpoints(riskScoreGroupType); @@ -58,6 +59,7 @@ private void updateRiskScoreApiGroups() { // Remove APIs from the original risk score group List removeApisFromRiskScoreGroupList = removeApisFromRiskScoreGroupMap.get(riskScoreGroupType); + loggerMaker.infoAndAddToDb("Removing " + removeApisFromRiskScoreGroupList.size() + " APIs from risk score group - " + riskScoreGroupType, LogDb.DASHBOARD); for (int start = 0; start < removeApisFromRiskScoreGroupList.size(); start += RiskScoreTestingEndpoints.BATCH_SIZE) { int end = Math.min(start + RiskScoreTestingEndpoints.BATCH_SIZE, removeApisFromRiskScoreGroupList.size()); @@ -69,6 +71,7 @@ private void updateRiskScoreApiGroups() { // Add APIs to the new risk score group List addApisToRiskScoreGroupList = addApisToRiskScoreGroupMap.get(riskScoreGroupType); + loggerMaker.infoAndAddToDb("Adding " + addApisToRiskScoreGroupList.size() + " APIs to risk score group - " + riskScoreGroupType, LogDb.DASHBOARD); for (int start = 0; start < addApisToRiskScoreGroupList.size(); start += RiskScoreTestingEndpoints.BATCH_SIZE) { int end = Math.min(start + RiskScoreTestingEndpoints.BATCH_SIZE, addApisToRiskScoreGroupList.size()); From 3a90f75ab99942c91ab7d65bc8562e746f0a37e0 Mon Sep 17 00:00:00 2001 From: Oren Saldanha Date: Tue, 14 May 2024 12:48:00 +0530 Subject: [PATCH 05/11] update method name --- .../main/java/com/akto/listener/InitializerListener.java | 8 ++++---- .../src/main/java/com/akto/dto/ApiCollectionUsers.java | 1 + 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java b/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java index 78fb095b73..16a6c650c1 100644 --- a/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java +++ b/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java @@ -1122,7 +1122,7 @@ public static void moveAuthMechanismDataToRole(BackwardCompatibility backwardCom } } - public static void createLoginApiGroup(int id, String name, String regex) { + public static void createApiGroup(int id, String name, String regex) { ApiCollection loginGroup = new ApiCollection(id, name, Context.now(), new HashSet<>(), null, 0, false, false); loginGroup.setType(ApiCollection.Type.API_GROUP); ArrayList loginConditions = new ArrayList<>(); @@ -1137,9 +1137,9 @@ public static void createLoginApiGroup(int id, String name, String regex) { public static void createLoginSignupGroups(BackwardCompatibility backwardCompatibility) { if (backwardCompatibility.getLoginSignupGroups() == 0) { - createLoginApiGroup(111_111_128, "Login APIs", "^((https?):\\/\\/)?(www\\.)?.*?(login|signin|sign-in|authenticate|session)(.*?)(\\?.*|\\/?|#.*?)?$"); - createLoginApiGroup(111_111_129, "Signup APIs", "^((https?):\\/\\/)?(www\\.)?.*?(register|signup|sign-up|users\\/create|account\\/create|account_create|create_account)(.*?)(\\?.*|\\/?|#.*?)?$"); - createLoginApiGroup(111_111_130, "Password Reset APIs", "^((https?):\\/\\/)?(www\\.)?.*?(password-reset|reset-password|forgot-password|user\\/reset|account\\/recover|api\\/password_reset|password\\/reset|account\\/reset-password-request|password_reset_request|account_recovery)(.*?)(\\?.*|\\/?|#.*?)?$"); + createApiGroup(111_111_128, "Login APIs", "^((https?):\\/\\/)?(www\\.)?.*?(login|signin|sign-in|authenticate|session)(.*?)(\\?.*|\\/?|#.*?)?$"); + createApiGroup(111_111_129, "Signup APIs", "^((https?):\\/\\/)?(www\\.)?.*?(register|signup|sign-up|users\\/create|account\\/create|account_create|create_account)(.*?)(\\?.*|\\/?|#.*?)?$"); + createApiGroup(111_111_130, "Password Reset APIs", "^((https?):\\/\\/)?(www\\.)?.*?(password-reset|reset-password|forgot-password|user\\/reset|account\\/recover|api\\/password_reset|password\\/reset|account\\/reset-password-request|password_reset_request|account_recovery)(.*?)(\\?.*|\\/?|#.*?)?$"); BackwardCompatibilityDao.instance.updateOne( Filters.eq("_id", backwardCompatibility.getId()), diff --git a/libs/dao/src/main/java/com/akto/dto/ApiCollectionUsers.java b/libs/dao/src/main/java/com/akto/dto/ApiCollectionUsers.java index da45ddde97..ef8c451ef0 100644 --- a/libs/dao/src/main/java/com/akto/dto/ApiCollectionUsers.java +++ b/libs/dao/src/main/java/com/akto/dto/ApiCollectionUsers.java @@ -124,6 +124,7 @@ public static void removeFromCollectionsForCollectionId(List c Bson matchFilter = Filters.in(SingleTypeInfo._COLLECTION_IDS, apiCollectionId); if (matchExactCondition) { + // Remove collection ids for exact match of filter operationForCollectionId(conditions, apiCollectionId, update, matchFilter, false); } else { operationForCollectionId(conditions, apiCollectionId, update, matchFilter, true); From cbf73ce4f4fadeee211a54ffbb5a43a831cd3c26 Mon Sep 17 00:00:00 2001 From: Oren Saldanha Date: Tue, 14 May 2024 14:35:18 +0530 Subject: [PATCH 06/11] add null check --- .../main/java/com/akto/utils/RiskScoreOfCollections.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/apps/dashboard/src/main/java/com/akto/utils/RiskScoreOfCollections.java b/apps/dashboard/src/main/java/com/akto/utils/RiskScoreOfCollections.java index 0ea810467d..24cc4e5aeb 100644 --- a/apps/dashboard/src/main/java/com/akto/utils/RiskScoreOfCollections.java +++ b/apps/dashboard/src/main/java/com/akto/utils/RiskScoreOfCollections.java @@ -156,11 +156,12 @@ public void updateSeverityScoreInApiInfo(int timeStampFilter){ boolean isSensitive = apiInfo != null ? apiInfo.getIsSensitive() : false; float riskScore = ApiInfoDao.getRiskScore(apiInfo, isSensitive, Utils.getRiskScoreValueFromSeverityScore(severityScore)); - if (apiInfo.getRiskScore() != riskScore) { - riskScoreTestingEndpointsUtils.updateApiRiskScoreGroup(apiInfo, riskScore); + if (apiInfo != null) { + if (apiInfo.getRiskScore() != riskScore) { + riskScoreTestingEndpointsUtils.updateApiRiskScoreGroup(apiInfo, riskScore); + } } - Bson update = Updates.combine( Updates.set(ApiInfo.SEVERITY_SCORE, severityScore), Updates.set(ApiInfo.RISK_SCORE, riskScore) From 7f5ae9ddfed5c54311f2765a0ed85b9f529dc232 Mon Sep 17 00:00:00 2001 From: Oren Saldanha Date: Wed, 15 May 2024 12:05:22 +0530 Subject: [PATCH 07/11] remove match condition, use thread pool --- .../com/akto/action/ApiCollectionsAction.java | 2 +- .../com/akto/listener/InitializerListener.java | 2 +- .../com/akto/utils/RiskScoreOfCollections.java | 7 ++++--- .../utils/RiskScoreTestingEndpointsUtils.java | 18 +++++++++--------- .../java/com/akto/dto/ApiCollectionUsers.java | 17 +++++++++++++---- 5 files changed, 28 insertions(+), 18 deletions(-) diff --git a/apps/dashboard/src/main/java/com/akto/action/ApiCollectionsAction.java b/apps/dashboard/src/main/java/com/akto/action/ApiCollectionsAction.java index 7da6d54077..645fd4bac9 100644 --- a/apps/dashboard/src/main/java/com/akto/action/ApiCollectionsAction.java +++ b/apps/dashboard/src/main/java/com/akto/action/ApiCollectionsAction.java @@ -281,7 +281,7 @@ public String removeApisFromCustomCollection(){ CustomTestingEndpoints condition = new CustomTestingEndpoints(apiList, CustomTestingEndpoints.Operator.OR); apiCollection.removeFromConditions(condition); ApiCollectionUsers.updateApiCollection(apiCollection.getConditions(), apiCollection.getId()); - ApiCollectionUsers.removeFromCollectionsForCollectionId(apiCollection.getConditions(), apiCollection.getId(), false); + ApiCollectionUsers.removeFromCollectionsForCollectionId(apiCollection.getConditions(), apiCollection.getId()); fetchAllCollections(); diff --git a/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java b/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java index 16a6c650c1..b2c1db4c06 100644 --- a/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java +++ b/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java @@ -1128,7 +1128,7 @@ public static void createApiGroup(int id, String name, String regex) { ArrayList loginConditions = new ArrayList<>(); loginConditions.add(new RegexTestingEndpoints(TestingEndpoints.Operator.OR, regex)); loginGroup.setConditions(loginConditions); - ApiCollectionUsers.removeFromCollectionsForCollectionId(loginGroup.getConditions(), id, false); + ApiCollectionUsers.removeFromCollectionsForCollectionId(loginGroup.getConditions(), id); ApiCollectionsDao.instance.insertOne(loginGroup); ApiCollectionUsers.addToCollectionsForCollectionId(loginGroup.getConditions(), loginGroup.getId()); diff --git a/apps/dashboard/src/main/java/com/akto/utils/RiskScoreOfCollections.java b/apps/dashboard/src/main/java/com/akto/utils/RiskScoreOfCollections.java index 24cc4e5aeb..bc329100b2 100644 --- a/apps/dashboard/src/main/java/com/akto/utils/RiskScoreOfCollections.java +++ b/apps/dashboard/src/main/java/com/akto/utils/RiskScoreOfCollections.java @@ -290,11 +290,12 @@ public void calculateRiskScoreForAllApis() { float oldRiskScore = apiInfo.getRiskScore(); RiskScoreTestingEndpoints.RiskScoreGroupType oldRiskScoreGroupType = RiskScoreTestingEndpoints.calculateRiskScoreGroup(oldRiskScore); int oldRiskScoreGroupCollectionId = RiskScoreTestingEndpoints.getApiCollectionId(oldRiskScoreGroupType); - - if (!collectionIds.contains(oldRiskScoreGroupCollectionId) && oldRiskScore == riskScore) { - // handle case when api has risk score 0 and the new risk score is also 0 + + if (!collectionIds.contains(oldRiskScoreGroupCollectionId)) { + // Add API to risk score API group if it is not already added riskScoreTestingEndpointsUtils.updateApiRiskScoreGroup(apiInfo, riskScore); } else if (oldRiskScore != riskScore) { + // Update API in risk score API group if risk score has changed riskScoreTestingEndpointsUtils.updateApiRiskScoreGroup(apiInfo, riskScore); } } diff --git a/apps/dashboard/src/main/java/com/akto/utils/RiskScoreTestingEndpointsUtils.java b/apps/dashboard/src/main/java/com/akto/utils/RiskScoreTestingEndpointsUtils.java index 5f43cbe5fe..0e31ffd1eb 100644 --- a/apps/dashboard/src/main/java/com/akto/utils/RiskScoreTestingEndpointsUtils.java +++ b/apps/dashboard/src/main/java/com/akto/utils/RiskScoreTestingEndpointsUtils.java @@ -4,6 +4,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; @@ -32,7 +33,7 @@ public class RiskScoreTestingEndpointsUtils { put(RiskScoreTestingEndpoints.RiskScoreGroupType.HIGH, new ArrayList<>()); }}; - private static final ScheduledExecutorService executorService = Executors.newSingleThreadScheduledExecutor(); + private static final ExecutorService executorService = Executors.newFixedThreadPool(1); public RiskScoreTestingEndpointsUtils() { } @@ -48,7 +49,6 @@ public void updateApiRiskScoreGroup(ApiInfo apiInfo, float updatedRiskScore) { } private void updateRiskScoreApiGroups() { - loggerMaker.infoAndAddToDb("Updating risk score API groups", LogDb.DASHBOARD); try { for(RiskScoreTestingEndpoints.RiskScoreGroupType riskScoreGroupType: RiskScoreTestingEndpoints.RiskScoreGroupType.values()) { RiskScoreTestingEndpoints riskScoreTestingEndpoints = new RiskScoreTestingEndpoints(riskScoreGroupType); @@ -66,7 +66,7 @@ private void updateRiskScoreApiGroups() { List batch = removeApisFromRiskScoreGroupList.subList(start, end); riskScoreTestingEndpoints.setFilterRiskScoreGroupApis(batch); - ApiCollectionUsers.removeFromCollectionsForCollectionId(testingEndpoints, apiCollectionId, true); + ApiCollectionUsers.removeFromCollectionsForCollectionId(testingEndpoints, apiCollectionId); } // Add APIs to the new risk score group @@ -88,11 +88,11 @@ private void updateRiskScoreApiGroups() { public void syncRiskScoreGroupApis() { int accountId = Context.accountId.get(); - executorService.schedule( new Runnable() { - public void run() { - Context.accountId.set(accountId); - updateRiskScoreApiGroups(); - } - }, 0, TimeUnit.SECONDS); + + executorService.submit(() -> { + Context.accountId.set(accountId); + loggerMaker.infoAndAddToDb("Updating risk score API groups", LogDb.DASHBOARD); + updateRiskScoreApiGroups(); + }); } } \ No newline at end of file diff --git a/libs/dao/src/main/java/com/akto/dto/ApiCollectionUsers.java b/libs/dao/src/main/java/com/akto/dto/ApiCollectionUsers.java index ef8c451ef0..318821ad57 100644 --- a/libs/dao/src/main/java/com/akto/dto/ApiCollectionUsers.java +++ b/libs/dao/src/main/java/com/akto/dto/ApiCollectionUsers.java @@ -119,21 +119,30 @@ public static void addToCollectionsForCollectionId(List condit operationForCollectionId(conditions, apiCollectionId, update, matchFilter, false); } - public static void removeFromCollectionsForCollectionId(List conditions, int apiCollectionId, boolean matchExactCondition) { + public static void removeFromCollectionsForCollectionId(List conditions, int apiCollectionId) { Bson update = Updates.pull(SingleTypeInfo._COLLECTION_IDS, apiCollectionId); Bson matchFilter = Filters.in(SingleTypeInfo._COLLECTION_IDS, apiCollectionId); - if (matchExactCondition) { - // Remove collection ids for exact match of filter + // Check if any of the conditions is delta update based + boolean isDeltaUpdateBasedApiGroup = false; + for (TestingEndpoints testingEndpoints : conditions) { + if (TestingEndpoints.checkDeltaUpdateBased(testingEndpoints.getType())) { + isDeltaUpdateBasedApiGroup = true; + break; + } + } + + if (isDeltaUpdateBasedApiGroup) { operationForCollectionId(conditions, apiCollectionId, update, matchFilter, false); } else { operationForCollectionId(conditions, apiCollectionId, update, matchFilter, true); } + } public static void computeCollectionsForCollectionId(List conditions, int apiCollectionId) { addToCollectionsForCollectionId(conditions, apiCollectionId); - removeFromCollectionsForCollectionId(conditions, apiCollectionId, false); + removeFromCollectionsForCollectionId(conditions, apiCollectionId); updateApiCollection(conditions, apiCollectionId); } From 6a718e0bd2a1e7a9ddfb15959c0d08d716c8aa81 Mon Sep 17 00:00:00 2001 From: Oren Saldanha Date: Wed, 15 May 2024 12:07:48 +0530 Subject: [PATCH 08/11] Add catch block --- .../akto/utils/RiskScoreTestingEndpointsUtils.java | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/apps/dashboard/src/main/java/com/akto/utils/RiskScoreTestingEndpointsUtils.java b/apps/dashboard/src/main/java/com/akto/utils/RiskScoreTestingEndpointsUtils.java index 0e31ffd1eb..98e30b703f 100644 --- a/apps/dashboard/src/main/java/com/akto/utils/RiskScoreTestingEndpointsUtils.java +++ b/apps/dashboard/src/main/java/com/akto/utils/RiskScoreTestingEndpointsUtils.java @@ -89,10 +89,14 @@ private void updateRiskScoreApiGroups() { public void syncRiskScoreGroupApis() { int accountId = Context.accountId.get(); - executorService.submit(() -> { - Context.accountId.set(accountId); - loggerMaker.infoAndAddToDb("Updating risk score API groups", LogDb.DASHBOARD); - updateRiskScoreApiGroups(); - }); + try { + executorService.submit(() -> { + Context.accountId.set(accountId); + loggerMaker.infoAndAddToDb("Updating risk score API groups", LogDb.DASHBOARD); + updateRiskScoreApiGroups(); + }); + } catch (Exception e) { + loggerMaker.errorAndAddToDb("Error syncing risk score group APIs - " + e.getMessage(), LogDb.DASHBOARD); + } } } \ No newline at end of file From 94ecd2141609c0868f49a59865a24c35853db255 Mon Sep 17 00:00:00 2001 From: Oren Saldanha Date: Wed, 15 May 2024 12:16:19 +0530 Subject: [PATCH 09/11] remove creating risk score groups from initializer listener --- .../akto/listener/InitializerListener.java | 90 ++++++++++--------- 1 file changed, 47 insertions(+), 43 deletions(-) diff --git a/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java b/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java index b2c1db4c06..c6c14fcd00 100644 --- a/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java +++ b/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java @@ -1152,65 +1152,69 @@ public static void createLoginSignupGroups(BackwardCompatibility backwardCompati public static void createRiskScoreApiGroup(int id, String name, RiskScoreTestingEndpoints.RiskScoreGroupType riskScoreGroupType) { loggerMaker.infoAndAddToDb("Creating risk score group: " + name, LogDb.DASHBOARD); - ApiCollection riskScoreGroup = ApiCollectionsDao.instance.findByName(name); + //ApiCollection riskScoreGroup = ApiCollectionsDao.instance.findByName(name); + ApiCollection riskScoreGroup = new ApiCollection(id, name, Context.now(), new HashSet<>(), null, 0, false, false); + + List riskScoreConditions = new ArrayList<>(); + RiskScoreTestingEndpoints riskScoreTestingEndpoints = new RiskScoreTestingEndpoints(riskScoreGroupType); + riskScoreConditions.add(riskScoreTestingEndpoints); + + riskScoreGroup.setConditions(riskScoreConditions); + riskScoreGroup.setType(ApiCollection.Type.API_GROUP); - if (riskScoreGroup == null) { - riskScoreGroup = new ApiCollection(id, name, Context.now(), new HashSet<>(), null, 0, false, false); + ApiCollectionsDao.instance.insertOne(riskScoreGroup); + + // if (riskScoreGroup == null) { + // riskScoreGroup = new ApiCollection(id, name, Context.now(), new HashSet<>(), null, 0, false, false); - List riskScoreConditions = new ArrayList<>(); - RiskScoreTestingEndpoints riskScoreTestingEndpoints = new RiskScoreTestingEndpoints(riskScoreGroupType); - riskScoreConditions.add(riskScoreTestingEndpoints); + // List riskScoreConditions = new ArrayList<>(); + // RiskScoreTestingEndpoints riskScoreTestingEndpoints = new RiskScoreTestingEndpoints(riskScoreGroupType); + // riskScoreConditions.add(riskScoreTestingEndpoints); - riskScoreGroup.setConditions(riskScoreConditions); - riskScoreGroup.setType(ApiCollection.Type.API_GROUP); + // riskScoreGroup.setConditions(riskScoreConditions); + // riskScoreGroup.setType(ApiCollection.Type.API_GROUP); - ApiCollectionsDao.instance.insertOne(riskScoreGroup); - } + // ApiCollectionsDao.instance.insertOne(riskScoreGroup); + // } - List riskScoreConditions = riskScoreGroup.getConditions(); - RiskScoreTestingEndpoints riskScoreTestingEndpoints = (RiskScoreTestingEndpoints) riskScoreConditions.get(0); + // List riskScoreConditions = riskScoreGroup.getConditions(); + // RiskScoreTestingEndpoints riskScoreTestingEndpoints = (RiskScoreTestingEndpoints) riskScoreConditions.get(0); - // Add APIs to the risk score group - List riskScoreGroupApisList = riskScoreTestingEndpoints.fetchRiskScoreGroupApis(); + // // Add APIs to the risk score group + // List riskScoreGroupApisList = riskScoreTestingEndpoints.fetchRiskScoreGroupApis(); + // // Skip API's wwhich have already been added to the risk score group + // Iterator riskScoreGroupApisListIt = riskScoreGroupApisList.iterator(); + // while (riskScoreGroupApisListIt.hasNext()) { + // ApiInfo riskScoreGroupApi = riskScoreGroupApisListIt.next(); + // List collectionIds = riskScoreGroupApi.getCollectionIds(); - // Skip API's wwhich have already been added to the risk score group - Iterator riskScoreGroupApisListIt = riskScoreGroupApisList.iterator(); - while (riskScoreGroupApisListIt.hasNext()) { - ApiInfo riskScoreGroupApi = riskScoreGroupApisListIt.next(); - List collectionIds = riskScoreGroupApi.getCollectionIds(); + // if (collectionIds.contains(id)) { + // riskScoreGroupApisListIt.remove(); + // } + // } - if (collectionIds.contains(id)) { - riskScoreGroupApisListIt.remove(); - } - } + // loggerMaker.infoAndAddToDb("Adding " + riskScoreGroupApisList.size() + " API's to API group - " + name , LogDb.DASHBOARD); + // for (int start = 0; start < riskScoreGroupApisList.size(); start += RiskScoreTestingEndpoints.BATCH_SIZE) { + // int end = Math.min(start + RiskScoreTestingEndpoints.BATCH_SIZE, riskScoreGroupApisList.size()); - loggerMaker.infoAndAddToDb("Adding " + riskScoreGroupApisList.size() + " API's to API group - " + name , LogDb.DASHBOARD); - for (int start = 0; start < riskScoreGroupApisList.size(); start += RiskScoreTestingEndpoints.BATCH_SIZE) { - int end = Math.min(start + RiskScoreTestingEndpoints.BATCH_SIZE, riskScoreGroupApisList.size()); + // List batch = riskScoreGroupApisList.subList(start, end); - List batch = riskScoreGroupApisList.subList(start, end); - - riskScoreTestingEndpoints.setFilterRiskScoreGroupApis(batch); - ApiCollectionUsers.addToCollectionsForCollectionId(riskScoreConditions, id); - } + // riskScoreTestingEndpoints.setFilterRiskScoreGroupApis(batch); + // ApiCollectionUsers.addToCollectionsForCollectionId(riskScoreConditions, id); + // } } public static void createRiskScoreGroups(BackwardCompatibility backwardCompatibility) { if (backwardCompatibility.getRiskScoreGroups() == 0) { - try { - createRiskScoreApiGroup(111_111_148, "Low Risk APIs", RiskScoreTestingEndpoints.RiskScoreGroupType.LOW); - createRiskScoreApiGroup(111_111_149, "Medium Risk APIs", RiskScoreTestingEndpoints.RiskScoreGroupType.MEDIUM); - createRiskScoreApiGroup(111_111_150, "High Risk APIs", RiskScoreTestingEndpoints.RiskScoreGroupType.HIGH); + createRiskScoreApiGroup(111_111_148, "Low Risk APIs", RiskScoreTestingEndpoints.RiskScoreGroupType.LOW); + createRiskScoreApiGroup(111_111_149, "Medium Risk APIs", RiskScoreTestingEndpoints.RiskScoreGroupType.MEDIUM); + createRiskScoreApiGroup(111_111_150, "High Risk APIs", RiskScoreTestingEndpoints.RiskScoreGroupType.HIGH); - BackwardCompatibilityDao.instance.updateOne( - Filters.eq("_id", backwardCompatibility.getId()), - Updates.set(BackwardCompatibility.RISK_SCORE_GROUPS, Context.now()) - ); - } - catch (Exception e) { - loggerMaker.errorAndAddToDb(e, "Error while creating risk score groups: " + e.getMessage(), LogDb.DASHBOARD); - } + BackwardCompatibilityDao.instance.updateOne( + Filters.eq("_id", backwardCompatibility.getId()), + Updates.set(BackwardCompatibility.RISK_SCORE_GROUPS, Context.now()) + ); } } From 315c1b530c1c20204e38f99da1fbb985add6dc75 Mon Sep 17 00:00:00 2001 From: Oren Saldanha Date: Wed, 15 May 2024 14:49:44 +0530 Subject: [PATCH 10/11] use unwind in pipelines --- .../java/com/akto/action/ApiCollectionsAction.java | 11 ++++++++++- .../dao/src/main/java/com/akto/dao/ApiInfoDao.java | 14 ++++++++++++-- .../testing_run_findings/TestingRunIssuesDao.java | 7 ++++++- 3 files changed, 28 insertions(+), 4 deletions(-) diff --git a/apps/dashboard/src/main/java/com/akto/action/ApiCollectionsAction.java b/apps/dashboard/src/main/java/com/akto/action/ApiCollectionsAction.java index 645fd4bac9..28419c5bc3 100644 --- a/apps/dashboard/src/main/java/com/akto/action/ApiCollectionsAction.java +++ b/apps/dashboard/src/main/java/com/akto/action/ApiCollectionsAction.java @@ -40,6 +40,7 @@ import com.mongodb.client.model.Updates; import com.mongodb.BasicDBObject; import com.mongodb.client.model.Sorts; +import com.mongodb.client.model.UnwindOptions; import com.opensymphony.xwork2.Action; import org.apache.commons.lang3.tuple.Pair; import org.bson.Document; @@ -417,7 +418,15 @@ public String fetchLastSeenInfoInCollections(){ public String fetchRiskScoreInfo(){ Map riskScoreMap = new HashMap<>(); List pipeline = new ArrayList<>(); - BasicDBObject groupId = new BasicDBObject("apiCollectionId", "$_id.apiCollectionId"); + + /* + * Use Unwind to unwind the collectionIds field resulting in a document for each collectionId in the collectionIds array + */ + UnwindOptions unwindOptions = new UnwindOptions(); + unwindOptions.preserveNullAndEmptyArrays(false); + pipeline.add(Aggregates.unwind("$collectionIds", unwindOptions)); + + BasicDBObject groupId = new BasicDBObject("apiCollectionId", "$collectionIds"); pipeline.add(Aggregates.sort( Sorts.descending(ApiInfo.RISK_SCORE) )); diff --git a/libs/dao/src/main/java/com/akto/dao/ApiInfoDao.java b/libs/dao/src/main/java/com/akto/dao/ApiInfoDao.java index 2ed79388cf..cb259059a0 100644 --- a/libs/dao/src/main/java/com/akto/dao/ApiInfoDao.java +++ b/libs/dao/src/main/java/com/akto/dao/ApiInfoDao.java @@ -13,6 +13,7 @@ import com.mongodb.client.model.Filters; import com.mongodb.client.model.Projections; import com.mongodb.client.model.Sorts; +import com.mongodb.client.model.UnwindOptions; import com.mongodb.client.model.Updates; import org.bson.Document; @@ -91,7 +92,11 @@ public Map getCoverageCount(){ int oneMonthAgo = Context.now() - Constants.ONE_MONTH_TIMESTAMP ; pipeline.add(Aggregates.match(Filters.gte("lastTested", oneMonthAgo))); - BasicDBObject groupedId2 = new BasicDBObject("apiCollectionId", "$_id.apiCollectionId"); + UnwindOptions unwindOptions = new UnwindOptions(); + unwindOptions.preserveNullAndEmptyArrays(false); + pipeline.add(Aggregates.unwind("$collectionIds", unwindOptions)); + + BasicDBObject groupedId2 = new BasicDBObject("apiCollectionId", "$collectionIds"); pipeline.add(Aggregates.group(groupedId2, Accumulators.sum("count",1))); pipeline.add(Aggregates.project( Projections.fields( @@ -115,7 +120,12 @@ public Map getCoverageCount(){ public Map getLastTrafficSeen(){ Map result = new HashMap<>(); List pipeline = new ArrayList<>(); - BasicDBObject groupedId = new BasicDBObject("apiCollectionId", "$_id.apiCollectionId"); + + UnwindOptions unwindOptions = new UnwindOptions(); + unwindOptions.preserveNullAndEmptyArrays(false); + pipeline.add(Aggregates.unwind("$collectionIds", unwindOptions)); + + BasicDBObject groupedId = new BasicDBObject("apiCollectionId", "$collectionIds"); pipeline.add(Aggregates.sort(Sorts.orderBy(Sorts.descending(ApiInfo.ID_API_COLLECTION_ID), Sorts.descending(ApiInfo.LAST_SEEN)))); pipeline.add(Aggregates.group(groupedId, Accumulators.first(ApiInfo.LAST_SEEN, "$lastSeen"))); diff --git a/libs/dao/src/main/java/com/akto/dao/testing_run_findings/TestingRunIssuesDao.java b/libs/dao/src/main/java/com/akto/dao/testing_run_findings/TestingRunIssuesDao.java index 5f38dfa992..eecca8a4e7 100644 --- a/libs/dao/src/main/java/com/akto/dao/testing_run_findings/TestingRunIssuesDao.java +++ b/libs/dao/src/main/java/com/akto/dao/testing_run_findings/TestingRunIssuesDao.java @@ -21,6 +21,7 @@ import com.mongodb.client.model.Aggregates; import com.mongodb.client.model.Filters; import com.mongodb.client.model.Projections; +import com.mongodb.client.model.UnwindOptions; public class TestingRunIssuesDao extends AccountsContextDao { @@ -61,7 +62,11 @@ public Map> getSeveritiesMapForCollections(){ List pipeline = new ArrayList<>(); pipeline.add(Aggregates.match(Filters.eq(TestingRunIssues.TEST_RUN_ISSUES_STATUS, "OPEN"))); - BasicDBObject groupedId = new BasicDBObject("apiCollectionId", "$_id.apiInfoKey.apiCollectionId") + UnwindOptions unwindOptions = new UnwindOptions(); + unwindOptions.preserveNullAndEmptyArrays(false); + pipeline.add(Aggregates.unwind("$collectionIds", unwindOptions)); + + BasicDBObject groupedId = new BasicDBObject("apiCollectionId", "$collectionIds") .append("severity", "$severity") ; pipeline.add(Aggregates.group(groupedId, Accumulators.sum("count", 1))); From f1ada1c99b0433414be17db22bc40b35356b7be7 Mon Sep 17 00:00:00 2001 From: Oren Saldanha Date: Wed, 15 May 2024 17:15:53 +0530 Subject: [PATCH 11/11] remove commit --- .../akto/listener/InitializerListener.java | 41 ------------------- 1 file changed, 41 deletions(-) diff --git a/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java b/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java index 51c78f5a4b..700ce7fb96 100644 --- a/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java +++ b/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java @@ -1153,7 +1153,6 @@ public static void createLoginSignupGroups(BackwardCompatibility backwardCompati public static void createRiskScoreApiGroup(int id, String name, RiskScoreTestingEndpoints.RiskScoreGroupType riskScoreGroupType) { loggerMaker.infoAndAddToDb("Creating risk score group: " + name, LogDb.DASHBOARD); - //ApiCollection riskScoreGroup = ApiCollectionsDao.instance.findByName(name); ApiCollection riskScoreGroup = new ApiCollection(id, name, Context.now(), new HashSet<>(), null, 0, false, false); List riskScoreConditions = new ArrayList<>(); @@ -1164,46 +1163,6 @@ public static void createRiskScoreApiGroup(int id, String name, RiskScoreTesting riskScoreGroup.setType(ApiCollection.Type.API_GROUP); ApiCollectionsDao.instance.insertOne(riskScoreGroup); - - // if (riskScoreGroup == null) { - // riskScoreGroup = new ApiCollection(id, name, Context.now(), new HashSet<>(), null, 0, false, false); - - // List riskScoreConditions = new ArrayList<>(); - // RiskScoreTestingEndpoints riskScoreTestingEndpoints = new RiskScoreTestingEndpoints(riskScoreGroupType); - // riskScoreConditions.add(riskScoreTestingEndpoints); - - // riskScoreGroup.setConditions(riskScoreConditions); - // riskScoreGroup.setType(ApiCollection.Type.API_GROUP); - - // ApiCollectionsDao.instance.insertOne(riskScoreGroup); - // } - - // List riskScoreConditions = riskScoreGroup.getConditions(); - // RiskScoreTestingEndpoints riskScoreTestingEndpoints = (RiskScoreTestingEndpoints) riskScoreConditions.get(0); - - // // Add APIs to the risk score group - // List riskScoreGroupApisList = riskScoreTestingEndpoints.fetchRiskScoreGroupApis(); - - // // Skip API's wwhich have already been added to the risk score group - // Iterator riskScoreGroupApisListIt = riskScoreGroupApisList.iterator(); - // while (riskScoreGroupApisListIt.hasNext()) { - // ApiInfo riskScoreGroupApi = riskScoreGroupApisListIt.next(); - // List collectionIds = riskScoreGroupApi.getCollectionIds(); - - // if (collectionIds.contains(id)) { - // riskScoreGroupApisListIt.remove(); - // } - // } - - // loggerMaker.infoAndAddToDb("Adding " + riskScoreGroupApisList.size() + " API's to API group - " + name , LogDb.DASHBOARD); - // for (int start = 0; start < riskScoreGroupApisList.size(); start += RiskScoreTestingEndpoints.BATCH_SIZE) { - // int end = Math.min(start + RiskScoreTestingEndpoints.BATCH_SIZE, riskScoreGroupApisList.size()); - - // List batch = riskScoreGroupApisList.subList(start, end); - - // riskScoreTestingEndpoints.setFilterRiskScoreGroupApis(batch); - // ApiCollectionUsers.addToCollectionsForCollectionId(riskScoreConditions, id); - // } } public static void createRiskScoreGroups(BackwardCompatibility backwardCompatibility) {