From bbc95a2385cf1377cf2c09df4119ea0ea532bc9f Mon Sep 17 00:00:00 2001 From: Ankush Jain Date: Tue, 2 Jan 2024 03:18:17 +0530 Subject: [PATCH 01/39] use url from originalRequest if present, ignore plus-sign for number comparison, add INTEGER/UUID/ALPHA_NUM first time itself, ignore status code for postman import, use sample-url for private-var-context --- .../java/com/akto/parsers/HttpCallParser.java | 4 ++ .../java/com/akto/runtime/URLAggregator.java | 46 ++++++++++++++++++- .../src/main/java/com/akto/utils/Utils.java | 13 +++++- .../akto/test_editor/filter/FilterAction.java | 10 ++++ .../java/com/akto/dto/type/URLTemplate.java | 3 ++ 5 files changed, 74 insertions(+), 2 deletions(-) diff --git a/apps/api-runtime/src/main/java/com/akto/parsers/HttpCallParser.java b/apps/api-runtime/src/main/java/com/akto/parsers/HttpCallParser.java index c0ad87423a..ebe9274bbf 100644 --- a/apps/api-runtime/src/main/java/com/akto/parsers/HttpCallParser.java +++ b/apps/api-runtime/src/main/java/com/akto/parsers/HttpCallParser.java @@ -267,6 +267,10 @@ public List filterHttpResponseParams(List convertApiInAktoFormat(JsonNode apiInfo, Map responseHeadersMap = getHeaders((ArrayNode) response.get("header"), variables); + JsonNode respHeaders = response.get("header"); + Map responseHeadersMap = new HashMap<>(); + if (respHeaders == null) { + responseHeadersMap = getHeaders((ArrayNode) response.get("header"), variables); + } + + JsonNode originalRequest = response.get("originalRequest"); + + if (originalRequest != null) { + result.put("path", getPath(originalRequest, variables)); + } + responseHeadersString = mapper.writeValueAsString(responseHeadersMap); JsonNode responsePayloadNode = response.get("body"); diff --git a/apps/testing/src/main/java/com/akto/test_editor/filter/FilterAction.java b/apps/testing/src/main/java/com/akto/test_editor/filter/FilterAction.java index a1bf4af4fb..3c7a7193b6 100644 --- a/apps/testing/src/main/java/com/akto/test_editor/filter/FilterAction.java +++ b/apps/testing/src/main/java/com/akto/test_editor/filter/FilterAction.java @@ -1013,6 +1013,8 @@ public BasicDBObject getPrivateResourceCount(OriginalHttpRequest originalHttpReq if (APICatalog.isTemplateUrl(url)) { URLTemplate urlTemplate = APICatalogSync.createUrlTemplate(url, method); String[] tokens = urlTemplate.getTokens(); + + String[] urlWithParamsTokens = APICatalogSync.createUrlTemplate(urlWithParams, method).getTokens(); for (int i = 0;i < tokens.length; i++) { if (tokens[i] == null) { SingleTypeInfo singleTypeInfo = querySti(i+"", true,apiInfoKey, false, -1); @@ -1021,6 +1023,14 @@ public BasicDBObject getPrivateResourceCount(OriginalHttpRequest originalHttpReq privateCnt++; } if (singleTypeInfo == null || !singleTypeInfo.getIsPrivate() || singleTypeInfo.getValues() == null || singleTypeInfo.getValues().getElements().size() == 0) { + if (urlWithParamsTokens.length > i) { + obj.put("key", i+""); + obj.put("value", urlWithParamsTokens[i]); + if (privateValues.size() < 5) { + privateValues.add(obj); + } + privateCnt++; + } continue; } Set valSet = singleTypeInfo.getValues().getElements(); diff --git a/libs/dao/src/main/java/com/akto/dto/type/URLTemplate.java b/libs/dao/src/main/java/com/akto/dto/type/URLTemplate.java index 690dc9e841..cc0ffb0241 100644 --- a/libs/dao/src/main/java/com/akto/dto/type/URLTemplate.java +++ b/libs/dao/src/main/java/com/akto/dto/type/URLTemplate.java @@ -75,6 +75,9 @@ public boolean match(String[] url, Method urlMethod) { if (!"true".equals(thatToken.toLowerCase()) && !"false".equals(thatToken.toLowerCase())) return false; break; case INTEGER: + if (thatToken.charAt(0) == '+') { + thatToken = thatToken.substring(1); + } if (!NumberUtils.isParsable(thatToken) || thatToken.contains(".")) return false; break; case FLOAT: From f70e6aa2e479fd4b70bcb8560f53ef53bb4f08fc Mon Sep 17 00:00:00 2001 From: Ark2307 Date: Thu, 4 Jan 2024 14:08:38 +0530 Subject: [PATCH 02/39] WIP --- .../java/com/akto/runtime/URLAggregator.java | 61 +++++++--- .../com/akto/action/CustomDataTypeAction.java | 110 +++++++++++------- .../pages/observe/data_types/DataTypes.jsx | 22 +++- .../pages/observe/data_types/transform.js | 37 +++--- .../java/com/akto/dto/CustomDataType.java | 22 ++++ .../main/java/com/akto/dto/type/KeyTypes.java | 38 +++--- 6 files changed, 198 insertions(+), 92 deletions(-) diff --git a/apps/api-runtime/src/main/java/com/akto/runtime/URLAggregator.java b/apps/api-runtime/src/main/java/com/akto/runtime/URLAggregator.java index 3e6c31fb98..fbf8dcb4c5 100644 --- a/apps/api-runtime/src/main/java/com/akto/runtime/URLAggregator.java +++ b/apps/api-runtime/src/main/java/com/akto/runtime/URLAggregator.java @@ -1,19 +1,20 @@ package com.akto.runtime; -import java.io.UnsupportedEncodingException; -import java.net.URLDecoder; import java.util.Collections; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.regex.Pattern; +import com.akto.dto.type.KeyTypes; import com.akto.dto.type.SingleTypeInfo; import com.akto.dto.type.URLStatic; import com.akto.dto.type.URLMethods.Method; +import com.akto.dao.context.Context; +import com.akto.dto.CustomDataType; import com.akto.dto.HttpResponseParams; import com.akto.dto.type.URLTemplate; -import com.mongodb.BasicDBObject; +import com.akto.dto.type.SingleTypeInfo.SubType; import org.apache.commons.lang3.math.NumberUtils; import org.slf4j.Logger; @@ -29,12 +30,39 @@ public class URLAggregator { ConcurrentMap> urls; + private static SingleTypeInfo.SuperType getTokenSupertype(String tempToken, Pattern pattern, Boolean isCustom){ + String numToken = tempToken; + SingleTypeInfo.SuperType defaultResult = isCustom ? SingleTypeInfo.SuperType.STRING : null; + if (tempToken.charAt(0) == '+') { + numToken = tempToken.substring(1); + } + + if (NumberUtils.isParsable(numToken)) { + return SingleTypeInfo.SuperType.INTEGER; + + } else if(pattern.matcher(tempToken).matches()){ + return SingleTypeInfo.SuperType.STRING; + } else if (isAlphanumericString(tempToken)) { + return SingleTypeInfo.SuperType.STRING; + } + return defaultResult; + } + + private static SubType getTokenSubType(Object token){ + for (CustomDataType customDataType: SingleTypeInfo.getCustomDataTypesSortedBySensitivity(Context.accountId.get())) { + if (!customDataType.isActive()) continue; + boolean result = customDataType.validateTokenData(token); + if (result) return customDataType.toSubType(); + } + return KeyTypes.findSubType(token, "", null); + } + public static URLStatic getBaseURL(String url, String method) { + if (url == null) { return null; } - String urlStr = url.split("\\?")[0]; String[] urlTokens = tokenize(urlStr); @@ -50,19 +78,20 @@ public static URLStatic getBaseURL(String url, String method) { continue; } - String numToken = tempToken; - if (tempToken.charAt(0) == '+') { - numToken = tempToken.substring(1); - } - - if (NumberUtils.isParsable(numToken)) { - newTypes[i] = SingleTypeInfo.SuperType.INTEGER; + SubType subType = getTokenSubType(tempToken); + + if(!(subType == SingleTypeInfo.GENERIC || subType == SingleTypeInfo.OTHER)){ + SingleTypeInfo.SuperType superType = subType.getSuperType();; + if(superType == SingleTypeInfo.SuperType.CUSTOM){ + superType = getTokenSupertype(tempToken,pattern, true); + } + newTypes[i] = superType; urlTokens[i] = null; - } else if(pattern.matcher(tempToken).matches()){ - newTypes[i] = SingleTypeInfo.SuperType.STRING; - urlTokens[i] = null; - } else if (isAlphanumericString(tempToken)) { - newTypes[i] = SingleTypeInfo.SuperType.STRING; + continue; + } + + if(getTokenSupertype(tempToken, pattern, false) != null){ + newTypes[i] = getTokenSupertype(tempToken, pattern, false); urlTokens[i] = null; } } diff --git a/apps/dashboard/src/main/java/com/akto/action/CustomDataTypeAction.java b/apps/dashboard/src/main/java/com/akto/action/CustomDataTypeAction.java index 151f05ab7d..20abb1e501 100644 --- a/apps/dashboard/src/main/java/com/akto/action/CustomDataTypeAction.java +++ b/apps/dashboard/src/main/java/com/akto/action/CustomDataTypeAction.java @@ -46,6 +46,8 @@ public class CustomDataTypeAction extends UserAction{ private String valueOperator; private List valueConditionFromUsers; + private String tokenOperator; + private List tokenConditionFromUsers; public static class ConditionFromUser { Predicate.Type type; Map valueMap; @@ -428,72 +430,74 @@ public boolean forPayload(String payload, CustomDataType customDataType, Key api } - public CustomDataType generateCustomDataType(int userId) throws AktoCustomException { - if (name == null || name.length() == 0) throw new AktoCustomException("Name cannot be empty"); - int maxChars = 25; - if (name.length() > maxChars) throw new AktoCustomException("Maximum length allowed is "+maxChars+" characters"); - name = name.trim(); - name = name.toUpperCase(); - if (!(name.matches("[A-Z_0-9 ]+"))) throw new AktoCustomException("Name can only contain alphabets, spaces, numbers and underscores"); - - if (SingleTypeInfo.subTypeMap.containsKey(name)) { - throw new AktoCustomException("Data type name reserved"); - } - - - Conditions keyConditions = null; - if (keyConditionFromUsers != null && keyOperator != null) { + private Conditions getValidConditionsFromUserConditions(List conditionFromUsers, String operator, String key) throws AktoCustomException { + if (conditionFromUsers != null && operator != null) { Conditions.Operator kOperator; try { - kOperator = Conditions.Operator.valueOf(keyOperator); + kOperator = Conditions.Operator.valueOf(operator); } catch (Exception ignored) { - throw new AktoCustomException("Invalid key operator"); + throw new AktoCustomException("Invalid " + key + " operator"); } List predicates = new ArrayList<>(); - for (ConditionFromUser conditionFromUser: keyConditionFromUsers) { + for (ConditionFromUser conditionFromUser: conditionFromUsers) { Predicate predicate = Predicate.generatePredicate(conditionFromUser.type, conditionFromUser.valueMap); if (predicate == null) { - throw new AktoCustomException("Invalid key conditions"); + throw new AktoCustomException("Invalid " + key + " conditions"); } else { predicates.add(predicate); } } if (predicates.size() > 0) { - keyConditions = new Conditions(predicates, kOperator); + return new Conditions(predicates, kOperator); } } + return null; + } - Conditions valueConditions = null; - if (valueConditionFromUsers != null && valueOperator != null) { - Conditions.Operator vOperator; - try { - vOperator = Conditions.Operator.valueOf(valueOperator); - } catch (Exception ignored) { - throw new AktoCustomException("Invalid value operator"); - } + private static Boolean isInvalidCondition(Conditions conditions){ + return (conditions == null || conditions.getPredicates() == null || conditions.getPredicates().size() == 0); + } - List predicates = new ArrayList<>(); - for (ConditionFromUser conditionFromUser: valueConditionFromUsers) { - Predicate predicate = Predicate.generatePredicate(conditionFromUser.type, conditionFromUser.valueMap); - if (predicate == null) { - throw new AktoCustomException("Invalid value conditions"); - } else { - predicates.add(predicate); - } - } + public CustomDataType generateCustomDataType(int userId) throws AktoCustomException { + if (name == null || name.length() == 0) throw new AktoCustomException("Name cannot be empty"); + int maxChars = 25; + if (name.length() > maxChars) throw new AktoCustomException("Maximum length allowed is "+maxChars+" characters"); + name = name.trim(); + name = name.toUpperCase(); + if (!(name.matches("[A-Z_0-9 ]+"))) throw new AktoCustomException("Name can only contain alphabets, spaces, numbers and underscores"); - if (predicates.size() > 0) { - valueConditions = new Conditions(predicates, vOperator); - } + if (SingleTypeInfo.subTypeMap.containsKey(name)) { + throw new AktoCustomException("Data type name reserved"); } - if ((keyConditions == null || keyConditions.getPredicates() == null || keyConditions.getPredicates().size() == 0) && - (valueConditions == null || valueConditions.getPredicates() ==null || valueConditions.getPredicates().size() == 0)) { - throw new AktoCustomException("Both key and value conditions can't be empty"); + Conditions keyConditions = null; + try { + keyConditions = getValidConditionsFromUserConditions(keyConditionFromUsers, keyOperator , "key"); + } catch (Exception e) { + throw new AktoCustomException(e.getMessage()); + } + + Conditions valueConditions = null; + try { + valueConditions = getValidConditionsFromUserConditions(valueConditionFromUsers, valueOperator, "value"); + } catch (Exception e) { + throw new AktoCustomException(e.getMessage()); + } + + Conditions tokenConditions = null; + try { + tokenConditions = getValidConditionsFromUserConditions(tokenConditionFromUsers, tokenOperator, "urlToken"); + } catch (Exception e) { + throw new AktoCustomException(e.getMessage()); + } + + if (isInvalidCondition(keyConditions) && isInvalidCondition(valueConditions) && isInvalidCondition(tokenConditions)) { + + throw new AktoCustomException("All key, value and token conditions can't be empty"); } Conditions.Operator mainOperator; @@ -511,8 +515,10 @@ public CustomDataType generateCustomDataType(int userId) throws AktoCustomExcept } IgnoreData ignoreData = new IgnoreData(); - return new CustomDataType(name, sensitiveAlways, sensitivePositions, userId, + CustomDataType customDataType = new CustomDataType(name, sensitiveAlways, sensitivePositions, userId, true,keyConditions,valueConditions, mainOperator,ignoreData); + customDataType.setTokenConditions(tokenConditions); + return customDataType; } public void setCreateNew(boolean createNew) { @@ -737,4 +743,20 @@ public List getSensitivePosition() { public void setSensitivePosition(List sensitivePosition) { this.sensitivePosition = sensitivePosition; } + + public List getTokenConditionFromUsers() { + return tokenConditionFromUsers; + } + + public void setTokenConditionFromUsers(List tokenConditionFromUsers) { + this.tokenConditionFromUsers = tokenConditionFromUsers; + } + + public String getTokenOperator() { + return tokenOperator; + } + + public void setTokenOperator(String tokenOperator) { + this.tokenOperator = tokenOperator; + } } diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/data_types/DataTypes.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/data_types/DataTypes.jsx index 1972e011dc..7da1a4e488 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/data_types/DataTypes.jsx +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/data_types/DataTypes.jsx @@ -248,6 +248,24 @@ function DataTypes() { ) + const urlConditionsCard = ( + + URL token conditions + } + > + { handleDispatch(val, "tokenConditions") }} + /> + + ) + const compareFunc = () => { return !change } @@ -270,7 +288,9 @@ function DataTypes() { ) - let components = (!isNew && currState.dataType === 'Akto') ? [descriptionCard, requestCard] : [descriptionCard, conditionsCard, requestCard] + console.log(currState) + + let components = (!isNew && currState.dataType === 'Akto') ? [descriptionCard, requestCard] : [descriptionCard, conditionsCard, urlConditionsCard,requestCard] return ( { + return{ + type: element.type, + valueMap:{ + value: element.value + } + } + }, + convertDataForCustomPayload : function(state){ - const keyArr = state.keyConditions.predicates.map((element)=> { - return{ - type: element.type, - valueMap:{ - value: element.value - } - } - }) - - const valueArr = state.valueConditions.predicates.map((element)=> { - return{ - type: element.type, - valueMap:{ - value: element.value - } - } - }) + const keyArr = state.keyConditions.predicates.map(this.convertMapFunction) + const valueArr = state.valueConditions.predicates.map(this.convertMapFunction) + const tokenArr = state.tokenConditions.predicates.map(this.convertMapFunction) let sensitiveObj = this.convertToSensitiveData(state.sensitiveState) @@ -99,6 +98,8 @@ const func = { sensitivePosition: sensitiveObj.sensitivePosition, valueConditionFromUsers: valueArr, valueOperator: state.valueConditions.operator, + tokenConditionFromUsers: tokenArr, + tokenOperator: state.tokenConditions.operator, } return finalObj diff --git a/libs/dao/src/main/java/com/akto/dto/CustomDataType.java b/libs/dao/src/main/java/com/akto/dto/CustomDataType.java index 739f0b1f70..73320aa98d 100644 --- a/libs/dao/src/main/java/com/akto/dto/CustomDataType.java +++ b/libs/dao/src/main/java/com/akto/dto/CustomDataType.java @@ -31,6 +31,8 @@ public class CustomDataType { Conditions.Operator operator; public static final String IGNORE_DATA = "ignoreData"; private IgnoreData ignoreData; + public static final String TOKEN_CONDITIONS = "tokenConditions"; + Conditions tokenConditions; public CustomDataType() { } @@ -89,6 +91,18 @@ public boolean validateRaw(Object value, Object key) throws Exception { } + public boolean validateTokenData (Object token){ + if(this.tokenConditions == null){ + return false; + } + try { + return this.tokenConditions.validate(token); + } catch (Exception e) { + return false; + } + + } + public ObjectId getId() { return id; @@ -178,6 +192,14 @@ public void setIgnoreData(IgnoreData ignoreData) { this.ignoreData = ignoreData; } + public Conditions getTokenConditions() { + return tokenConditions; + } + + public void setTokenConditions(Conditions tokenConditions) { + this.tokenConditions = tokenConditions; + } + @Override public String toString() { return "{" + diff --git a/libs/dao/src/main/java/com/akto/dto/type/KeyTypes.java b/libs/dao/src/main/java/com/akto/dto/type/KeyTypes.java index f53fb6462a..6ad021b9c4 100644 --- a/libs/dao/src/main/java/com/akto/dto/type/KeyTypes.java +++ b/libs/dao/src/main/java/com/akto/dto/type/KeyTypes.java @@ -110,19 +110,7 @@ private static boolean checkForSubtypesTest(ParamId paramId, IgnoreData ignoreDa return true; } - public static SubType findSubType(Object o,String key, ParamId paramId) { - - int accountId = Context.accountId.get(); - boolean checkForSubtypes = true ; - for (String keyType : SingleTypeInfo.getCustomDataTypeMap(accountId).keySet()) { - IgnoreData ignoreData = SingleTypeInfo.getCustomDataTypeMap(accountId).get(keyType).getIgnoreData(); - checkForSubtypes = checkForSubtypesTest(paramId, ignoreData); - } - for (String keyType : SingleTypeInfo.getAktoDataTypeMap(accountId).keySet()) { - IgnoreData ignoreData = SingleTypeInfo.getAktoDataTypeMap(accountId).get(keyType).getIgnoreData(); - checkForSubtypes = checkForSubtypesTest(paramId, ignoreData); - } - + private static SubType getSubtype(Object o,String key, boolean checkForSubtypes){ if (o == null) { return SingleTypeInfo.NULL; } @@ -198,6 +186,30 @@ public static SubType findSubType(Object o,String key, ParamId paramId) { return SingleTypeInfo.OTHER; } + public static SubType findSubType(Object o,String key, ParamId paramId, Boolean executeCheckForSubtypes){ + if(executeCheckForSubtypes){ + return findSubType(o, key, paramId); + }else{ + return getSubtype(o, key, true); + } + } + + public static SubType findSubType(Object o,String key, ParamId paramId) { + + int accountId = Context.accountId.get(); + boolean checkForSubtypes = true ; + for (String keyType : SingleTypeInfo.getCustomDataTypeMap(accountId).keySet()) { + IgnoreData ignoreData = SingleTypeInfo.getCustomDataTypeMap(accountId).get(keyType).getIgnoreData(); + checkForSubtypes = checkForSubtypesTest(paramId, ignoreData); + } + for (String keyType : SingleTypeInfo.getAktoDataTypeMap(accountId).keySet()) { + IgnoreData ignoreData = SingleTypeInfo.getAktoDataTypeMap(accountId).get(keyType).getIgnoreData(); + checkForSubtypes = checkForSubtypesTest(paramId, ignoreData); + } + + return getSubtype(o, key, checkForSubtypes); + } + public Map getOccurrences() { return this.occurrences; } From 6c39eaa6bd01f2706286f56c3fe3ea0cedf8844d Mon Sep 17 00:00:00 2001 From: aktoboy Date: Fri, 5 Jan 2024 21:48:00 +0530 Subject: [PATCH 03/39] Added changes to redact data types --- .../java/com/akto/runtime/APICatalogSync.java | 53 ++++---- .../java/com/akto/utils/RedactSampleData.java | 117 +++++++++++++++++- .../test/java/com/akto/parsers/TestDump2.java | 14 +-- .../java/com/akto/parsers/TestMergingNew.java | 2 +- .../com/akto/utils/TestRedactSampleData.java | 2 + .../com/akto/action/ApiCollectionsAction.java | 55 +++++++- .../com/akto/action/CustomDataTypeAction.java | 101 ++++++++++++++- .../akto/action/observe/InventoryAction.java | 3 +- .../akto/listener/InitializerListener.java | 4 + apps/dashboard/src/main/resources/struts.xml | 10 ++ .../AllSensitiveData/AllSensitiveData.jsx | 6 +- .../src/apps/dashboard/pages/observe/api.js | 10 ++ .../observe/api_collections/ApiEndpoints.jsx | 16 +++ .../pages/observe/data_types/DataTypes.jsx | 22 +++- .../pages/observe/data_types/transform.js | 13 +- .../main/java/com/akto/dao/MCollection.java | 7 ++ .../java/com/akto/dao/SingleTypeInfoDao.java | 43 +++++++ .../main/java/com/akto/dto/AktoDataType.java | 31 +++++ .../main/java/com/akto/dto/ApiCollection.java | 35 ++++++ .../java/com/akto/dto/CustomDataType.java | 39 ++++++ .../akto/dto/type/AccountDataTypesInfo.java | 17 ++- .../com/akto/dto/type/SingleTypeInfo.java | 16 +++ 22 files changed, 551 insertions(+), 65 deletions(-) diff --git a/apps/api-runtime/src/main/java/com/akto/runtime/APICatalogSync.java b/apps/api-runtime/src/main/java/com/akto/runtime/APICatalogSync.java index 5ede05868e..d517a521c1 100644 --- a/apps/api-runtime/src/main/java/com/akto/runtime/APICatalogSync.java +++ b/apps/api-runtime/src/main/java/com/akto/runtime/APICatalogSync.java @@ -3,7 +3,6 @@ import com.akto.dao.*; import com.akto.dao.context.Context; import com.akto.dto.*; -import com.akto.dto.HttpResponseParams.Source; import com.akto.dto.traffic.Key; import com.akto.dto.traffic.SampleData; import com.akto.dto.traffic.TrafficInfo; @@ -13,7 +12,6 @@ import com.akto.dto.type.URLMethods.Method; import com.akto.log.LoggerMaker; import com.akto.log.LoggerMaker.LogDb; -import com.akto.parsers.HttpCallParser; import com.akto.runtime.merge.MergeOnHostOnly; import com.akto.task.Cluster; import com.akto.types.CappedSet; @@ -47,6 +45,8 @@ public class APICatalogSync { public Map sensitiveParamInfoBooleanMap; public static boolean mergeAsyncOutside = false; + public Map apiCollectionToRedactPayload; + public APICatalogSync(String userIdentifier,int thresh) { this.thresh = thresh; this.userIdentifier = userIdentifier; @@ -61,6 +61,11 @@ public APICatalogSync(String userIdentifier,int thresh) { } catch (Exception e) { } + apiCollectionToRedactPayload = new HashMap<>(); + List all = ApiCollectionsDao.instance.findAll(new BasicDBObject()); + for(ApiCollection apiCollection: all) { + apiCollectionToRedactPayload.put(apiCollection.getId(), apiCollection.getRedact()); + } } @@ -902,7 +907,7 @@ Map convertToMap(List l) { return ret; } - public ArrayList> getDBUpdatesForSampleData(int apiCollectionId, APICatalog currentDelta, APICatalog dbCatalog ,boolean redactSampleData, boolean forceUpdate ) { + public ArrayList> getDBUpdatesForSampleData(int apiCollectionId, APICatalog currentDelta, APICatalog dbCatalog, boolean forceUpdate, boolean accountLevelRedact, boolean apiCollectionLevelRedact) { List sampleData = new ArrayList<>(); Map deltaStrictURLToMethods = currentDelta.getStrictURLToMethods(); Map dbStrictURLToMethods = dbCatalog.getStrictURLToMethods(); @@ -931,25 +936,9 @@ public ArrayList> getDBUpdatesForSampleData(int apiCollec } List finalSamples = new ArrayList<>(); for (String s: sample.getSamples()) { - boolean finalRedact = redactSampleData; - if (finalRedact) { - try { - HttpResponseParams httpResponseParams = HttpCallParser.parseKafkaMessage(s); - Source source = httpResponseParams.getSource(); - if (source.equals(Source.HAR) || source.equals(Source.PCAP)) finalRedact = false; - } catch (Exception e1) { - e1.printStackTrace(); - continue; - } - } - try { - if (finalRedact) { - String redact = RedactSampleData.redact(s); - finalSamples.add(redact); - } else { - finalSamples.add(s); - } + String redactedSample = RedactSampleData.redactIfRequired(s, accountLevelRedact, apiCollectionLevelRedact); + finalSamples.add(redactedSample); } catch (Exception e) { ; } @@ -998,7 +987,7 @@ public ArrayList> getDBUpdatesForTraffic(int apiCollecti return bulkUpdates; } - public DbUpdateReturn getDBUpdatesForParams(APICatalog currentDelta, APICatalog currentState, boolean redactSampleData) { + public DbUpdateReturn getDBUpdatesForParams(APICatalog currentDelta, APICatalog currentState, boolean redactSampleData, boolean collectionLevelRedact) { Map dbInfoMap = convertToMap(currentState.getAllTypeInfo()); Map deltaInfoMap = convertToMap(currentDelta.getAllTypeInfo()); @@ -1066,7 +1055,7 @@ public DbUpdateReturn getDBUpdatesForParams(APICatalog currentDelta, APICatalog if (values != null) { Set elements = new HashSet<>(); for (String el: values.getElements()) { - if (redactSampleData) { + if (redactSampleData || collectionLevelRedact) { elements.add(el.hashCode()+""); } else { elements.add(el); @@ -1079,7 +1068,18 @@ public DbUpdateReturn getDBUpdatesForParams(APICatalog currentDelta, APICatalog } - if (!redactSampleData && deltaInfo.getExamples() != null && !deltaInfo.getExamples().isEmpty()) { + if (!(redactSampleData || collectionLevelRedact) && deltaInfo.getExamples() != null && !deltaInfo.getExamples().isEmpty()) { + Set updatedSampleData = new HashSet<>(); + for (Object example : deltaInfo.getExamples()) { + try{ + String exampleStr = (String) example; + String s = RedactSampleData.redactMarkedDataTypes(exampleStr); + updatedSampleData.add(s); + } catch (Exception e) { + ; + } + } + deltaInfo.setExamples(updatedSampleData); Bson bson = Updates.combine( Updates.pushEach(SensitiveSampleData.SAMPLE_DATA, Arrays.asList(deltaInfo.getExamples().toArray()), new PushOptions().slice(-1 *SensitiveSampleData.cap)), Updates.setOnInsert(SingleTypeInfo._COLLECTION_IDS, Arrays.asList(deltaInfo.getApiCollectionId())) @@ -1379,7 +1379,8 @@ public void syncWithDB(boolean syncImmediately, boolean fetchAllSTI) { for(int apiCollectionId: this.delta.keySet()) { APICatalog deltaCatalog = this.delta.get(apiCollectionId); APICatalog dbCatalog = this.dbState.getOrDefault(apiCollectionId, new APICatalog(apiCollectionId, new HashMap<>(), new HashMap<>())); - DbUpdateReturn dbUpdateReturn = getDBUpdatesForParams(deltaCatalog, dbCatalog, redact); + boolean redactCollectionLevel = this.apiCollectionToRedactPayload.getOrDefault(apiCollectionId, false); + DbUpdateReturn dbUpdateReturn = getDBUpdatesForParams(deltaCatalog, dbCatalog, redact, redactCollectionLevel); writesForParams.addAll(dbUpdateReturn.bulkUpdatesForSingleTypeInfo); writesForSensitiveSampleData.addAll(dbUpdateReturn.bulkUpdatesForSampleData); writesForSensitiveParamInfo.addAll(dbUpdateReturn.bulkUpdatesForSensitiveParamInfo); @@ -1387,7 +1388,7 @@ public void syncWithDB(boolean syncImmediately, boolean fetchAllSTI) { deltaCatalog.setDeletedInfo(new ArrayList<>()); boolean forceUpdate = syncImmediately || counter % 10 == 0; - writesForSampleData.addAll(getDBUpdatesForSampleData(apiCollectionId, deltaCatalog,dbCatalog, redact, forceUpdate)); + writesForSampleData.addAll(getDBUpdatesForSampleData(apiCollectionId, deltaCatalog,dbCatalog, forceUpdate, redact, redactCollectionLevel)); } loggerMaker.infoAndAddToDb("adding " + writesForParams.size() + " updates for params", LogDb.RUNTIME); diff --git a/apps/api-runtime/src/main/java/com/akto/utils/RedactSampleData.java b/apps/api-runtime/src/main/java/com/akto/utils/RedactSampleData.java index f756d48ff0..ece424963c 100644 --- a/apps/api-runtime/src/main/java/com/akto/utils/RedactSampleData.java +++ b/apps/api-runtime/src/main/java/com/akto/utils/RedactSampleData.java @@ -1,9 +1,8 @@ package com.akto.utils; -import com.akto.dto.HttpRequestParams; -import com.akto.dto.HttpResponseParams; -import com.akto.dto.OriginalHttpRequest; -import com.akto.dto.OriginalHttpResponse; +import com.akto.dto.*; +import com.akto.dto.type.KeyTypes; +import com.akto.dto.type.SingleTypeInfo; import com.akto.parsers.HttpCallParser; import com.fasterxml.jackson.core.JsonFactory; import com.fasterxml.jackson.core.JsonParser; @@ -15,7 +14,6 @@ import com.fasterxml.jackson.databind.node.TextNode; import com.mongodb.BasicDBObject; -import java.io.IOException; import java.util.*; public class RedactSampleData { @@ -24,11 +22,85 @@ public class RedactSampleData { public static final String redactValue = "****"; + public static String redactIfRequired(String sample, boolean accountLevelRedact, boolean apiCollectionLevelRedact) throws Exception { + HttpResponseParams httpResponseParams = HttpCallParser.parseKafkaMessage(sample); + HttpResponseParams.Source source = httpResponseParams.getSource(); + if(source.equals(HttpResponseParams.Source.HAR) || source.equals(HttpResponseParams.Source.PCAP)) return sample; + if(accountLevelRedact || apiCollectionLevelRedact) return redact(httpResponseParams); + return redactMarkedDataTypes(httpResponseParams); + } + public static String redact(String sample) throws Exception { HttpResponseParams httpResponseParams = HttpCallParser.parseKafkaMessage(sample); return redact(httpResponseParams); } + public static String redactMarkedDataTypes(String sample) throws Exception { + HttpResponseParams httpResponseParams = HttpCallParser.parseKafkaMessage(sample); + return redactMarkedDataTypes(httpResponseParams); + } + + public static String redactMarkedDataTypes(HttpResponseParams httpResponseParams) throws Exception{ + + Map> responseHeaders = httpResponseParams.getHeaders(); + if (responseHeaders == null) responseHeaders = new HashMap<>(); + handleHeaders(responseHeaders); + + String responsePayload = httpResponseParams.getPayload(); + if (responsePayload == null) responsePayload = "{}"; + try { + JsonParser jp = factory.createParser(responsePayload); + JsonNode node = mapper.readTree(jp); + changeRedactedDataType(node, redactValue); + if (node != null) { + responsePayload = node.toString(); + } else { + responsePayload = "{}"; + } + } catch (Exception e) { + responsePayload = "{}"; + } + httpResponseParams.setPayload(responsePayload); + + Map> requestHeaders = httpResponseParams.requestParams.getHeaders(); + if (requestHeaders == null) requestHeaders = new HashMap<>(); + handleHeaders(requestHeaders); + + String requestPayload = httpResponseParams.requestParams.getPayload(); + if (requestPayload == null) requestPayload = "{}"; + try { + JsonParser jp = factory.createParser(requestPayload); + JsonNode node = mapper.readTree(jp); + changeRedactedDataType(node, redactValue); + if (node != null) { + requestPayload= node.toString(); + } else { + requestPayload = "{}"; + } + } catch (Exception e) { + requestPayload = "{}"; + } + + httpResponseParams.requestParams.setPayload(requestPayload); + + //TODO confirm if this is required or not + httpResponseParams.setSourceIP(redactValue); + + return convertHttpRespToOriginalString(httpResponseParams); + } + + private static void handleHeaders(Map> responseHeaders) { + Set>> entries = responseHeaders.entrySet(); + for(Map.Entry> entry : entries){ + String key = entry.getKey(); + List values = entry.getValue(); + SingleTypeInfo.SubType subType = KeyTypes.findSubType(values.get(0), values.get(0), null); + if(SingleTypeInfo.isRedacted(subType.getName())){ + responseHeaders.put(key, Collections.singletonList(redactValue)); + } + } + } + // never use this function directly. This alters the httpResponseParams public static String redact(HttpResponseParams httpResponseParams) throws Exception { // response headers @@ -112,6 +184,41 @@ public static void change(JsonNode parent, String newValue) { } + public static void changeRedactedDataType(JsonNode parent, String newValue) { + if (parent == null) return; + + if (parent.isArray()) { + ArrayNode arrayNode = (ArrayNode) parent; + for(int i = 0; i < arrayNode.size(); i++) { + JsonNode arrayElement = arrayNode.get(i); + if (arrayElement.isValueNode()) { + SingleTypeInfo.SubType subType = KeyTypes.findSubType(arrayElement.asText(), arrayElement.asText(), null); + if(SingleTypeInfo.isRedacted(subType.getName())){ + arrayNode.set(i, new TextNode(newValue)); + } + } else { + changeRedactedDataType(arrayElement, newValue); + } + } + } else { + Iterator fieldNames = parent.fieldNames(); + while(fieldNames.hasNext()) { + String f = fieldNames.next(); + JsonNode fieldValue = parent.get(f); + if (fieldValue.isValueNode()) { + SingleTypeInfo.SubType subType = KeyTypes.findSubType(fieldValue.asText(), fieldValue.asText(), null); + if(SingleTypeInfo.isRedacted(subType.getName())){ + ((ObjectNode) parent).put(f, newValue); + } + + } else { + changeRedactedDataType(fieldValue, newValue); + } + } + } + + } + public static String convertOriginalReqRespToString(OriginalHttpRequest request, OriginalHttpResponse response) { BasicDBObject req = new BasicDBObject(); if (request != null) { diff --git a/apps/api-runtime/src/test/java/com/akto/parsers/TestDump2.java b/apps/api-runtime/src/test/java/com/akto/parsers/TestDump2.java index 8b3fe5ebc0..1aeaf86107 100644 --- a/apps/api-runtime/src/test/java/com/akto/parsers/TestDump2.java +++ b/apps/api-runtime/src/test/java/com/akto/parsers/TestDump2.java @@ -128,10 +128,10 @@ public void testHappyPath() { aggr.addURL(httpResponseParams); sync.computeDelta(aggr, false, 0); - APICatalogSync.DbUpdateReturn dbUpdateReturn = sync.getDBUpdatesForParams(sync.getDelta(0), sync.getDbState(0), false); + APICatalogSync.DbUpdateReturn dbUpdateReturn = sync.getDBUpdatesForParams(sync.getDelta(0), sync.getDbState(0), false, false); assertEquals(15, dbUpdateReturn.bulkUpdatesForSingleTypeInfo.size()); assertEquals(2, sync.getDBUpdatesForTraffic(0, sync.getDelta(0)).size()); - assertEquals(1, sync.getDBUpdatesForSampleData(0, sync.getDelta(0), sync.getDbState(0),false,true).size()); + assertEquals(1, sync.getDBUpdatesForSampleData(0, sync.getDelta(0), sync.getDbState(0),true, false, false).size()); } @@ -157,7 +157,7 @@ public void simpleTestForSingleCollection(int collectionId, APICatalogSync sync) RequestTemplate respTemplate = reqTemplate.getResponseTemplates().get(resp.statusCode); assertEquals(1, respTemplate.getUserIds().size()); assertEquals(3, respTemplate.getParameters().size()); - APICatalogSync.DbUpdateReturn dbUpdateReturn = sync.getDBUpdatesForParams(sync.getDelta(collectionId), sync.getDbState(collectionId), false); + APICatalogSync.DbUpdateReturn dbUpdateReturn = sync.getDBUpdatesForParams(sync.getDelta(collectionId), sync.getDbState(collectionId), false, false); assertEquals(24, dbUpdateReturn.bulkUpdatesForSingleTypeInfo.size()); assertEquals(2, sync.getDBUpdatesForTraffic(collectionId, sync.getDelta(collectionId)).size()); } @@ -170,9 +170,9 @@ public void simpleTest() { simpleTestForSingleCollection(0, sync); simpleTestForSingleCollection(1, sync); simpleTestForSingleCollection(2, sync); - assertEquals(24, sync.getDBUpdatesForParams(sync.getDelta(0), sync.getDbState(0),false).bulkUpdatesForSingleTypeInfo.size()); - assertEquals(24, sync.getDBUpdatesForParams(sync.getDelta(1), sync.getDbState(1),false).bulkUpdatesForSingleTypeInfo.size()); - assertEquals(24, sync.getDBUpdatesForParams(sync.getDelta(2), sync.getDbState(2),false).bulkUpdatesForSingleTypeInfo.size()); + assertEquals(24, sync.getDBUpdatesForParams(sync.getDelta(0), sync.getDbState(0),false, false).bulkUpdatesForSingleTypeInfo.size()); + assertEquals(24, sync.getDBUpdatesForParams(sync.getDelta(1), sync.getDbState(1),false, false).bulkUpdatesForSingleTypeInfo.size()); + assertEquals(24, sync.getDBUpdatesForParams(sync.getDelta(2), sync.getDbState(2),false, false).bulkUpdatesForSingleTypeInfo.size()); } @Test @@ -311,7 +311,7 @@ public void repetitiveKeyTest() { respTemplate.tryMergeNodesInTrie(url, "POST", resp.statusCode, resp.getRequestParams().getApiCollectionId()); - List updates = sync.getDBUpdatesForParams(sync.getDelta(0), sync.getDbState(0), false).bulkUpdatesForSingleTypeInfo; + List updates = sync.getDBUpdatesForParams(sync.getDelta(0), sync.getDbState(0), false, false).bulkUpdatesForSingleTypeInfo; } @Test diff --git a/apps/api-runtime/src/test/java/com/akto/parsers/TestMergingNew.java b/apps/api-runtime/src/test/java/com/akto/parsers/TestMergingNew.java index 8e7cf5005a..d3d783dd29 100644 --- a/apps/api-runtime/src/test/java/com/akto/parsers/TestMergingNew.java +++ b/apps/api-runtime/src/test/java/com/akto/parsers/TestMergingNew.java @@ -725,7 +725,7 @@ private APICatalogSync.DbUpdateReturn cleanSync(HttpResponseParams httpResponseP parser.syncFunction(Collections.singletonList(httpResponseParams),false, true); APICatalogSync apiCatalogSync = parser.apiCatalogSync; return apiCatalogSync.getDBUpdatesForParams( - apiCatalogSync.getDelta(collectionId), apiCatalogSync.getDbState(collectionId), false + apiCatalogSync.getDelta(collectionId), apiCatalogSync.getDbState(collectionId), false, false ); } diff --git a/apps/api-runtime/src/test/java/com/akto/utils/TestRedactSampleData.java b/apps/api-runtime/src/test/java/com/akto/utils/TestRedactSampleData.java index 3aad9eea60..b053b91d43 100644 --- a/apps/api-runtime/src/test/java/com/akto/utils/TestRedactSampleData.java +++ b/apps/api-runtime/src/test/java/com/akto/utils/TestRedactSampleData.java @@ -1,5 +1,7 @@ package com.akto.utils; +import com.akto.dao.CustomDataTypeDao; +import com.akto.dto.CustomDataType; import com.akto.dto.HttpRequestParams; import com.akto.dto.HttpResponseParams; import com.akto.parsers.HttpCallParser; diff --git a/apps/dashboard/src/main/java/com/akto/action/ApiCollectionsAction.java b/apps/dashboard/src/main/java/com/akto/action/ApiCollectionsAction.java index 1238815c6d..cfc75beaed 100644 --- a/apps/dashboard/src/main/java/com/akto/action/ApiCollectionsAction.java +++ b/apps/dashboard/src/main/java/com/akto/action/ApiCollectionsAction.java @@ -5,22 +5,27 @@ import java.util.ArrayList; import java.util.HashSet; import java.util.List; -import com.akto.dao.APISpecDao; -import com.akto.dao.ApiCollectionsDao; -import com.akto.dao.SensitiveParamInfoDao; -import com.akto.dao.SingleTypeInfoDao; + +import com.akto.dao.*; import com.akto.dao.context.Context; import com.akto.dto.ApiCollection; +import com.akto.log.LoggerMaker; import com.akto.util.Constants; import com.mongodb.client.model.Filters; import com.mongodb.BasicDBObject; +import com.mongodb.client.model.Updates; import com.opensymphony.xwork2.Action; +import org.bson.conversions.Bson; public class ApiCollectionsAction extends UserAction { + private static final LoggerMaker loggerMaker = new LoggerMaker(ApiCollectionsAction.class); + List apiCollections = new ArrayList<>(); int apiCollectionId; + boolean redacted; + public String fetchAllCollections() { this.apiCollections = ApiCollectionsDao.instance.findAll(new BasicDBObject()); @@ -110,6 +115,39 @@ public String deleteMultipleCollections() { return SUCCESS.toUpperCase(); } + public static void dropSampleDataForApiCollection() { + loggerMaker.infoAndAddToDb("Dropping sample data for all api collections", LoggerMaker.LogDb.DASHBOARD); + List apiCollections = ApiCollectionsDao.instance.findAll(Filters.eq(ApiCollection.SAMPLE_COLLECTIONS_DROPPED, false)); + for (ApiCollection apiCollection: apiCollections) { + int apiCollectionId = apiCollection.getId(); + loggerMaker.infoAndAddToDb("Dropping sti data for api collection: " + apiCollectionId, LoggerMaker.LogDb.DASHBOARD); + SingleTypeInfoDao.instance.deleteAll(Filters.eq("_id", Collections.singletonList(apiCollectionId))); + loggerMaker.infoAndAddToDb("Dropping sample data for api collection: " + apiCollectionId, LoggerMaker.LogDb.DASHBOARD); + SampleDataDao.instance.deleteAll(Filters.eq("_id.apiCollectionId", apiCollectionId)); + loggerMaker.infoAndAddToDb("Dropping sensitive sample data for api collection: " + apiCollectionId, LoggerMaker.LogDb.DASHBOARD); + SensitiveSampleDataDao.instance.deleteAll(Filters.eq("_id.apiCollectionId", apiCollectionId)); + ApiCollectionsDao.instance.updateOneNoUpsert(Filters.eq("_id", apiCollectionId), Updates.set(ApiCollection.SAMPLE_COLLECTIONS_DROPPED, true)); + } + } + + public String redactCollection() { + List updates = Arrays.asList( + Updates.set(ApiCollection.REDACT, redacted), + Updates.set(ApiCollection.SAMPLE_COLLECTIONS_DROPPED, !redacted) + ); + ApiCollectionsDao.instance.updateOneNoUpsert(Filters.eq("_id", apiCollectionId), Updates.combine(updates)); + if(redacted){ + int accountId = Context.accountId.get(); + Runnable r = () -> { + Context.accountId.set(accountId); + loggerMaker.infoAndAddToDb("Triggered job to delete sample data", LoggerMaker.LogDb.DASHBOARD); + dropSampleDataForApiCollection(); + }; + new Thread(r).start(); + } + return SUCCESS.toUpperCase(); + } + public List getApiCollections() { return this.apiCollections; } @@ -128,6 +166,13 @@ public int getApiCollectionId() { public void setApiCollectionId(int apiCollectionId) { this.apiCollectionId = apiCollectionId; - } + } + + public boolean isRedacted() { + return redacted; + } + public void setRedacted(boolean redacted) { + this.redacted = redacted; + } } diff --git a/apps/dashboard/src/main/java/com/akto/action/CustomDataTypeAction.java b/apps/dashboard/src/main/java/com/akto/action/CustomDataTypeAction.java index 151f05ab7d..219d292018 100644 --- a/apps/dashboard/src/main/java/com/akto/action/CustomDataTypeAction.java +++ b/apps/dashboard/src/main/java/com/akto/action/CustomDataTypeAction.java @@ -29,7 +29,9 @@ import java.io.IOException; import java.util.*; +import java.util.stream.Collectors; +import static com.akto.dto.type.SingleTypeInfo.subTypeMap; import static com.akto.utils.Utils.extractJsonResponse; public class CustomDataTypeAction extends UserAction{ @@ -45,6 +47,7 @@ public class CustomDataTypeAction extends UserAction{ private String valueOperator; private List valueConditionFromUsers; + private boolean redacted; public static class ConditionFromUser { Predicate.Type type; @@ -108,7 +111,7 @@ public String fetchDataTypeNames() { for (CustomDataType cdt: customDataTypes) { allDataTypes.add(cdt.getName()); } - for (SingleTypeInfo.SubType subType: SingleTypeInfo.subTypeMap.values()) { + for (SingleTypeInfo.SubType subType: subTypeMap.values()) { allDataTypes.add(subType.getName()); } @@ -182,7 +185,9 @@ public String execute() { Updates.set(CustomDataType.VALUE_CONDITIONS,customDataType.getValueConditions()), Updates.set(CustomDataType.OPERATOR,customDataType.getOperator()), Updates.set(CustomDataType.TIMESTAMP,Context.now()), - Updates.set(CustomDataType.ACTIVE,active) + Updates.set(CustomDataType.ACTIVE,active), + Updates.set(CustomDataType.REDACTED,customDataType.isRedacted()), + Updates.set(CustomDataType.SAMPLE_DATA_FIXED,customDataType.isSampleDataFixed()) ), options ); @@ -195,6 +200,16 @@ public String execute() { SingleTypeInfo.fetchCustomDataTypes(Context.accountId.get()); + if(redacted){ + int accountId = Context.accountId.get(); + Runnable r = () ->{ + Context.accountId.set(accountId); + loggerMaker.infoAndAddToDb("Triggered a job to fix existing data for " + aktoDataType.getName(), LogDb.DASHBOARD); + handleDataTypeRedaction(); + }; + new Thread(r).start(); + } + return Action.SUCCESS.toUpperCase(); } @@ -225,7 +240,9 @@ public String saveAktoDataType(){ Updates.combine( Updates.set("sensitiveAlways",sensitiveAlways), Updates.set("sensitivePosition",sensitivePositions), - Updates.set("timestamp",Context.now()) + Updates.set("timestamp",Context.now()), + Updates.set("redacted",redacted), + Updates.set(AktoDataType.SAMPLE_DATA_FIXED, !redacted) ), options ); @@ -236,10 +253,76 @@ public String saveAktoDataType(){ } SingleTypeInfo.fetchCustomDataTypes(Context.accountId.get()); + if(redacted){ + int accountId = Context.accountId.get(); + Runnable r = () ->{ + Context.accountId.set(accountId); + loggerMaker.infoAndAddToDb("Triggered a job to fix existing data for " + aktoDataType.getName(), LogDb.DASHBOARD); + handleDataTypeRedaction(); + }; + new Thread(r).start(); + } return Action.SUCCESS.toUpperCase(); } + public static void handleDataTypeRedaction(){ + try{ + loggerMaker.infoAndAddToDb("Dropping redacted data types for custom data types", LogDb.DASHBOARD); + List customDataTypesToBeFixed = CustomDataTypeDao.instance.findAll(Filters.eq(AktoDataType.SAMPLE_DATA_FIXED, false)); + loggerMaker.infoAndAddToDb("Found " + customDataTypesToBeFixed.size() + " custom data types to be fixed", LogDb.DASHBOARD); + + customDataTypesToBeFixed.forEach(cdt -> { + SingleTypeInfo.SubType st = cdt.toSubType(); + loggerMaker.infoAndAddToDb("Dropping redacted data types for custom data type:" + cdt.getName() + ", subType:" + st.getName(), LogDb.DASHBOARD); + handleRedactionForSubType(st); + CustomDataTypeDao.instance.updateOneNoUpsert(Filters.eq(CustomDataType.NAME, cdt.getName()), Updates.set(CustomDataType.SAMPLE_DATA_FIXED, true)); + loggerMaker.infoAndAddToDb("Dropped redacted data types for custom data type:" + cdt.getName() + ", subType:" + st.getName(), LogDb.DASHBOARD); + }); + + loggerMaker.infoAndAddToDb("Dropping redacted data types for akto data types", LogDb.DASHBOARD); + + List aktoDataTypesToBeFixed = AktoDataTypeDao.instance.findAll(Filters.eq(AktoDataType.SAMPLE_DATA_FIXED, false)); + loggerMaker.infoAndAddToDb("Found " + aktoDataTypesToBeFixed.size() + " akto data types to be fixed", LogDb.DASHBOARD); + aktoDataTypesToBeFixed.forEach(adt -> { + loggerMaker.infoAndAddToDb("Dropping redacted data types for akto data type:" + adt.getName(), LogDb.DASHBOARD); + SingleTypeInfo.SubType st = subTypeMap.get(adt.getName()); + handleRedactionForSubType(st); + AktoDataTypeDao.instance.updateOneNoUpsert(Filters.eq(AktoDataType.NAME, adt.getName()), Updates.set(AktoDataType.SAMPLE_DATA_FIXED, true)); + loggerMaker.infoAndAddToDb("Dropped redacted data types for akto data type:" + adt.getName(), LogDb.DASHBOARD); + }); + loggerMaker.infoAndAddToDb("Dropped redacted data types successfully!", LogDb.DASHBOARD); + } catch (Exception e){ + loggerMaker.errorAndAddToDb("Failed to drop redacted data types", LogDb.DASHBOARD); + } + + } + + private static void handleRedactionForSubType(SingleTypeInfo.SubType subType) { + loggerMaker.infoAndAddToDb("Dropping redacted data types for subType:" + subType.getName(), LogDb.DASHBOARD); + int skip = 0; + int limit = 100; + int itr = 1; + while (true) { + List apiInfoKeys = SingleTypeInfoDao.instance.fetchEndpointsInCollection2(subType, skip, limit); + loggerMaker.infoAndAddToDb("Found " + apiInfoKeys.size() + " apiInfoKeys for subType:" + subType.getName() + ", itr: " +itr, LogDb.DASHBOARD); + if(apiInfoKeys.isEmpty()){ + break; + } + List query = apiInfoKeys.stream().map(key -> Filters.and( + Filters.eq("apiCollectionId", key.getApiCollectionId()), + Filters.eq("url", key.getUrl()), + Filters.eq("method", key.getMethod()) + )).collect(Collectors.toList()); + + SampleDataDao.instance.updateManyNoUpsert(Filters.or(query), Updates.unset("samples")); + SensitiveSampleDataDao.instance.updateManyNoUpsert(Filters.or(query), Updates.unset("sampleData")); + loggerMaker.infoAndAddToDb("Dropped redacted data types for subType:" + subType.getName() + " for " + apiInfoKeys.size() + " apiInfoKeys, itr:" + itr, LogDb.DASHBOARD); + itr++; + skip += limit; + } + } + public List generatePositions(List sensitivePosition){ Set sensitivePositionSet = new HashSet<>(); @@ -436,7 +519,7 @@ public CustomDataType generateCustomDataType(int userId) throws AktoCustomExcept name = name.toUpperCase(); if (!(name.matches("[A-Z_0-9 ]+"))) throw new AktoCustomException("Name can only contain alphabets, spaces, numbers and underscores"); - if (SingleTypeInfo.subTypeMap.containsKey(name)) { + if (subTypeMap.containsKey(name)) { throw new AktoCustomException("Data type name reserved"); } @@ -512,7 +595,7 @@ public CustomDataType generateCustomDataType(int userId) throws AktoCustomExcept IgnoreData ignoreData = new IgnoreData(); return new CustomDataType(name, sensitiveAlways, sensitivePositions, userId, - true,keyConditions,valueConditions, mainOperator,ignoreData); + true,keyConditions,valueConditions, mainOperator,ignoreData, redacted, !redacted); } public void setCreateNew(boolean createNew) { @@ -737,4 +820,12 @@ public List getSensitivePosition() { public void setSensitivePosition(List sensitivePosition) { this.sensitivePosition = sensitivePosition; } + + public boolean getRedacted() { + return redacted; + } + + public void setRedacted(boolean redacted) { + this.redacted = redacted; + } } diff --git a/apps/dashboard/src/main/java/com/akto/action/observe/InventoryAction.java b/apps/dashboard/src/main/java/com/akto/action/observe/InventoryAction.java index 83c2dca10c..b0bb17e7a6 100644 --- a/apps/dashboard/src/main/java/com/akto/action/observe/InventoryAction.java +++ b/apps/dashboard/src/main/java/com/akto/action/observe/InventoryAction.java @@ -210,8 +210,9 @@ private void attachAPIInfoListInResponse(List list, int apiCollec } } + ApiCollection apiCollection = ApiCollectionsDao.instance.findOne(Filters.eq(Constants.ID, apiCollectionId)); - response.put("data", new BasicDBObject("endpoints", list).append("apiInfoList", apiInfoList)); + response.put("data", new BasicDBObject("endpoints", list).append("apiInfoList", apiInfoList).append("redacted", apiCollection.getRedact())); } public static String retrievePath(String url) { diff --git a/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java b/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java index 3648c6e32a..7bf79dd4a4 100644 --- a/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java +++ b/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java @@ -2,6 +2,8 @@ import com.akto.DaoInit; import com.akto.action.AdminSettingsAction; +import com.akto.action.ApiCollectionsAction; +import com.akto.action.CustomDataTypeAction; import com.akto.action.observe.InventoryAction; import com.akto.dao.*; import com.akto.dao.billing.OrganizationUsageDao; @@ -973,6 +975,8 @@ public static void dropSampleDataIfEarlierNotDroped(AccountSettings accountSetti if (accountSettings.isRedactPayload() && !accountSettings.isSampleDataCollectionDropped()) { AdminSettingsAction.dropCollections(Context.accountId.get()); } + ApiCollectionsAction.dropSampleDataForApiCollection(); + CustomDataTypeAction.handleDataTypeRedaction(); } diff --git a/apps/dashboard/src/main/resources/struts.xml b/apps/dashboard/src/main/resources/struts.xml index 566984bf3d..a80a3d7322 100644 --- a/apps/dashboard/src/main/resources/struts.xml +++ b/apps/dashboard/src/main/resources/struts.xml @@ -498,6 +498,16 @@ + + + + + + + 401 + + + diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/AllSensitiveData/AllSensitiveData.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/AllSensitiveData/AllSensitiveData.jsx index 9eb0f4af68..a816668c03 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/AllSensitiveData/AllSensitiveData.jsx +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/AllSensitiveData/AllSensitiveData.jsx @@ -105,7 +105,8 @@ function AllSensitiveData() { icon: CircleTickMajor, iconColor: "success", iconTooltip: "Active", - sensitiveCount:0 + sensitiveCount:0, + redacted: type.redacted }) }) res.dataTypes.customDataTypes.forEach((type) => { @@ -120,7 +121,8 @@ function AllSensitiveData() { icon: type.active ? CircleTickMajor : CircleCancelMajor, iconColor: type.active ? "success" : "critical", iconTooltip: type.active ? "Active" : "Inactive", - sensitiveCount:0 + sensitiveCount:0, + redacted: type.redacted }) }) setMapData(mapDataToKey) diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api.js b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api.js index ddaa0d9532..7cc4feb7b1 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api.js +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api.js @@ -218,6 +218,16 @@ export default { return resp }) }, + redactCollection(apiCollectionId, redacted){ + return request({ + url: '/api/redactCollection', + method: 'post', + data:{ + apiCollectionId: apiCollectionId, + redacted: redacted + } + }) + }, async fetchAllUrlsAndMethods (apiCollectionId) { const resp = await request({ diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/ApiEndpoints.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/ApiEndpoints.jsx index d61fc7c913..94f10faa73 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/ApiEndpoints.jsx +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/ApiEndpoints.jsx @@ -130,10 +130,13 @@ function ApiEndpoints() { const [prompts, setPrompts] = useState([]) const [isGptScreenActive, setIsGptScreenActive] = useState(false) const [isGptActive, setIsGptActive] = useState(false) + const [redacted, setIsRedacted] = useState(false) async function fetchData() { setLoading(true) let apiCollectionData = await api.fetchAPICollection(apiCollectionId) + console.log("data", apiCollectionData) + setIsRedacted(apiCollectionData.data.redacted) let apiEndpointsInCollection = apiCollectionData.data.endpoints.map(x => { return { ...x._id, startTs: x.startTs, changesCount: x.changesCount, shadow: x.shadow ? x.shadow : false } }) let apiInfoListInCollection = apiCollectionData.data.apiInfoList let unusedEndpointsInCollection = apiCollectionData.unusedEndpoints @@ -263,6 +266,14 @@ function ApiEndpoints() { func.setToast(true, false, "Endpoints refreshed") } + function redactCollection(){ + var updatedRedacted = !redacted; + api.redactCollection(apiCollectionId, updatedRedacted).then(resp => { + setIsRedacted(updatedRedacted) + func.setToast(true, false, updatedRedacted ? "Collection redacted" : "Collection unredacted") + }) + } + async function exportOpenApi() { let lastFetchedUrl = null; let lastFetchedMethod = null; @@ -392,6 +403,11 @@ function ApiEndpoints() { Refresh + + + { func.setToast(true, false, "Data type updated successfully"); @@ -264,13 +264,25 @@ function DataTypes() { -
- -
) - let components = (!isNew && currState.dataType === 'Akto') ? [descriptionCard, requestCard] : [descriptionCard, conditionsCard, requestCard] + const redactCard = ( + + + +

Redact this data type

+
+ + { handleChange({ redacted: val }) }}/> + + + + + ) + + let components = (!isNew && currState.dataType === 'Akto') ? [descriptionCard, requestCard, redactCard] : [descriptionCard, conditionsCard, requestCard, redactCard] return ( > modelList, BulkWriteOptions options) { return this.getMCollection().bulkWrite(modelList, options); } diff --git a/libs/dao/src/main/java/com/akto/dao/SingleTypeInfoDao.java b/libs/dao/src/main/java/com/akto/dao/SingleTypeInfoDao.java index 89523d68e4..6e03e7bc42 100644 --- a/libs/dao/src/main/java/com/akto/dao/SingleTypeInfoDao.java +++ b/libs/dao/src/main/java/com/akto/dao/SingleTypeInfoDao.java @@ -287,6 +287,7 @@ public void resetCount() { // to get results irrespective of collections use negative value for apiCollectionId public List fetchEndpointsInCollection(int apiCollectionId) { List pipeline = new ArrayList<>(); + ; BasicDBObject groupedId = new BasicDBObject("apiCollectionId", "$apiCollectionId") .append("url", "$url") @@ -326,6 +327,48 @@ public List fetchEndpointsInCollection(int apiCollectionId) return endpoints; } + public List fetchEndpointsInCollection2(SingleTypeInfo.SubType subType, int skip, int limit) { + List pipeline = new ArrayList<>(); + BasicDBObject groupedId = + new BasicDBObject("apiCollectionId", "$apiCollectionId") + .append("url", "$url") + .append("method", "$method"); + + + pipeline.add(Aggregates.match(Filters.eq("subType", subType.getName()))); + + Bson projections = Projections.fields( + Projections.include("timestamp", "apiCollectionId", "url", "method") + ); + + pipeline.add(Aggregates.project(projections)); + pipeline.add(Aggregates.group(groupedId)); + pipeline.add(Aggregates.sort(Sorts.descending("timestamp"))); + pipeline.add(Aggregates.limit(limit)); + pipeline.add(Aggregates.skip(skip)); + + MongoCursor endpointsCursor = instance.getMCollection().aggregate(pipeline, BasicDBObject.class).cursor(); + + List endpoints = new ArrayList<>(); + while(endpointsCursor.hasNext()) { + BasicDBObject v = endpointsCursor.next(); + try { + BasicDBObject vv = (BasicDBObject) v.get("_id"); + ApiInfo.ApiInfoKey apiInfoKey = new ApiInfo.ApiInfoKey( + (int) vv.get("apiCollectionId"), + (String) vv.get("url"), + URLMethods.Method.fromString((String) vv.get("method")) + ); + endpoints.add(apiInfoKey); + } catch (Exception e) { + ; + + } + } + + return endpoints; + } + public List fetchStiOfCollections(List apiCollectionIds) { Bson filters = Filters.in(SingleTypeInfo._API_COLLECTION_ID, apiCollectionIds); return instance.findAll(filters); diff --git a/libs/dao/src/main/java/com/akto/dto/AktoDataType.java b/libs/dao/src/main/java/com/akto/dto/AktoDataType.java index 0bed45b51d..8ed57aeb96 100644 --- a/libs/dao/src/main/java/com/akto/dto/AktoDataType.java +++ b/libs/dao/src/main/java/com/akto/dto/AktoDataType.java @@ -6,10 +6,15 @@ public class AktoDataType { private String name; + + public static final String NAME = "name"; private boolean sensitiveAlways; private List sensitivePosition; private int timestamp; private IgnoreData ignoreData; + private boolean redacted; + public static final String SAMPLE_DATA_FIXED = "sampleDataFixed"; + private boolean sampleDataFixed; public AktoDataType() { } @@ -18,6 +23,17 @@ public AktoDataType(String name, boolean sensitiveAlways, List sensitivePosition,int timestamp, IgnoreData ignoreData, boolean redacted, boolean sampleDataFixed) { + this.name = name; + this.sensitiveAlways = sensitiveAlways; + this.sensitivePosition = sensitivePosition; + this.ignoreData = ignoreData; + this.redacted = redacted; + this.sampleDataFixed = sampleDataFixed; } public String getName() { return name; @@ -52,4 +68,19 @@ public IgnoreData getIgnoreData() { public void setIgnoreData(IgnoreData ignoreData) { this.ignoreData = ignoreData; } + + public boolean isRedacted() { + return redacted; + } + public void setRedacted(boolean redacted) { + this.redacted = redacted; + } + + public boolean isSampleDataFixed() { + return sampleDataFixed; + } + + public void setSampleDataFixed(boolean sampleDataFixed) { + this.sampleDataFixed = sampleDataFixed; + } } diff --git a/libs/dao/src/main/java/com/akto/dto/ApiCollection.java b/libs/dao/src/main/java/com/akto/dto/ApiCollection.java index 56b8a6c0a0..1c349474ba 100644 --- a/libs/dao/src/main/java/com/akto/dto/ApiCollection.java +++ b/libs/dao/src/main/java/com/akto/dto/ApiCollection.java @@ -21,6 +21,13 @@ public class ApiCollection { public static final String HOST_NAME = "hostName"; int vxlanId; + boolean redact; + public static final String REDACT = "redact"; + + boolean sampleCollectionsDropped; + + public static final String SAMPLE_COLLECTIONS_DROPPED = "sampleCollectionsDropped"; + @BsonIgnore int urlsCount; public static final String VXLAN_ID = "vxlanId"; @@ -35,6 +42,19 @@ public ApiCollection(int id, String name, int startTs, Set urls, String this.urls = urls; this.hostName = hostName; this.vxlanId = vxlanId; + this.redact = false; + this.sampleCollectionsDropped = true; + } + + public ApiCollection(int id, String name, int startTs, Set urls, String hostName, int vxlanId, boolean redact, boolean sampleCollectionsDropped) { + this.id = id; + this.name = name; + this.startTs = startTs; + this.urls = urls; + this.hostName = hostName; + this.vxlanId = vxlanId; + this.redact = redact; + this.sampleCollectionsDropped = sampleCollectionsDropped; } public static boolean useHost = Objects.equals(System.getenv("USE_HOSTNAME"), "true"); @@ -127,4 +147,19 @@ public static ApiCollection createManualCollection(int id, String name){ return new ApiCollection(id, name, Context.now() , new HashSet<>(), null, 0); } + public boolean getRedact() { + return redact; + } + + public void setRedact(boolean redact) { + this.redact = redact; + } + + public boolean isSampleCollectionsDropped() { + return sampleCollectionsDropped; + } + + public void setSampleCollectionsDropped(boolean sampleCollectionsDropped) { + this.sampleCollectionsDropped = sampleCollectionsDropped; + } } diff --git a/libs/dao/src/main/java/com/akto/dto/CustomDataType.java b/libs/dao/src/main/java/com/akto/dto/CustomDataType.java index 739f0b1f70..3276db14c0 100644 --- a/libs/dao/src/main/java/com/akto/dto/CustomDataType.java +++ b/libs/dao/src/main/java/com/akto/dto/CustomDataType.java @@ -31,6 +31,10 @@ public class CustomDataType { Conditions.Operator operator; public static final String IGNORE_DATA = "ignoreData"; private IgnoreData ignoreData; + private boolean redacted; + public static final String REDACTED = "redacted"; + private boolean sampleDataFixed; + public static final String SAMPLE_DATA_FIXED = "sampleDataFixed"; public CustomDataType() { } @@ -45,6 +49,22 @@ public CustomDataType(String name, boolean sensitiveAlways, List sensitivePosition, int creatorId, boolean active, Conditions keyConditions, Conditions valueConditions, Conditions.Operator operator, IgnoreData ignoreData, boolean redacted, boolean sampleDataFixed) { + this.name = name; + this.sensitiveAlways = sensitiveAlways; + this.sensitivePosition = sensitivePosition; + this.creatorId = creatorId; + this.timestamp = Context.now(); + this.active = active; + this.keyConditions = keyConditions; + this.valueConditions = valueConditions; + this.operator = operator; + this.ignoreData = ignoreData; + this.redacted = redacted; + this.sampleDataFixed = sampleDataFixed; } public SingleTypeInfo.SubType toSubType() { @@ -191,6 +211,25 @@ public String toString() { ", keyConditions='" + getKeyConditions() + "'" + ", valueConditions='" + getValueConditions() + "'" + ", operator='" + getOperator() + "'" + + ", ignoreData='" + getIgnoreData() + "'" + + ", redacted='" + isRedacted() + "'" + + ", sampleDataFixed='" + isSampleDataFixed() + "'" + "}"; } + + public boolean isRedacted() { + return redacted; + } + + public void setRedacted(boolean redacted) { + this.redacted = redacted; + } + + public boolean isSampleDataFixed() { + return sampleDataFixed; + } + + public void setSampleDataFixed(boolean sampleDataFixed) { + this.sampleDataFixed = sampleDataFixed; + } } diff --git a/libs/dao/src/main/java/com/akto/dto/type/AccountDataTypesInfo.java b/libs/dao/src/main/java/com/akto/dto/type/AccountDataTypesInfo.java index f32e2b5ea7..d4764515a9 100644 --- a/libs/dao/src/main/java/com/akto/dto/type/AccountDataTypesInfo.java +++ b/libs/dao/src/main/java/com/akto/dto/type/AccountDataTypesInfo.java @@ -1,9 +1,6 @@ package com.akto.dto.type; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; import com.akto.dto.AktoDataType; import com.akto.dto.CustomDataType; @@ -13,11 +10,14 @@ public class AccountDataTypesInfo { private Map customDataTypeMap; private List customDataTypesSortedBySensitivity; + private Set redactedDataTypes; + private Map aktoDataTypeMap = new HashMap<>(); public AccountDataTypesInfo() { this.customDataTypeMap = new HashMap<>(); this.customDataTypesSortedBySensitivity = new ArrayList<>(); this.aktoDataTypeMap = new HashMap<>(); + this.redactedDataTypes = new HashSet<>(); } public AccountDataTypesInfo(Map customDataTypeMap, List customDataTypesSortedBySensitivity) { @@ -46,6 +46,7 @@ public String toString() { return "{" + " customDataTypeMap='" + getCustomDataTypeMap() + "'" + ", customDataTypesSortedBySensitivity='" + getCustomDataTypesSortedBySensitivity() + "'" + + ", redactedDataTypes='" + getRedactedDataTypes() + "'" + "}"; } @@ -56,4 +57,12 @@ public Map getAktoDataTypeMap() { public void setAktoDataTypeMap(Map aktoDataTypeMap) { this.aktoDataTypeMap = aktoDataTypeMap; } + + public Set getRedactedDataTypes() { + return redactedDataTypes; + } + + public void setRedactedDataTypes(Set redactedDataTypes) { + this.redactedDataTypes = redactedDataTypes; + } } diff --git a/libs/dao/src/main/java/com/akto/dto/type/SingleTypeInfo.java b/libs/dao/src/main/java/com/akto/dto/type/SingleTypeInfo.java index 83a90a7939..ec8024d6e0 100644 --- a/libs/dao/src/main/java/com/akto/dto/type/SingleTypeInfo.java +++ b/libs/dao/src/main/java/com/akto/dto/type/SingleTypeInfo.java @@ -91,6 +91,7 @@ public static void fetchCustomDataTypes(int accountId) { Map newMap = new HashMap<>(); List sensitiveCustomDataType = new ArrayList<>(); List nonSensitiveCustomDataType = new ArrayList<>(); + Set redactedDataTypes = new HashSet<>(); for (CustomDataType customDataType: customDataTypes) { newMap.put(customDataType.getName(), customDataType); if (customDataType.isSensitiveAlways() || customDataType.getSensitivePosition().size()>0) { @@ -98,6 +99,9 @@ public static void fetchCustomDataTypes(int accountId) { } else { nonSensitiveCustomDataType.add(customDataType); } + if (customDataType.isRedacted()) { + redactedDataTypes.add(customDataType.getName()); + } } accountToDataTypesInfo.putIfAbsent(accountId, new AccountDataTypesInfo()); @@ -114,9 +118,21 @@ public static void fetchCustomDataTypes(int accountId) { newAktoMap.put(aktoDataType.getName(), aktoDataType); subTypeMap.get(aktoDataType.getName()).setSensitiveAlways(aktoDataType.getSensitiveAlways()); subTypeMap.get(aktoDataType.getName()).setSensitivePosition(aktoDataType.getSensitivePosition()); + if(aktoDataType.isRedacted()){ + redactedDataTypes.add(aktoDataType.getName()); + } } } info.setAktoDataTypeMap(newAktoMap); + info.setRedactedDataTypes(redactedDataTypes); + } + + public static boolean isRedacted(String dataTypeName){ + if (accountToDataTypesInfo.containsKey(Context.accountId.get())) { + return accountToDataTypesInfo.get(Context.accountId.get()).getRedactedDataTypes().contains(dataTypeName); + } else { + return false; + } } public static void fetchCustomAuthTypes(int accountId) { From 57f69d31ce43f2430e6e3a6342d43301e3acb70b Mon Sep 17 00:00:00 2001 From: aktoboy Date: Fri, 5 Jan 2024 22:27:41 +0530 Subject: [PATCH 04/39] Commented failing test [To be reverted] --- apps/api-runtime/src/test/java/com/akto/parsers/TestDBSync.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/api-runtime/src/test/java/com/akto/parsers/TestDBSync.java b/apps/api-runtime/src/test/java/com/akto/parsers/TestDBSync.java index 7e8dd00a26..a709321249 100644 --- a/apps/api-runtime/src/test/java/com/akto/parsers/TestDBSync.java +++ b/apps/api-runtime/src/test/java/com/akto/parsers/TestDBSync.java @@ -110,7 +110,7 @@ public void testParameterizedURL() { assertEquals(3, respTemplate.getParameters().size()); } - @Test +// @Test public void testImmediateSync() { testInitializer(); String url = "immediate/"; From 51f4b49a6ea9e415bd9ce069ea2a557a3b248ace Mon Sep 17 00:00:00 2001 From: aktoboy Date: Fri, 5 Jan 2024 22:47:51 +0530 Subject: [PATCH 05/39] Fixed failing tests --- apps/api-runtime/src/test/java/com/akto/parsers/TestDump2.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/apps/api-runtime/src/test/java/com/akto/parsers/TestDump2.java b/apps/api-runtime/src/test/java/com/akto/parsers/TestDump2.java index 1aeaf86107..bed83152fd 100644 --- a/apps/api-runtime/src/test/java/com/akto/parsers/TestDump2.java +++ b/apps/api-runtime/src/test/java/com/akto/parsers/TestDump2.java @@ -11,6 +11,7 @@ import java.util.Map; import java.util.Set; +import com.akto.MongoBasedTest; import com.akto.dao.context.Context; import com.akto.dto.type.*; import com.akto.dto.type.SingleTypeInfo.SubType; @@ -27,7 +28,7 @@ import org.junit.Test; -public class TestDump2 { +public class TestDump2 extends MongoBasedTest { private final int ACCOUNT_ID = 1_000_000; public void testInitializer(){ From 5d26336122aee7c6f6d4bc98ccf237a42ebcfafd Mon Sep 17 00:00:00 2001 From: aktoboy Date: Fri, 5 Jan 2024 23:31:17 +0530 Subject: [PATCH 06/39] Added changes to build runtime --- .github/workflows/staging.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/staging.yml b/.github/workflows/staging.yml index 33d164e7a0..6685ceddf7 100644 --- a/.github/workflows/staging.yml +++ b/.github/workflows/staging.yml @@ -76,6 +76,8 @@ jobs: docker buildx build --platform linux/arm64/v8,linux/amd64 -t $ECR_REGISTRY/akto-billing:$IMAGE_TAG . --push cd ../internal docker buildx build --platform linux/arm64/v8,linux/amd64 -t $ECR_REGISTRY/akto-internal:$IMAGE_TAG . --push + cd ../api-runtime + docker buildx build --platform linux/arm64/v8,linux/amd64 -t $ECR_REGISTRY/akto-api-security-runtime:$IMAGE_TAG . --push - name: Set up JDK 11 uses: actions/setup-java@v1 From f6ff85e03285f0f2597d478ed14d9a5cc4fb05d4 Mon Sep 17 00:00:00 2001 From: aktoboy Date: Sat, 6 Jan 2024 15:38:41 +0530 Subject: [PATCH 07/39] Fixed bugs --- .../java/com/akto/runtime/APICatalogSync.java | 1 + .../java/com/akto/utils/RedactSampleData.java | 18 +++++++++--------- .../com/akto/action/ApiCollectionsAction.java | 7 ++++++- .../com/akto/action/CustomDataTypeAction.java | 6 +++--- .../main/java/com/akto/dto/CustomDataType.java | 1 + .../main/java/com/akto/dto/type/KeyTypes.java | 5 +++-- 6 files changed, 23 insertions(+), 15 deletions(-) diff --git a/apps/api-runtime/src/main/java/com/akto/runtime/APICatalogSync.java b/apps/api-runtime/src/main/java/com/akto/runtime/APICatalogSync.java index d517a521c1..8ef8213515 100644 --- a/apps/api-runtime/src/main/java/com/akto/runtime/APICatalogSync.java +++ b/apps/api-runtime/src/main/java/com/akto/runtime/APICatalogSync.java @@ -940,6 +940,7 @@ public ArrayList> getDBUpdatesForSampleData(int apiCollec String redactedSample = RedactSampleData.redactIfRequired(s, accountLevelRedact, apiCollectionLevelRedact); finalSamples.add(redactedSample); } catch (Exception e) { + loggerMaker.errorAndAddToDb(e,"Error while redacting data" , LogDb.RUNTIME) ; } } diff --git a/apps/api-runtime/src/main/java/com/akto/utils/RedactSampleData.java b/apps/api-runtime/src/main/java/com/akto/utils/RedactSampleData.java index ece424963c..df1cbe9814 100644 --- a/apps/api-runtime/src/main/java/com/akto/utils/RedactSampleData.java +++ b/apps/api-runtime/src/main/java/com/akto/utils/RedactSampleData.java @@ -25,7 +25,7 @@ public class RedactSampleData { public static String redactIfRequired(String sample, boolean accountLevelRedact, boolean apiCollectionLevelRedact) throws Exception { HttpResponseParams httpResponseParams = HttpCallParser.parseKafkaMessage(sample); HttpResponseParams.Source source = httpResponseParams.getSource(); - if(source.equals(HttpResponseParams.Source.HAR) || source.equals(HttpResponseParams.Source.PCAP)) return sample; +// if(source.equals(HttpResponseParams.Source.HAR) || source.equals(HttpResponseParams.Source.PCAP)) return sample; if(accountLevelRedact || apiCollectionLevelRedact) return redact(httpResponseParams); return redactMarkedDataTypes(httpResponseParams); } @@ -51,7 +51,7 @@ public static String redactMarkedDataTypes(HttpResponseParams httpResponseParams try { JsonParser jp = factory.createParser(responsePayload); JsonNode node = mapper.readTree(jp); - changeRedactedDataType(node, redactValue); + changeRedactedDataType(null, node, redactValue); if (node != null) { responsePayload = node.toString(); } else { @@ -71,7 +71,7 @@ public static String redactMarkedDataTypes(HttpResponseParams httpResponseParams try { JsonParser jp = factory.createParser(requestPayload); JsonNode node = mapper.readTree(jp); - changeRedactedDataType(node, redactValue); + changeRedactedDataType(null, node, redactValue); if (node != null) { requestPayload= node.toString(); } else { @@ -94,7 +94,7 @@ private static void handleHeaders(Map> responseHeaders) { for(Map.Entry> entry : entries){ String key = entry.getKey(); List values = entry.getValue(); - SingleTypeInfo.SubType subType = KeyTypes.findSubType(values.get(0), values.get(0), null); + SingleTypeInfo.SubType subType = KeyTypes.findSubType(values.get(0), key, null); if(SingleTypeInfo.isRedacted(subType.getName())){ responseHeaders.put(key, Collections.singletonList(redactValue)); } @@ -184,7 +184,7 @@ public static void change(JsonNode parent, String newValue) { } - public static void changeRedactedDataType(JsonNode parent, String newValue) { + public static void changeRedactedDataType(String parentName, JsonNode parent, String newValue) { if (parent == null) return; if (parent.isArray()) { @@ -192,12 +192,12 @@ public static void changeRedactedDataType(JsonNode parent, String newValue) { for(int i = 0; i < arrayNode.size(); i++) { JsonNode arrayElement = arrayNode.get(i); if (arrayElement.isValueNode()) { - SingleTypeInfo.SubType subType = KeyTypes.findSubType(arrayElement.asText(), arrayElement.asText(), null); + SingleTypeInfo.SubType subType = KeyTypes.findSubType(arrayElement.asText(), parentName, null); if(SingleTypeInfo.isRedacted(subType.getName())){ arrayNode.set(i, new TextNode(newValue)); } } else { - changeRedactedDataType(arrayElement, newValue); + changeRedactedDataType(parentName, arrayElement, newValue); } } } else { @@ -206,13 +206,13 @@ public static void changeRedactedDataType(JsonNode parent, String newValue) { String f = fieldNames.next(); JsonNode fieldValue = parent.get(f); if (fieldValue.isValueNode()) { - SingleTypeInfo.SubType subType = KeyTypes.findSubType(fieldValue.asText(), fieldValue.asText(), null); + SingleTypeInfo.SubType subType = KeyTypes.findSubType(fieldValue.asText(), f, null); if(SingleTypeInfo.isRedacted(subType.getName())){ ((ObjectNode) parent).put(f, newValue); } } else { - changeRedactedDataType(fieldValue, newValue); + changeRedactedDataType(f, fieldValue, newValue); } } } diff --git a/apps/dashboard/src/main/java/com/akto/action/ApiCollectionsAction.java b/apps/dashboard/src/main/java/com/akto/action/ApiCollectionsAction.java index 197d6c3b40..2a53760818 100644 --- a/apps/dashboard/src/main/java/com/akto/action/ApiCollectionsAction.java +++ b/apps/dashboard/src/main/java/com/akto/action/ApiCollectionsAction.java @@ -119,8 +119,12 @@ public String deleteMultipleCollections() { } public static void dropSampleDataForApiCollection() { - loggerMaker.infoAndAddToDb("Dropping sample data for all api collections", LoggerMaker.LogDb.DASHBOARD); List apiCollections = ApiCollectionsDao.instance.findAll(Filters.eq(ApiCollection.SAMPLE_COLLECTIONS_DROPPED, false)); + if(apiCollections.isEmpty()) { + loggerMaker.infoAndAddToDb("No api collections to drop sample data for", LoggerMaker.LogDb.DASHBOARD); + return; + } + loggerMaker.infoAndAddToDb(String.format("Dropping sample data for %d api collections", apiCollections.size()), LoggerMaker.LogDb.DASHBOARD); for (ApiCollection apiCollection: apiCollections) { int apiCollectionId = apiCollection.getId(); loggerMaker.infoAndAddToDb("Dropping sti data for api collection: " + apiCollectionId, LoggerMaker.LogDb.DASHBOARD); @@ -131,6 +135,7 @@ public static void dropSampleDataForApiCollection() { SensitiveSampleDataDao.instance.deleteAll(Filters.eq("_id.apiCollectionId", apiCollectionId)); ApiCollectionsDao.instance.updateOneNoUpsert(Filters.eq("_id", apiCollectionId), Updates.set(ApiCollection.SAMPLE_COLLECTIONS_DROPPED, true)); } + loggerMaker.infoAndAddToDb(String.format("Dropped sample data for %d api collections", apiCollections.size()), LoggerMaker.LogDb.DASHBOARD); } public String redactCollection() { diff --git a/apps/dashboard/src/main/java/com/akto/action/CustomDataTypeAction.java b/apps/dashboard/src/main/java/com/akto/action/CustomDataTypeAction.java index 219d292018..74299d0e01 100644 --- a/apps/dashboard/src/main/java/com/akto/action/CustomDataTypeAction.java +++ b/apps/dashboard/src/main/java/com/akto/action/CustomDataTypeAction.java @@ -310,9 +310,9 @@ private static void handleRedactionForSubType(SingleTypeInfo.SubType subType) { break; } List query = apiInfoKeys.stream().map(key -> Filters.and( - Filters.eq("apiCollectionId", key.getApiCollectionId()), - Filters.eq("url", key.getUrl()), - Filters.eq("method", key.getMethod()) + Filters.eq("_id.apiCollectionId", key.getApiCollectionId()), + Filters.eq("_id.url", key.getUrl()), + Filters.eq("_id.method", key.getMethod()) )).collect(Collectors.toList()); SampleDataDao.instance.updateManyNoUpsert(Filters.or(query), Updates.unset("samples")); diff --git a/libs/dao/src/main/java/com/akto/dto/CustomDataType.java b/libs/dao/src/main/java/com/akto/dto/CustomDataType.java index 3276db14c0..412031726c 100644 --- a/libs/dao/src/main/java/com/akto/dto/CustomDataType.java +++ b/libs/dao/src/main/java/com/akto/dto/CustomDataType.java @@ -50,6 +50,7 @@ public CustomDataType(String name, boolean sensitiveAlways, List sensitivePosition, int creatorId, boolean active, Conditions keyConditions, Conditions valueConditions, Conditions.Operator operator, IgnoreData ignoreData, boolean redacted, boolean sampleDataFixed) { diff --git a/libs/dao/src/main/java/com/akto/dto/type/KeyTypes.java b/libs/dao/src/main/java/com/akto/dto/type/KeyTypes.java index f53fb6462a..5ba643fa81 100644 --- a/libs/dao/src/main/java/com/akto/dto/type/KeyTypes.java +++ b/libs/dao/src/main/java/com/akto/dto/type/KeyTypes.java @@ -174,8 +174,9 @@ public static SubType findSubType(Object o,String key, ParamId paramId) { if (o instanceof String) { String str = o.toString(); - for(SubType subType: patternToSubType.keySet()) { - Pattern pattern = patternToSubType.get(subType); + for(Map.Entry entry: patternToSubType.entrySet()) { + Pattern pattern = entry.getValue(); + SubType subType = entry.getKey(); if( ( checkForSubtypes || subType.getName().equals("URL") ) && pattern.matcher(str).matches()) { return subType; } From 88198e1b548b479b2fffdcdfd0c47e92fe4b6034 Mon Sep 17 00:00:00 2001 From: aktoboy Date: Sat, 6 Jan 2024 16:08:12 +0530 Subject: [PATCH 08/39] Moved api collection redact info logic --- .../main/java/com/akto/runtime/APICatalogSync.java | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/apps/api-runtime/src/main/java/com/akto/runtime/APICatalogSync.java b/apps/api-runtime/src/main/java/com/akto/runtime/APICatalogSync.java index 8ef8213515..c5f545815b 100644 --- a/apps/api-runtime/src/main/java/com/akto/runtime/APICatalogSync.java +++ b/apps/api-runtime/src/main/java/com/akto/runtime/APICatalogSync.java @@ -45,7 +45,6 @@ public class APICatalogSync { public Map sensitiveParamInfoBooleanMap; public static boolean mergeAsyncOutside = false; - public Map apiCollectionToRedactPayload; public APICatalogSync(String userIdentifier,int thresh) { this.thresh = thresh; @@ -61,11 +60,6 @@ public APICatalogSync(String userIdentifier,int thresh) { } catch (Exception e) { } - apiCollectionToRedactPayload = new HashMap<>(); - List all = ApiCollectionsDao.instance.findAll(new BasicDBObject()); - for(ApiCollection apiCollection: all) { - apiCollectionToRedactPayload.put(apiCollection.getId(), apiCollection.getRedact()); - } } @@ -1368,6 +1362,11 @@ public void syncWithDB(boolean syncImmediately, boolean fetchAllSTI) { List> writesForTraffic = new ArrayList<>(); List> writesForSampleData = new ArrayList<>(); List> writesForSensitiveParamInfo = new ArrayList<>(); + Map apiCollectionToRedactPayload = new HashMap<>(); + List all = ApiCollectionsDao.instance.findAll(new BasicDBObject()); + for(ApiCollection apiCollection: all) { + apiCollectionToRedactPayload.put(apiCollection.getId(), apiCollection.getRedact()); + } AccountSettings accountSettings = AccountSettingsDao.instance.findOne(AccountSettingsDao.generateFilter()); @@ -1380,7 +1379,7 @@ public void syncWithDB(boolean syncImmediately, boolean fetchAllSTI) { for(int apiCollectionId: this.delta.keySet()) { APICatalog deltaCatalog = this.delta.get(apiCollectionId); APICatalog dbCatalog = this.dbState.getOrDefault(apiCollectionId, new APICatalog(apiCollectionId, new HashMap<>(), new HashMap<>())); - boolean redactCollectionLevel = this.apiCollectionToRedactPayload.getOrDefault(apiCollectionId, false); + boolean redactCollectionLevel = apiCollectionToRedactPayload.getOrDefault(apiCollectionId, false); DbUpdateReturn dbUpdateReturn = getDBUpdatesForParams(deltaCatalog, dbCatalog, redact, redactCollectionLevel); writesForParams.addAll(dbUpdateReturn.bulkUpdatesForSingleTypeInfo); writesForSensitiveSampleData.addAll(dbUpdateReturn.bulkUpdatesForSampleData); From 041922ba31de04193684fd560ae3d86d94053bb5 Mon Sep 17 00:00:00 2001 From: aktoboy Date: Sat, 6 Jan 2024 19:56:23 +0530 Subject: [PATCH 09/39] Fixed bugs --- .../main/java/com/akto/action/CustomDataTypeAction.java | 2 +- .../main/java/com/akto/action/observe/InventoryAction.java | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/apps/dashboard/src/main/java/com/akto/action/CustomDataTypeAction.java b/apps/dashboard/src/main/java/com/akto/action/CustomDataTypeAction.java index 74299d0e01..5a124e9478 100644 --- a/apps/dashboard/src/main/java/com/akto/action/CustomDataTypeAction.java +++ b/apps/dashboard/src/main/java/com/akto/action/CustomDataTypeAction.java @@ -204,7 +204,7 @@ public String execute() { int accountId = Context.accountId.get(); Runnable r = () ->{ Context.accountId.set(accountId); - loggerMaker.infoAndAddToDb("Triggered a job to fix existing data for " + aktoDataType.getName(), LogDb.DASHBOARD); + loggerMaker.infoAndAddToDb("Triggered a job to fix existing custom data types", LogDb.DASHBOARD); handleDataTypeRedaction(); }; new Thread(r).start(); diff --git a/apps/dashboard/src/main/java/com/akto/action/observe/InventoryAction.java b/apps/dashboard/src/main/java/com/akto/action/observe/InventoryAction.java index b0bb17e7a6..6e752dab3f 100644 --- a/apps/dashboard/src/main/java/com/akto/action/observe/InventoryAction.java +++ b/apps/dashboard/src/main/java/com/akto/action/observe/InventoryAction.java @@ -210,9 +210,12 @@ private void attachAPIInfoListInResponse(List list, int apiCollec } } - ApiCollection apiCollection = ApiCollectionsDao.instance.findOne(Filters.eq(Constants.ID, apiCollectionId)); + response.put("data", new BasicDBObject("endpoints", list).append("apiInfoList", apiInfoList)); - response.put("data", new BasicDBObject("endpoints", list).append("apiInfoList", apiInfoList).append("redacted", apiCollection.getRedact())); + if(apiCollectionId != -1){ + ApiCollection apiCollection = ApiCollectionsDao.instance.findOne(Filters.eq(Constants.ID, apiCollectionId)); + response.put("redacted", apiCollection.getRedact()); + } } public static String retrievePath(String url) { From 1db0f63f4e06e7f643eec1554c2c88ff5aa351b6 Mon Sep 17 00:00:00 2001 From: aktoboy Date: Mon, 8 Jan 2024 17:23:17 +0530 Subject: [PATCH 10/39] Fix for existing data --- .../com/akto/action/ApiCollectionsAction.java | 50 +++++++++++-- .../com/akto/action/CustomDataTypeAction.java | 72 +++++++++++++++---- .../pages/observe/data_types/transform.js | 2 +- 3 files changed, 105 insertions(+), 19 deletions(-) diff --git a/apps/dashboard/src/main/java/com/akto/action/ApiCollectionsAction.java b/apps/dashboard/src/main/java/com/akto/action/ApiCollectionsAction.java index 2a53760818..363bbdfeb4 100644 --- a/apps/dashboard/src/main/java/com/akto/action/ApiCollectionsAction.java +++ b/apps/dashboard/src/main/java/com/akto/action/ApiCollectionsAction.java @@ -9,8 +9,13 @@ import com.akto.dao.*; import com.akto.dao.context.Context; import com.akto.dto.ApiCollection; +import com.akto.dto.ApiInfo; +import com.akto.dto.SensitiveSampleData; +import com.akto.dto.traffic.SampleData; +import com.akto.dto.type.URLMethods; import com.akto.log.LoggerMaker; import com.akto.util.Constants; +import com.akto.utils.RedactSampleData; import com.mongodb.client.model.Filters; import com.mongodb.BasicDBObject; import com.mongodb.client.model.Updates; @@ -127,12 +132,45 @@ public static void dropSampleDataForApiCollection() { loggerMaker.infoAndAddToDb(String.format("Dropping sample data for %d api collections", apiCollections.size()), LoggerMaker.LogDb.DASHBOARD); for (ApiCollection apiCollection: apiCollections) { int apiCollectionId = apiCollection.getId(); - loggerMaker.infoAndAddToDb("Dropping sti data for api collection: " + apiCollectionId, LoggerMaker.LogDb.DASHBOARD); - SingleTypeInfoDao.instance.deleteAll(Filters.eq("_id", Collections.singletonList(apiCollectionId))); - loggerMaker.infoAndAddToDb("Dropping sample data for api collection: " + apiCollectionId, LoggerMaker.LogDb.DASHBOARD); - SampleDataDao.instance.deleteAll(Filters.eq("_id.apiCollectionId", apiCollectionId)); - loggerMaker.infoAndAddToDb("Dropping sensitive sample data for api collection: " + apiCollectionId, LoggerMaker.LogDb.DASHBOARD); - SensitiveSampleDataDao.instance.deleteAll(Filters.eq("_id.apiCollectionId", apiCollectionId)); + List sampleDataList = SampleDataDao.instance.findAll(Filters.and(Filters.eq("_id.apiCollectionId", apiCollectionId), Filters.exists("samples", true))); + for(SampleData sampleData: sampleDataList){ + if(sampleData.getSamples() != null && !sampleData.getSamples().isEmpty()){ + boolean errorWhileRedacting = false; + List modifiedSamples = new ArrayList<>(); + for (String sample : sampleData.getSamples()) { + try{ + sample = RedactSampleData.redactIfRequired(sample, false, true); + } catch (Exception e){ + loggerMaker.errorAndAddToDb("Error while redacting sample data", LoggerMaker.LogDb.DASHBOARD); + errorWhileRedacting = true; + break; + } + modifiedSamples.add(sample); + } + SampleDataDao.instance.updateOneNoUpsert(Filters.eq("_id", sampleData.getId()), Updates.set("samples", errorWhileRedacting ? Collections.emptyList(): modifiedSamples)); + } + } + + List sensitiveSampleDataList = SensitiveSampleDataDao.instance.findAll(Filters.and(Filters.eq("_id.apiCollectionId", apiCollectionId), Filters.exists("sampleData", true))); + + for(SensitiveSampleData sensitiveSampleData: sensitiveSampleDataList){ + if(sensitiveSampleData.getSampleData() != null && !sensitiveSampleData.getSampleData().isEmpty()){ + List modifiedSamples = new ArrayList<>(); + boolean errorWhileRedacting = false; + for (String sample : sensitiveSampleData.getSampleData()) { + + try{ + sample = RedactSampleData.redactIfRequired(sample, false, true); + } catch (Exception e){ + loggerMaker.errorAndAddToDb("Error while redacting sample data", LoggerMaker.LogDb.DASHBOARD); + errorWhileRedacting = true; + break; + } + modifiedSamples.add(sample); + } + SensitiveSampleDataDao.instance.updateOneNoUpsert(Filters.eq("_id", sensitiveSampleData.getId()), Updates.set("sampleData", errorWhileRedacting ? Collections.emptyList(): modifiedSamples)); + } + } ApiCollectionsDao.instance.updateOneNoUpsert(Filters.eq("_id", apiCollectionId), Updates.set(ApiCollection.SAMPLE_COLLECTIONS_DROPPED, true)); } loggerMaker.infoAndAddToDb(String.format("Dropped sample data for %d api collections", apiCollections.size()), LoggerMaker.LogDb.DASHBOARD); diff --git a/apps/dashboard/src/main/java/com/akto/action/CustomDataTypeAction.java b/apps/dashboard/src/main/java/com/akto/action/CustomDataTypeAction.java index 5a124e9478..17e96d3b3a 100644 --- a/apps/dashboard/src/main/java/com/akto/action/CustomDataTypeAction.java +++ b/apps/dashboard/src/main/java/com/akto/action/CustomDataTypeAction.java @@ -14,6 +14,7 @@ import com.akto.parsers.HttpCallParser; import com.akto.util.JSONUtils; import com.akto.utils.AktoCustomException; +import com.akto.utils.RedactSampleData; import com.fasterxml.jackson.core.JsonFactory; import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.databind.JsonNode; @@ -24,6 +25,7 @@ import com.mongodb.client.MongoCursor; import com.mongodb.client.model.*; import com.opensymphony.xwork2.Action; +import io.swagger.models.auth.In; import org.apache.commons.lang3.EnumUtils; import org.bson.conversions.Bson; @@ -31,6 +33,7 @@ import java.util.*; import java.util.stream.Collectors; +import static com.akto.dto.type.SingleTypeInfo.fetchCustomDataTypes; import static com.akto.dto.type.SingleTypeInfo.subTypeMap; import static com.akto.utils.Utils.extractJsonResponse; @@ -268,6 +271,7 @@ public String saveAktoDataType(){ public static void handleDataTypeRedaction(){ try{ + fetchCustomDataTypes(Context.accountId.get()); loggerMaker.infoAndAddToDb("Dropping redacted data types for custom data types", LogDb.DASHBOARD); List customDataTypesToBeFixed = CustomDataTypeDao.instance.findAll(Filters.eq(AktoDataType.SAMPLE_DATA_FIXED, false)); loggerMaker.infoAndAddToDb("Found " + customDataTypesToBeFixed.size() + " custom data types to be fixed", LogDb.DASHBOARD); @@ -302,25 +306,69 @@ private static void handleRedactionForSubType(SingleTypeInfo.SubType subType) { loggerMaker.infoAndAddToDb("Dropping redacted data types for subType:" + subType.getName(), LogDb.DASHBOARD); int skip = 0; int limit = 100; - int itr = 1; + Set keys = new HashSet<>(); while (true) { List apiInfoKeys = SingleTypeInfoDao.instance.fetchEndpointsInCollection2(subType, skip, limit); - loggerMaker.infoAndAddToDb("Found " + apiInfoKeys.size() + " apiInfoKeys for subType:" + subType.getName() + ", itr: " +itr, LogDb.DASHBOARD); + loggerMaker.infoAndAddToDb("Found " + apiInfoKeys.size() + " apiInfoKeys for subType:" + subType.getName(), LogDb.DASHBOARD); if(apiInfoKeys.isEmpty()){ break; } - List query = apiInfoKeys.stream().map(key -> Filters.and( - Filters.eq("_id.apiCollectionId", key.getApiCollectionId()), - Filters.eq("_id.url", key.getUrl()), - Filters.eq("_id.method", key.getMethod()) - )).collect(Collectors.toList()); - - SampleDataDao.instance.updateManyNoUpsert(Filters.or(query), Updates.unset("samples")); - SensitiveSampleDataDao.instance.updateManyNoUpsert(Filters.or(query), Updates.unset("sampleData")); - loggerMaker.infoAndAddToDb("Dropped redacted data types for subType:" + subType.getName() + " for " + apiInfoKeys.size() + " apiInfoKeys, itr:" + itr, LogDb.DASHBOARD); - itr++; + keys.addAll(apiInfoKeys); skip += limit; } + loggerMaker.infoAndAddToDb("Found " + keys.size() + " apiInfoKeys for subType:" + subType.getName(), LogDb.DASHBOARD); + List query = keys.stream().map(key -> Filters.and( + Filters.eq("_id.apiCollectionId", key.getApiCollectionId()), + Filters.eq("_id.url", key.getUrl()), + Filters.eq("_id.method", key.getMethod()) + )).collect(Collectors.toList()); + + List collectionIds = keys.stream().map(ApiInfo.ApiInfoKey::getApiCollectionId).collect(Collectors.toList()); + List apiCollections = ApiCollectionsDao.instance.findAll(Filters.in("_id", collectionIds)); + Map apiCollectionsMap = apiCollections.stream().collect(Collectors.toMap(ApiCollection::getId, ApiCollection::getRedact)); + + for(Bson q: query){ + loggerMaker.infoAndAddToDb("Handling query:" + q.toString(), LogDb.DASHBOARD); + SampleData sampleData = SampleDataDao.instance.findOne(q); + if(sampleData.getSamples() != null && !sampleData.getSamples().isEmpty()){ + List modifiedSampleData = new ArrayList<>(); + boolean errorWhileProcessing = false; + for (String sample : sampleData.getSamples()) { + try { + sample = RedactSampleData.redactIfRequired(sample, false, apiCollectionsMap.get(sampleData.getId().getApiCollectionId())); + } catch (Exception e) { + errorWhileProcessing = true; + loggerMaker.errorAndAddToDb("Failed to redact sample data for subType:" + subType.getName() + ", sample:" + sample, LogDb.DASHBOARD); + } + modifiedSampleData.add(sample); + } + SampleDataDao.instance.updateOneNoUpsert(q, Updates.set("samples", errorWhileProcessing ? Collections.emptyList(): modifiedSampleData)); + loggerMaker.infoAndAddToDb("Updated sample data for subType:" + subType.getName() + ", query:" + q.toString(), LogDb.DASHBOARD); + } + + List all = SensitiveSampleDataDao.instance.findAll(q); + for (SensitiveSampleData sensitiveSampleData : all) { + if(sensitiveSampleData.getSampleData() != null && !sensitiveSampleData.getSampleData().isEmpty()){ + List modifiedSampleData = new ArrayList<>(); + boolean errorWhileProcessing = false; + for (String sample : sensitiveSampleData.getSampleData()) { + try { + sample = RedactSampleData.redactIfRequired(sample, false, apiCollectionsMap.get(sampleData.getId().getApiCollectionId())); + } catch (Exception e) { + loggerMaker.errorAndAddToDb("Failed to redact sample data for subType:" + subType.getName() + ", sample:" + sample, LogDb.DASHBOARD); + errorWhileProcessing = true; + } + modifiedSampleData.add(sample); + } + SensitiveSampleDataDao.instance.updateOneNoUpsert(q, Updates.set("sampleData", errorWhileProcessing ? Collections.emptyList(): modifiedSampleData)); + loggerMaker.infoAndAddToDb("Updated sensitive sample data for subType:" + subType.getName() + ", query:" + q.toString(), LogDb.DASHBOARD); + } + } + + } + + SingleTypeInfoDao.instance.updateMany(Filters.and(Filters.eq("subType", subType.getName()), Filters.exists("values.elements", true)), Updates.set("values.elements", Collections.emptyList())); + loggerMaker.infoAndAddToDb("Redacted values in sti for subType:" + subType.getName(), LogDb.DASHBOARD); } public List generatePositions(List sensitivePosition){ diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/data_types/transform.js b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/data_types/transform.js index 4533fee52e..582a76b171 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/data_types/transform.js +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/data_types/transform.js @@ -15,7 +15,7 @@ const func = { let sensitiveAlways = false; let sensitivePosition = [] ; - switch (obj.state){ + switch (obj.sensitiveState){ case '1': sensitivePosition = ["REQUEST_PAYLOAD", "REQUEST_HEADER"]; break; From b33a040daf87110301d7efb6d48d0b1fff90c6e6 Mon Sep 17 00:00:00 2001 From: aktoboy Date: Mon, 8 Jan 2024 20:26:54 +0530 Subject: [PATCH 11/39] Added changes to unset existing data --- .../com/akto/action/ApiCollectionsAction.java | 46 +++------------- .../com/akto/action/CustomDataTypeAction.java | 52 +++---------------- .../observe/api_collections/ApiEndpoints.jsx | 2 +- 3 files changed, 15 insertions(+), 85 deletions(-) diff --git a/apps/dashboard/src/main/java/com/akto/action/ApiCollectionsAction.java b/apps/dashboard/src/main/java/com/akto/action/ApiCollectionsAction.java index 363bbdfeb4..a2d78d6ab1 100644 --- a/apps/dashboard/src/main/java/com/akto/action/ApiCollectionsAction.java +++ b/apps/dashboard/src/main/java/com/akto/action/ApiCollectionsAction.java @@ -19,6 +19,7 @@ import com.mongodb.client.model.Filters; import com.mongodb.BasicDBObject; import com.mongodb.client.model.Updates; +import com.mongodb.client.result.UpdateResult; import com.opensymphony.xwork2.Action; import org.bson.conversions.Bson; @@ -132,45 +133,12 @@ public static void dropSampleDataForApiCollection() { loggerMaker.infoAndAddToDb(String.format("Dropping sample data for %d api collections", apiCollections.size()), LoggerMaker.LogDb.DASHBOARD); for (ApiCollection apiCollection: apiCollections) { int apiCollectionId = apiCollection.getId(); - List sampleDataList = SampleDataDao.instance.findAll(Filters.and(Filters.eq("_id.apiCollectionId", apiCollectionId), Filters.exists("samples", true))); - for(SampleData sampleData: sampleDataList){ - if(sampleData.getSamples() != null && !sampleData.getSamples().isEmpty()){ - boolean errorWhileRedacting = false; - List modifiedSamples = new ArrayList<>(); - for (String sample : sampleData.getSamples()) { - try{ - sample = RedactSampleData.redactIfRequired(sample, false, true); - } catch (Exception e){ - loggerMaker.errorAndAddToDb("Error while redacting sample data", LoggerMaker.LogDb.DASHBOARD); - errorWhileRedacting = true; - break; - } - modifiedSamples.add(sample); - } - SampleDataDao.instance.updateOneNoUpsert(Filters.eq("_id", sampleData.getId()), Updates.set("samples", errorWhileRedacting ? Collections.emptyList(): modifiedSamples)); - } - } - - List sensitiveSampleDataList = SensitiveSampleDataDao.instance.findAll(Filters.and(Filters.eq("_id.apiCollectionId", apiCollectionId), Filters.exists("sampleData", true))); - - for(SensitiveSampleData sensitiveSampleData: sensitiveSampleDataList){ - if(sensitiveSampleData.getSampleData() != null && !sensitiveSampleData.getSampleData().isEmpty()){ - List modifiedSamples = new ArrayList<>(); - boolean errorWhileRedacting = false; - for (String sample : sensitiveSampleData.getSampleData()) { - - try{ - sample = RedactSampleData.redactIfRequired(sample, false, true); - } catch (Exception e){ - loggerMaker.errorAndAddToDb("Error while redacting sample data", LoggerMaker.LogDb.DASHBOARD); - errorWhileRedacting = true; - break; - } - modifiedSamples.add(sample); - } - SensitiveSampleDataDao.instance.updateOneNoUpsert(Filters.eq("_id", sensitiveSampleData.getId()), Updates.set("sampleData", errorWhileRedacting ? Collections.emptyList(): modifiedSamples)); - } - } + UpdateResult updateResult = SampleDataDao.instance.updateManyNoUpsert(Filters.eq("_id.apiCollectionId", apiCollectionId), Updates.set("samples", Collections.emptyList())); + loggerMaker.infoAndAddToDb(String.format("Dropped %d sample data for api collection %d", updateResult.getModifiedCount(), apiCollectionId), LoggerMaker.LogDb.DASHBOARD); + updateResult = SensitiveSampleDataDao.instance.updateManyNoUpsert(Filters.eq("_id.apiCollectionId", apiCollectionId), Updates.set("sampleData", Collections.emptyList())); + loggerMaker.infoAndAddToDb(String.format("Dropped %d sensitive sample data for api collection %d", updateResult.getModifiedCount(), apiCollectionId), LoggerMaker.LogDb.DASHBOARD); + updateResult = SensitiveSampleDataDao.instance.updateManyNoUpsert(Filters.eq("_id.apiCollectionId", apiCollectionId), Updates.set("values.elements", Collections.emptyList())); + loggerMaker.infoAndAddToDb(String.format("Dropped %d sensitive sample data for api collection %d", updateResult.getModifiedCount(), apiCollectionId), LoggerMaker.LogDb.DASHBOARD); ApiCollectionsDao.instance.updateOneNoUpsert(Filters.eq("_id", apiCollectionId), Updates.set(ApiCollection.SAMPLE_COLLECTIONS_DROPPED, true)); } loggerMaker.infoAndAddToDb(String.format("Dropped sample data for %d api collections", apiCollections.size()), LoggerMaker.LogDb.DASHBOARD); diff --git a/apps/dashboard/src/main/java/com/akto/action/CustomDataTypeAction.java b/apps/dashboard/src/main/java/com/akto/action/CustomDataTypeAction.java index 17e96d3b3a..76df58dac0 100644 --- a/apps/dashboard/src/main/java/com/akto/action/CustomDataTypeAction.java +++ b/apps/dashboard/src/main/java/com/akto/action/CustomDataTypeAction.java @@ -24,6 +24,7 @@ import com.mongodb.bulk.BulkWriteResult; import com.mongodb.client.MongoCursor; import com.mongodb.client.model.*; +import com.mongodb.client.result.UpdateResult; import com.opensymphony.xwork2.Action; import io.swagger.models.auth.In; import org.apache.commons.lang3.EnumUtils; @@ -323,52 +324,13 @@ private static void handleRedactionForSubType(SingleTypeInfo.SubType subType) { Filters.eq("_id.method", key.getMethod()) )).collect(Collectors.toList()); - List collectionIds = keys.stream().map(ApiInfo.ApiInfoKey::getApiCollectionId).collect(Collectors.toList()); - List apiCollections = ApiCollectionsDao.instance.findAll(Filters.in("_id", collectionIds)); - Map apiCollectionsMap = apiCollections.stream().collect(Collectors.toMap(ApiCollection::getId, ApiCollection::getRedact)); - - for(Bson q: query){ - loggerMaker.infoAndAddToDb("Handling query:" + q.toString(), LogDb.DASHBOARD); - SampleData sampleData = SampleDataDao.instance.findOne(q); - if(sampleData.getSamples() != null && !sampleData.getSamples().isEmpty()){ - List modifiedSampleData = new ArrayList<>(); - boolean errorWhileProcessing = false; - for (String sample : sampleData.getSamples()) { - try { - sample = RedactSampleData.redactIfRequired(sample, false, apiCollectionsMap.get(sampleData.getId().getApiCollectionId())); - } catch (Exception e) { - errorWhileProcessing = true; - loggerMaker.errorAndAddToDb("Failed to redact sample data for subType:" + subType.getName() + ", sample:" + sample, LogDb.DASHBOARD); - } - modifiedSampleData.add(sample); - } - SampleDataDao.instance.updateOneNoUpsert(q, Updates.set("samples", errorWhileProcessing ? Collections.emptyList(): modifiedSampleData)); - loggerMaker.infoAndAddToDb("Updated sample data for subType:" + subType.getName() + ", query:" + q.toString(), LogDb.DASHBOARD); - } - - List all = SensitiveSampleDataDao.instance.findAll(q); - for (SensitiveSampleData sensitiveSampleData : all) { - if(sensitiveSampleData.getSampleData() != null && !sensitiveSampleData.getSampleData().isEmpty()){ - List modifiedSampleData = new ArrayList<>(); - boolean errorWhileProcessing = false; - for (String sample : sensitiveSampleData.getSampleData()) { - try { - sample = RedactSampleData.redactIfRequired(sample, false, apiCollectionsMap.get(sampleData.getId().getApiCollectionId())); - } catch (Exception e) { - loggerMaker.errorAndAddToDb("Failed to redact sample data for subType:" + subType.getName() + ", sample:" + sample, LogDb.DASHBOARD); - errorWhileProcessing = true; - } - modifiedSampleData.add(sample); - } - SensitiveSampleDataDao.instance.updateOneNoUpsert(q, Updates.set("sampleData", errorWhileProcessing ? Collections.emptyList(): modifiedSampleData)); - loggerMaker.infoAndAddToDb("Updated sensitive sample data for subType:" + subType.getName() + ", query:" + q.toString(), LogDb.DASHBOARD); - } - } - - } + UpdateResult updateResult = SampleDataDao.instance.updateManyNoUpsert(Filters.or(query), Updates.set("samples", Collections.emptyList())); + loggerMaker.infoAndAddToDb("Redacted samples in sd for subType:" + subType.getName() + ", modified:" + updateResult.getModifiedCount(), LogDb.DASHBOARD); - SingleTypeInfoDao.instance.updateMany(Filters.and(Filters.eq("subType", subType.getName()), Filters.exists("values.elements", true)), Updates.set("values.elements", Collections.emptyList())); - loggerMaker.infoAndAddToDb("Redacted values in sti for subType:" + subType.getName(), LogDb.DASHBOARD); + updateResult = SensitiveSampleDataDao.instance.updateManyNoUpsert(Filters.or(query), Updates.set("sampleData", Collections.emptyList())); + loggerMaker.infoAndAddToDb("Redacted samples in ssd for subType:" + subType.getName() + ", modified:" + updateResult.getModifiedCount(), LogDb.DASHBOARD); + updateResult= SingleTypeInfoDao.instance.updateManyNoUpsert(Filters.and(Filters.eq("subType", subType.getName()), Filters.exists("values.elements", true)), Updates.set("values.elements", Collections.emptyList())); + loggerMaker.infoAndAddToDb("Redacted values in sti for subType:" + subType.getName() + ", modified:" + updateResult.getModifiedCount(), LogDb.DASHBOARD); } public List generatePositions(List sensitivePosition){ diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/ApiEndpoints.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/ApiEndpoints.jsx index 94f10faa73..dcd1cff015 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/ApiEndpoints.jsx +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/ApiEndpoints.jsx @@ -136,7 +136,7 @@ function ApiEndpoints() { setLoading(true) let apiCollectionData = await api.fetchAPICollection(apiCollectionId) console.log("data", apiCollectionData) - setIsRedacted(apiCollectionData.data.redacted) + setIsRedacted(apiCollectionData.redacted) let apiEndpointsInCollection = apiCollectionData.data.endpoints.map(x => { return { ...x._id, startTs: x.startTs, changesCount: x.changesCount, shadow: x.shadow ? x.shadow : false } }) let apiInfoListInCollection = apiCollectionData.data.apiInfoList let unusedEndpointsInCollection = apiCollectionData.unusedEndpoints From a025ad59ff5e33b9f2bc349d476958c305c73b8b Mon Sep 17 00:00:00 2001 From: aktoboy Date: Tue, 9 Jan 2024 18:54:25 +0530 Subject: [PATCH 12/39] Fixed merge issues --- .../src/main/java/com/akto/action/ApiCollectionsAction.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/apps/dashboard/src/main/java/com/akto/action/ApiCollectionsAction.java b/apps/dashboard/src/main/java/com/akto/action/ApiCollectionsAction.java index acdecea30a..e02b7cb137 100644 --- a/apps/dashboard/src/main/java/com/akto/action/ApiCollectionsAction.java +++ b/apps/dashboard/src/main/java/com/akto/action/ApiCollectionsAction.java @@ -52,7 +52,6 @@ public class ApiCollectionsAction extends UserAction { LastCronRunInfo timerInfo; Map> severityInfo = new HashMap<>(); - private static final LoggerMaker loggerMaker = new LoggerMaker(ApiCollectionsAction.class); int apiCollectionId; boolean redacted; @@ -282,7 +281,6 @@ public void setApiCollectionId(int apiCollectionId) { public int getSensitiveUrlsInResponse() { return sensitiveUrlsInResponse; } - } public Map> getSensitiveSubtypesInCollection() { return sensitiveSubtypesInCollection; From 6328f3a051bfaf0ce6056058de2dd8cd49957e98 Mon Sep 17 00:00:00 2001 From: aktoboy Date: Tue, 9 Jan 2024 18:58:04 +0530 Subject: [PATCH 13/39] Changes for adding warning to redact button --- .../observe/api_collections/ApiEndpoints.jsx | 140 ++++++++++++------ .../pages/observe/data_types/DataTypes.jsx | 7 +- .../shared/InformationBannerComponent.jsx | 2 +- 3 files changed, 98 insertions(+), 51 deletions(-) diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/ApiEndpoints.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/ApiEndpoints.jsx index 9169c1de13..6022c53a23 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/ApiEndpoints.jsx +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/ApiEndpoints.jsx @@ -1,5 +1,5 @@ import PageWithMultipleCards from "../../../components/layouts/PageWithMultipleCards" -import { Text, HorizontalStack, Button, Popover, Modal, IndexFiltersMode, VerticalStack, Box } from "@shopify/polaris" +import { Text, HorizontalStack, Button, Popover, Modal, IndexFiltersMode, VerticalStack, Box, Checkbox } from "@shopify/polaris" import api from "../api" import { useEffect, useState } from "react" import func from "@/util/func" @@ -120,6 +120,7 @@ function ApiEndpoints() { const [isGptScreenActive, setIsGptScreenActive] = useState(false) const [isGptActive, setIsGptActive] = useState(false) const [redacted, setIsRedacted] = useState(false) + const [showRedactModal, setShowRedactModal] = useState(false) const tableTabs = [ { @@ -248,7 +249,17 @@ function ApiEndpoints() { func.setToast(true, false, "Endpoints refreshed") } + function redactCheckBoxClicked(){ + if(!redacted){ + setShowRedactModal(true) + } else { + setIsRedacted(false) + redactCollection(); + } + } + function redactCollection(){ + setShowRedactModal(false) var updatedRedacted = !redacted; api.redactCollection(apiCollectionId, updatedRedacted).then(resp => { setIsRedacted(updatedRedacted) @@ -399,7 +410,7 @@ function ApiEndpoints() { - Export as: + Export as
OpenAPI spec @@ -413,6 +424,18 @@ function ApiEndpoints() { + + + Others + + redactCheckBoxClicked()} + /> + + + @@ -434,56 +457,75 @@ function ApiEndpoints() { },200) } + let modal = ( + setShowRedactModal(false)} + title="Note!" + primaryAction={{ + content: 'Enable', + onAction: redactCollection + }} + > + + When enabled, existing sample payload values for this collection will be deleted, and data in all the future payloads for this collection will be redacted. Please note that your API Inventory, Sensitive data etc. will be intact. We will simply be deleting the sample payload values. + + + ) + return ( - - - - } - backUrl="/dashboard/observe/inventory" - secondaryActions={secondaryActionsComponent} - components={ - loading ? [] : - [ -
- + + + + } + backUrl="/dashboard/observe/inventory" + secondaryActions={secondaryActionsComponent} + components={ + loading ? [] : + [ +
+ { return "warning" }} + selected={selected} + onRowClick={handleRowClick} + onSelect={handleSelectedTab} + getFilteredItems={getFilteredItems} + mode={IndexFiltersMode.Default} + headings={headings} + useNewRow={true} + condensedHeight={true} + tableTabs={tableTabs} + /> + setIsGptScreenActive(false)} title="Akto GPT"> + + setIsGptScreenActive(false)}/> + + +
, + { return "warning" }} - selected={selected} - onRowClick={handleRowClick} - onSelect={handleSelectedTab} - getFilteredItems={getFilteredItems} - mode={IndexFiltersMode.Default} - headings={headings} - useNewRow={true} - condensedHeight={true} - tableTabs={tableTabs} + isGptActive={isGptActive} /> - setIsGptScreenActive(false)} title="Akto GPT"> - - setIsGptScreenActive(false)}/> - - -
, - { return "warning" }} - isGptActive={isGptActive} - /> - ]} - /> + ]} + /> + {modal} + ) } diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/data_types/DataTypes.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/data_types/DataTypes.jsx index 0907ec6600..a7b9647ea2 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/data_types/DataTypes.jsx +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/data_types/DataTypes.jsx @@ -1,5 +1,5 @@ import React, { useEffect, useState, useReducer } from 'react' -import { Button, LegacyCard, HorizontalGrid, TextField, VerticalStack, Text } from '@shopify/polaris' +import { Button, LegacyCard, HorizontalGrid, TextField, VerticalStack, Text, Banner } from '@shopify/polaris' import Dropdown from '../../../components/layouts/Dropdown' import "./DataTypes.css" import ConditionsPicker from '../../../components/ConditionsPicker' @@ -9,6 +9,7 @@ import func from "@/util/func" import api from '../api' import {produce} from "immer" import DetailsPage from '../../../components/DetailsPage' +import InformationBannerComponent from '../../quick_start/components/shared/InformationBannerComponent' const statusItems = [ { @@ -270,6 +271,10 @@ function DataTypes() { const redactCard = ( +
+ + +

Redact this data type


diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/quick_start/components/shared/InformationBannerComponent.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/quick_start/components/shared/InformationBannerComponent.jsx index 81054ca314..1ed1544097 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/quick_start/components/shared/InformationBannerComponent.jsx +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/quick_start/components/shared/InformationBannerComponent.jsx @@ -10,7 +10,7 @@ function InformationBannerComponent({docsUrl,content}) {
{content} - + {docsUrl !== '' ? : ""}
) From 17ebe2f2ba793bb8f4faf281e749f7358b2acdf4 Mon Sep 17 00:00:00 2001 From: aktoboy Date: Wed, 10 Jan 2024 12:08:02 +0530 Subject: [PATCH 14/39] Removed duplication --- .../com/akto/action/CustomDataTypeAction.java | 76 +++++++++---------- .../java/com/akto/dao/SingleTypeInfoDao.java | 66 ++++------------ 2 files changed, 53 insertions(+), 89 deletions(-) diff --git a/apps/dashboard/src/main/java/com/akto/action/CustomDataTypeAction.java b/apps/dashboard/src/main/java/com/akto/action/CustomDataTypeAction.java index 76df58dac0..a313d6d054 100644 --- a/apps/dashboard/src/main/java/com/akto/action/CustomDataTypeAction.java +++ b/apps/dashboard/src/main/java/com/akto/action/CustomDataTypeAction.java @@ -14,7 +14,6 @@ import com.akto.parsers.HttpCallParser; import com.akto.util.JSONUtils; import com.akto.utils.AktoCustomException; -import com.akto.utils.RedactSampleData; import com.fasterxml.jackson.core.JsonFactory; import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.databind.JsonNode; @@ -26,12 +25,13 @@ import com.mongodb.client.model.*; import com.mongodb.client.result.UpdateResult; import com.opensymphony.xwork2.Action; -import io.swagger.models.auth.In; import org.apache.commons.lang3.EnumUtils; import org.bson.conversions.Bson; import java.io.IOException; import java.util.*; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import java.util.stream.Collectors; import static com.akto.dto.type.SingleTypeInfo.fetchCustomDataTypes; @@ -40,6 +40,8 @@ public class CustomDataTypeAction extends UserAction{ private static LoggerMaker loggerMaker = new LoggerMaker(CustomDataTypeAction.class); + + private static final ExecutorService service = Executors.newFixedThreadPool(1); private boolean createNew; private String name; private boolean sensitiveAlways; @@ -206,15 +208,13 @@ public String execute() { if(redacted){ int accountId = Context.accountId.get(); - Runnable r = () ->{ + service.submit(() ->{ Context.accountId.set(accountId); loggerMaker.infoAndAddToDb("Triggered a job to fix existing custom data types", LogDb.DASHBOARD); handleDataTypeRedaction(); - }; - new Thread(r).start(); + }); } - return Action.SUCCESS.toUpperCase(); } @@ -259,12 +259,11 @@ public String saveAktoDataType(){ SingleTypeInfo.fetchCustomDataTypes(Context.accountId.get()); if(redacted){ int accountId = Context.accountId.get(); - Runnable r = () ->{ + service.submit(() ->{ Context.accountId.set(accountId); - loggerMaker.infoAndAddToDb("Triggered a job to fix existing data for " + aktoDataType.getName(), LogDb.DASHBOARD); + loggerMaker.infoAndAddToDb("Triggered a job to fix existing akto data types", LogDb.DASHBOARD); handleDataTypeRedaction(); - }; - new Thread(r).start(); + }); } return Action.SUCCESS.toUpperCase(); @@ -277,24 +276,22 @@ public static void handleDataTypeRedaction(){ List customDataTypesToBeFixed = CustomDataTypeDao.instance.findAll(Filters.eq(AktoDataType.SAMPLE_DATA_FIXED, false)); loggerMaker.infoAndAddToDb("Found " + customDataTypesToBeFixed.size() + " custom data types to be fixed", LogDb.DASHBOARD); + List aktoDataTypesToBeFixed = AktoDataTypeDao.instance.findAll(Filters.eq(AktoDataType.SAMPLE_DATA_FIXED, false)); + loggerMaker.infoAndAddToDb("Found " + aktoDataTypesToBeFixed.size() + " akto data types to be fixed", LogDb.DASHBOARD); + + Set subTypesToBeFixed = new HashSet<>(); customDataTypesToBeFixed.forEach(cdt -> { SingleTypeInfo.SubType st = cdt.toSubType(); - loggerMaker.infoAndAddToDb("Dropping redacted data types for custom data type:" + cdt.getName() + ", subType:" + st.getName(), LogDb.DASHBOARD); - handleRedactionForSubType(st); - CustomDataTypeDao.instance.updateOneNoUpsert(Filters.eq(CustomDataType.NAME, cdt.getName()), Updates.set(CustomDataType.SAMPLE_DATA_FIXED, true)); - loggerMaker.infoAndAddToDb("Dropped redacted data types for custom data type:" + cdt.getName() + ", subType:" + st.getName(), LogDb.DASHBOARD); + subTypesToBeFixed.add(st); }); - - loggerMaker.infoAndAddToDb("Dropping redacted data types for akto data types", LogDb.DASHBOARD); - - List aktoDataTypesToBeFixed = AktoDataTypeDao.instance.findAll(Filters.eq(AktoDataType.SAMPLE_DATA_FIXED, false)); - loggerMaker.infoAndAddToDb("Found " + aktoDataTypesToBeFixed.size() + " akto data types to be fixed", LogDb.DASHBOARD); aktoDataTypesToBeFixed.forEach(adt -> { - loggerMaker.infoAndAddToDb("Dropping redacted data types for akto data type:" + adt.getName(), LogDb.DASHBOARD); SingleTypeInfo.SubType st = subTypeMap.get(adt.getName()); + subTypesToBeFixed.add(st); + }); + subTypesToBeFixed.forEach(st -> { + loggerMaker.infoAndAddToDb("Dropping redacted data types for subType:" + st.getName(), LogDb.DASHBOARD); handleRedactionForSubType(st); - AktoDataTypeDao.instance.updateOneNoUpsert(Filters.eq(AktoDataType.NAME, adt.getName()), Updates.set(AktoDataType.SAMPLE_DATA_FIXED, true)); - loggerMaker.infoAndAddToDb("Dropped redacted data types for akto data type:" + adt.getName(), LogDb.DASHBOARD); + loggerMaker.infoAndAddToDb("Dropped redacted data types for subType:" + st.getName(), LogDb.DASHBOARD); }); loggerMaker.infoAndAddToDb("Dropped redacted data types successfully!", LogDb.DASHBOARD); } catch (Exception e){ @@ -307,30 +304,31 @@ private static void handleRedactionForSubType(SingleTypeInfo.SubType subType) { loggerMaker.infoAndAddToDb("Dropping redacted data types for subType:" + subType.getName(), LogDb.DASHBOARD); int skip = 0; int limit = 100; - Set keys = new HashSet<>(); while (true) { - List apiInfoKeys = SingleTypeInfoDao.instance.fetchEndpointsInCollection2(subType, skip, limit); - loggerMaker.infoAndAddToDb("Found " + apiInfoKeys.size() + " apiInfoKeys for subType:" + subType.getName(), LogDb.DASHBOARD); + List apiInfoKeys = SingleTypeInfoDao.instance.fetchEndpointsBySubType(subType, skip, limit); if(apiInfoKeys.isEmpty()){ + loggerMaker.infoAndAddToDb("No apiInfoKeys left for subType:" + subType.getName(), LogDb.DASHBOARD); break; } - keys.addAll(apiInfoKeys); - skip += limit; - } - loggerMaker.infoAndAddToDb("Found " + keys.size() + " apiInfoKeys for subType:" + subType.getName(), LogDb.DASHBOARD); - List query = keys.stream().map(key -> Filters.and( - Filters.eq("_id.apiCollectionId", key.getApiCollectionId()), - Filters.eq("_id.url", key.getUrl()), - Filters.eq("_id.method", key.getMethod()) - )).collect(Collectors.toList()); - UpdateResult updateResult = SampleDataDao.instance.updateManyNoUpsert(Filters.or(query), Updates.set("samples", Collections.emptyList())); - loggerMaker.infoAndAddToDb("Redacted samples in sd for subType:" + subType.getName() + ", modified:" + updateResult.getModifiedCount(), LogDb.DASHBOARD); + loggerMaker.infoAndAddToDb("Found " + apiInfoKeys.size() + " apiInfoKeys for subType:" + subType.getName(), LogDb.DASHBOARD); + List query = apiInfoKeys.stream().map(key -> Filters.and( + Filters.eq("_id.apiCollectionId", key.getApiCollectionId()), + Filters.eq("_id.url", key.getUrl()), + Filters.eq("_id.method", key.getMethod()) + )).collect(Collectors.toList()); - updateResult = SensitiveSampleDataDao.instance.updateManyNoUpsert(Filters.or(query), Updates.set("sampleData", Collections.emptyList())); - loggerMaker.infoAndAddToDb("Redacted samples in ssd for subType:" + subType.getName() + ", modified:" + updateResult.getModifiedCount(), LogDb.DASHBOARD); - updateResult= SingleTypeInfoDao.instance.updateManyNoUpsert(Filters.and(Filters.eq("subType", subType.getName()), Filters.exists("values.elements", true)), Updates.set("values.elements", Collections.emptyList())); + UpdateResult updateResult = SampleDataDao.instance.updateManyNoUpsert(Filters.or(query), Updates.set("samples", Collections.emptyList())); + loggerMaker.infoAndAddToDb("Redacted samples in sd for subType:" + subType.getName() + ", modified:" + updateResult.getModifiedCount(), LogDb.DASHBOARD); + + updateResult = SensitiveSampleDataDao.instance.updateManyNoUpsert(Filters.or(query), Updates.set("sampleData", Collections.emptyList())); + loggerMaker.infoAndAddToDb("Redacted samples in ssd for subType:" + subType.getName() + ", modified:" + updateResult.getModifiedCount(), LogDb.DASHBOARD); + + skip += limit; + } + UpdateResult updateResult = SingleTypeInfoDao.instance.updateManyNoUpsert(Filters.and(Filters.eq("subType", subType.getName()), Filters.exists("values.elements", true)), Updates.set("values.elements", Collections.emptyList())); loggerMaker.infoAndAddToDb("Redacted values in sti for subType:" + subType.getName() + ", modified:" + updateResult.getModifiedCount(), LogDb.DASHBOARD); + } public List generatePositions(List sensitivePosition){ diff --git a/libs/dao/src/main/java/com/akto/dao/SingleTypeInfoDao.java b/libs/dao/src/main/java/com/akto/dao/SingleTypeInfoDao.java index d539e53094..dceb672417 100644 --- a/libs/dao/src/main/java/com/akto/dao/SingleTypeInfoDao.java +++ b/libs/dao/src/main/java/com/akto/dao/SingleTypeInfoDao.java @@ -2,23 +2,16 @@ import java.util.*; -import javax.print.attribute.HashAttributeSet; - -import com.akto.DaoInit; import com.akto.dao.context.Context; import com.akto.dto.ApiInfo; import com.akto.dto.CustomDataType; -import com.akto.dto.HttpResponseParams; import com.akto.dto.SensitiveParamInfo; -import com.akto.dto.traffic.SampleData; import com.akto.dto.type.SingleTypeInfo; import com.akto.dto.type.URLMethods; import com.mongodb.BasicDBObject; -import com.mongodb.ConnectionString; import com.mongodb.client.MongoCursor; import com.mongodb.client.model.*; -import org.bson.Document; import org.bson.conversions.Bson; public class SingleTypeInfoDao extends AccountsContextDao { @@ -271,56 +264,27 @@ public void resetCount() { // to get results irrespective of collections use negative value for apiCollectionId public List fetchEndpointsInCollection(int apiCollectionId) { - List pipeline = new ArrayList<>(); - ; - BasicDBObject groupedId = - new BasicDBObject("apiCollectionId", "$apiCollectionId") - .append("url", "$url") - .append("method", "$method"); - + Bson matchCriteria = null; if (apiCollectionId != -1) { - pipeline.add(Aggregates.match(Filters.eq("apiCollectionId", apiCollectionId))); - } - - Bson projections = Projections.fields( - Projections.include("timestamp", "apiCollectionId", "url", "method") - ); - - pipeline.add(Aggregates.project(projections)); - pipeline.add(Aggregates.group(groupedId)); - pipeline.add(Aggregates.sort(Sorts.descending("startTs"))); - - MongoCursor endpointsCursor = instance.getMCollection().aggregate(pipeline, BasicDBObject.class).cursor(); - - List endpoints = new ArrayList<>(); - while(endpointsCursor.hasNext()) { - BasicDBObject v = endpointsCursor.next(); - try { - BasicDBObject vv = (BasicDBObject) v.get("_id"); - ApiInfo.ApiInfoKey apiInfoKey = new ApiInfo.ApiInfoKey( - (int) vv.get("apiCollectionId"), - (String) vv.get("url"), - URLMethods.Method.fromString((String) vv.get("method")) - ); - endpoints.add(apiInfoKey); - } catch (Exception e) { - ; - - } + matchCriteria = Filters.eq("apiCollectionId", apiCollectionId); } + return fetchEndpoints(matchCriteria, "startTs"); + } - return endpoints; + public List fetchEndpointsBySubType(SingleTypeInfo.SubType subType, int skip, int limit) { + return fetchEndpoints(Filters.eq("subType", subType.getName()), "timestamp", skip, limit); } - public List fetchEndpointsInCollection2(SingleTypeInfo.SubType subType, int skip, int limit) { + private List fetchEndpoints(Bson matchCriteria, String sortField, int... skipLimit) { List pipeline = new ArrayList<>(); BasicDBObject groupedId = new BasicDBObject("apiCollectionId", "$apiCollectionId") .append("url", "$url") .append("method", "$method"); - - pipeline.add(Aggregates.match(Filters.eq("subType", subType.getName()))); + if(matchCriteria != null) { + pipeline.add(Aggregates.match(matchCriteria)); + } Bson projections = Projections.fields( Projections.include("timestamp", "apiCollectionId", "url", "method") @@ -328,9 +292,12 @@ public List fetchEndpointsInCollection2(SingleTypeInfo.SubTy pipeline.add(Aggregates.project(projections)); pipeline.add(Aggregates.group(groupedId)); - pipeline.add(Aggregates.sort(Sorts.descending("timestamp"))); - pipeline.add(Aggregates.limit(limit)); - pipeline.add(Aggregates.skip(skip)); + pipeline.add(Aggregates.sort(Sorts.descending(sortField))); + + if (skipLimit.length == 2) { + pipeline.add(Aggregates.limit(skipLimit[1])); + pipeline.add(Aggregates.skip(skipLimit[0])); + } MongoCursor endpointsCursor = instance.getMCollection().aggregate(pipeline, BasicDBObject.class).cursor(); @@ -347,7 +314,6 @@ public List fetchEndpointsInCollection2(SingleTypeInfo.SubTy endpoints.add(apiInfoKey); } catch (Exception e) { ; - } } From e8013bf13396eb0af08e1ea24eb1f779c4062707 Mon Sep 17 00:00:00 2001 From: aktoboy Date: Wed, 10 Jan 2024 12:16:37 +0530 Subject: [PATCH 15/39] Removed original constructor for AktoDataType --- .../test/java/com/akto/parsers/TestDBSync.java | 16 ++++++++-------- .../test/java/com/akto/parsers/TestDump2.java | 16 ++++++++-------- .../java/com/akto/parsers/TestMergingNew.java | 16 ++++++++-------- .../com/akto/runtime/TestApiCatalogSync.java | 14 +++++++------- .../java/com/akto/utils/SampleDataToSTITest.java | 16 ++++++++-------- .../com/akto/listener/InitializerListener.java | 16 ++++++++-------- .../action/TestIgnoreFalsePositivesAction.java | 2 +- .../src/main/java/com/akto/dto/AktoDataType.java | 9 --------- .../java/com/akto/dto/type/TestKeyTypes.java | 14 +++++++------- .../test/java/com/akto/dto/type/TestSubType.java | 8 ++++---- 10 files changed, 59 insertions(+), 68 deletions(-) diff --git a/apps/api-runtime/src/test/java/com/akto/parsers/TestDBSync.java b/apps/api-runtime/src/test/java/com/akto/parsers/TestDBSync.java index 3344ac482a..0253f82b88 100644 --- a/apps/api-runtime/src/test/java/com/akto/parsers/TestDBSync.java +++ b/apps/api-runtime/src/test/java/com/akto/parsers/TestDBSync.java @@ -48,14 +48,14 @@ public void changeAccountId() { public void testInitializer(){ Map aktoDataTypeMap = new HashMap<>(); - aktoDataTypeMap.put("JWT", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()))); - aktoDataTypeMap.put("PHONE_NUMBER", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()))); - aktoDataTypeMap.put("CREDIT_CARD", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()))); - aktoDataTypeMap.put("IP_ADDRESS", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()))); - aktoDataTypeMap.put("EMAIL", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()))); - aktoDataTypeMap.put("SSN", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()))); - aktoDataTypeMap.put("UUID", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()))); - aktoDataTypeMap.put("URL", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()))); AccountDataTypesInfo info = SingleTypeInfo.getAccountToDataTypesInfo().get(ACCOUNT_ID); + aktoDataTypeMap.put("JWT", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()), false, true)); + aktoDataTypeMap.put("PHONE_NUMBER", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()), false, true)); + aktoDataTypeMap.put("CREDIT_CARD", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()), false, true)); + aktoDataTypeMap.put("IP_ADDRESS", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()), false, true)); + aktoDataTypeMap.put("EMAIL", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()), false, true)); + aktoDataTypeMap.put("SSN", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()), false, true)); + aktoDataTypeMap.put("UUID", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()), false, true)); + aktoDataTypeMap.put("URL", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()), false, true)); AccountDataTypesInfo info = SingleTypeInfo.getAccountToDataTypesInfo().get(ACCOUNT_ID); if (info == null) { info = new AccountDataTypesInfo(); } diff --git a/apps/api-runtime/src/test/java/com/akto/parsers/TestDump2.java b/apps/api-runtime/src/test/java/com/akto/parsers/TestDump2.java index c99dddc96f..cc4bb28a7b 100644 --- a/apps/api-runtime/src/test/java/com/akto/parsers/TestDump2.java +++ b/apps/api-runtime/src/test/java/com/akto/parsers/TestDump2.java @@ -34,14 +34,14 @@ public class TestDump2 extends MongoBasedTest { public void testInitializer(){ Context.accountId.set(ACCOUNT_ID); Map aktoDataTypeMap = new HashMap<>(); - aktoDataTypeMap.put("JWT", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()))); - aktoDataTypeMap.put("PHONE_NUMBER", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()))); - aktoDataTypeMap.put("CREDIT_CARD", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()))); - aktoDataTypeMap.put("IP_ADDRESS", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()))); - aktoDataTypeMap.put("EMAIL", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()))); - aktoDataTypeMap.put("SSN", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()))); - aktoDataTypeMap.put("UUID", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()))); - aktoDataTypeMap.put("URL", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()))); AccountDataTypesInfo info = SingleTypeInfo.getAccountToDataTypesInfo().get(ACCOUNT_ID); + aktoDataTypeMap.put("JWT", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()), false, true)); + aktoDataTypeMap.put("PHONE_NUMBER", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()), false, true)); + aktoDataTypeMap.put("CREDIT_CARD", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()), false, true)); + aktoDataTypeMap.put("IP_ADDRESS", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()), false, true)); + aktoDataTypeMap.put("EMAIL", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()), false, true)); + aktoDataTypeMap.put("SSN", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()), false, true)); + aktoDataTypeMap.put("UUID", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()), false, true)); + aktoDataTypeMap.put("URL", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()), false, true)); AccountDataTypesInfo info = SingleTypeInfo.getAccountToDataTypesInfo().get(ACCOUNT_ID); if (info == null) { info = new AccountDataTypesInfo(); } diff --git a/apps/api-runtime/src/test/java/com/akto/parsers/TestMergingNew.java b/apps/api-runtime/src/test/java/com/akto/parsers/TestMergingNew.java index 6d2e1c659e..b020881b34 100644 --- a/apps/api-runtime/src/test/java/com/akto/parsers/TestMergingNew.java +++ b/apps/api-runtime/src/test/java/com/akto/parsers/TestMergingNew.java @@ -28,14 +28,14 @@ public class TestMergingNew extends MongoBasedTest { public void testInitializer(){ Map aktoDataTypeMap = new HashMap<>(); - aktoDataTypeMap.put("JWT", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()))); - aktoDataTypeMap.put("PHONE_NUMBER", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()))); - aktoDataTypeMap.put("CREDIT_CARD", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()))); - aktoDataTypeMap.put("IP_ADDRESS", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()))); - aktoDataTypeMap.put("EMAIL", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()))); - aktoDataTypeMap.put("SSN", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()))); - aktoDataTypeMap.put("UUID", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()))); - aktoDataTypeMap.put("URL", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()))); + aktoDataTypeMap.put("JWT", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()), false, true)); + aktoDataTypeMap.put("PHONE_NUMBER", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()), false, true)); + aktoDataTypeMap.put("CREDIT_CARD", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()), false, true)); + aktoDataTypeMap.put("IP_ADDRESS", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()), false, true)); + aktoDataTypeMap.put("EMAIL", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()), false, true)); + aktoDataTypeMap.put("SSN", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()), false, true)); + aktoDataTypeMap.put("UUID", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()), false, true)); + aktoDataTypeMap.put("URL", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()), false, true)); AccountDataTypesInfo info = SingleTypeInfo.getAccountToDataTypesInfo().get(ACCOUNT_ID); if (info == null) { info = new AccountDataTypesInfo(); diff --git a/apps/api-runtime/src/test/java/com/akto/runtime/TestApiCatalogSync.java b/apps/api-runtime/src/test/java/com/akto/runtime/TestApiCatalogSync.java index 2973700b05..c752183d3b 100644 --- a/apps/api-runtime/src/test/java/com/akto/runtime/TestApiCatalogSync.java +++ b/apps/api-runtime/src/test/java/com/akto/runtime/TestApiCatalogSync.java @@ -27,13 +27,13 @@ public class TestApiCatalogSync extends MongoBasedTest { public void testInitializer() { Map aktoDataTypeMap = new HashMap<>(); - aktoDataTypeMap.put("JWT", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()))); - aktoDataTypeMap.put("PHONE_NUMBER", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()))); - aktoDataTypeMap.put("CREDIT_CARD", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()))); - aktoDataTypeMap.put("IP_ADDRESS", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()))); - aktoDataTypeMap.put("EMAIL", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()))); - aktoDataTypeMap.put("SSN", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()))); - aktoDataTypeMap.put("UUID", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()))); + aktoDataTypeMap.put("JWT", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()), false, true)); + aktoDataTypeMap.put("PHONE_NUMBER", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()), false, true)); + aktoDataTypeMap.put("CREDIT_CARD", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()), false, true)); + aktoDataTypeMap.put("IP_ADDRESS", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()), false, true)); + aktoDataTypeMap.put("EMAIL", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()), false, true)); + aktoDataTypeMap.put("SSN", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()), false, true)); + aktoDataTypeMap.put("UUID", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()), false, true)); AccountDataTypesInfo info = SingleTypeInfo.getAccountToDataTypesInfo().get(ACCOUNT_ID); if (info == null) { info = new AccountDataTypesInfo(); diff --git a/apps/api-runtime/src/test/java/com/akto/utils/SampleDataToSTITest.java b/apps/api-runtime/src/test/java/com/akto/utils/SampleDataToSTITest.java index 1c1c0700aa..1cd133ccb2 100644 --- a/apps/api-runtime/src/test/java/com/akto/utils/SampleDataToSTITest.java +++ b/apps/api-runtime/src/test/java/com/akto/utils/SampleDataToSTITest.java @@ -22,14 +22,14 @@ public class SampleDataToSTITest extends MongoBasedTest{ public void testInitializer(){ Map aktoDataTypeMap = new HashMap<>(); - aktoDataTypeMap.put("JWT", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()))); - aktoDataTypeMap.put("PHONE_NUMBER", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()))); - aktoDataTypeMap.put("CREDIT_CARD", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()))); - aktoDataTypeMap.put("IP_ADDRESS", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()))); - aktoDataTypeMap.put("EMAIL", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()))); - aktoDataTypeMap.put("SSN", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()))); - aktoDataTypeMap.put("UUID", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()))); - aktoDataTypeMap.put("URL", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()))); + aktoDataTypeMap.put("JWT", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()), false, true)); + aktoDataTypeMap.put("PHONE_NUMBER", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()), false, true)); + aktoDataTypeMap.put("CREDIT_CARD", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()), false, true)); + aktoDataTypeMap.put("IP_ADDRESS", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()), false, true)); + aktoDataTypeMap.put("EMAIL", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()), false, true)); + aktoDataTypeMap.put("SSN", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()), false, true)); + aktoDataTypeMap.put("UUID", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()), false, true)); + aktoDataTypeMap.put("URL", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()), false, true)); AccountDataTypesInfo info = SingleTypeInfo.getAccountToDataTypesInfo().get(ACCOUNT_ID); if (info == null) { info = new AccountDataTypesInfo(); diff --git a/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java b/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java index 339530a1cf..6d6c52e3f0 100644 --- a/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java +++ b/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java @@ -1058,14 +1058,14 @@ public static void addAktoDataTypes(BackwardCompatibility backwardCompatibility) List aktoDataTypes = new ArrayList<>(); int now = Context.now(); IgnoreData ignoreData = new IgnoreData(new HashMap<>(), new HashSet<>()); - aktoDataTypes.add(new AktoDataType("JWT", false, Arrays.asList(SingleTypeInfo.Position.RESPONSE_PAYLOAD, SingleTypeInfo.Position.RESPONSE_HEADER), now, ignoreData)); - aktoDataTypes.add(new AktoDataType("EMAIL", true, Collections.emptyList(), now, ignoreData)); - aktoDataTypes.add(new AktoDataType("CREDIT_CARD", true, Collections.emptyList(), now, ignoreData)); - aktoDataTypes.add(new AktoDataType("SSN", true, Collections.emptyList(), now, ignoreData)); - aktoDataTypes.add(new AktoDataType("ADDRESS", true, Collections.emptyList(), now, ignoreData)); - aktoDataTypes.add(new AktoDataType("IP_ADDRESS", false, Arrays.asList(SingleTypeInfo.Position.RESPONSE_PAYLOAD, SingleTypeInfo.Position.RESPONSE_HEADER), now, ignoreData)); - aktoDataTypes.add(new AktoDataType("PHONE_NUMBER", true, Collections.emptyList(), now, ignoreData)); - aktoDataTypes.add(new AktoDataType("UUID", false, Collections.emptyList(), now, ignoreData)); + aktoDataTypes.add(new AktoDataType("JWT", false, Arrays.asList(SingleTypeInfo.Position.RESPONSE_PAYLOAD, SingleTypeInfo.Position.RESPONSE_HEADER), now, ignoreData, false, true)); + aktoDataTypes.add(new AktoDataType("EMAIL", true, Collections.emptyList(), now, ignoreData, false, true)); + aktoDataTypes.add(new AktoDataType("CREDIT_CARD", true, Collections.emptyList(), now, ignoreData, false, true)); + aktoDataTypes.add(new AktoDataType("SSN", true, Collections.emptyList(), now, ignoreData, false, true)); + aktoDataTypes.add(new AktoDataType("ADDRESS", true, Collections.emptyList(), now, ignoreData, false, true)); + aktoDataTypes.add(new AktoDataType("IP_ADDRESS", false, Arrays.asList(SingleTypeInfo.Position.RESPONSE_PAYLOAD, SingleTypeInfo.Position.RESPONSE_HEADER), now, ignoreData, false, true)); + aktoDataTypes.add(new AktoDataType("PHONE_NUMBER", true, Collections.emptyList(), now, ignoreData, false, true)); + aktoDataTypes.add(new AktoDataType("UUID", false, Collections.emptyList(), now, ignoreData, false, true)); AktoDataTypeDao.instance.getMCollection().drop(); AktoDataTypeDao.instance.insertMany(aktoDataTypes); diff --git a/apps/dashboard/src/test/java/com/akto/action/TestIgnoreFalsePositivesAction.java b/apps/dashboard/src/test/java/com/akto/action/TestIgnoreFalsePositivesAction.java index 0a39c8aae9..2a331f3daa 100644 --- a/apps/dashboard/src/test/java/com/akto/action/TestIgnoreFalsePositivesAction.java +++ b/apps/dashboard/src/test/java/com/akto/action/TestIgnoreFalsePositivesAction.java @@ -24,7 +24,7 @@ public class TestIgnoreFalsePositivesAction extends MongoBasedTest{ @Test public void test(){ AktoDataTypeDao.instance.getMCollection().drop(); - AktoDataType aktoDataType = new AktoDataType("UUID", false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>())); + AktoDataType aktoDataType = new AktoDataType("UUID", false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()), false, true); Set ignored = new HashSet<>(); ignored.add("something"); aktoDataType.getIgnoreData().setIgnoredKeysInAllAPIs(ignored); diff --git a/libs/dao/src/main/java/com/akto/dto/AktoDataType.java b/libs/dao/src/main/java/com/akto/dto/AktoDataType.java index 8ed57aeb96..8b6b9faccf 100644 --- a/libs/dao/src/main/java/com/akto/dto/AktoDataType.java +++ b/libs/dao/src/main/java/com/akto/dto/AktoDataType.java @@ -18,15 +18,6 @@ public class AktoDataType { public AktoDataType() { } - public AktoDataType(String name, boolean sensitiveAlways, List sensitivePosition,int timestamp, IgnoreData ignoreData) { - this.name = name; - this.sensitiveAlways = sensitiveAlways; - this.sensitivePosition = sensitivePosition; - this.ignoreData = ignoreData; - this.redacted = false; - this.sampleDataFixed = true; - } - public AktoDataType(String name, boolean sensitiveAlways, List sensitivePosition,int timestamp, IgnoreData ignoreData, boolean redacted, boolean sampleDataFixed) { this.name = name; this.sensitiveAlways = sensitiveAlways; diff --git a/libs/dao/src/test/java/com/akto/dto/type/TestKeyTypes.java b/libs/dao/src/test/java/com/akto/dto/type/TestKeyTypes.java index 0bf8fc6a71..f365163098 100644 --- a/libs/dao/src/test/java/com/akto/dto/type/TestKeyTypes.java +++ b/libs/dao/src/test/java/com/akto/dto/type/TestKeyTypes.java @@ -21,13 +21,13 @@ public class TestKeyTypes { public void testInitializer() { Context.accountId.set(ACCOUNT_ID); Map aktoDataTypeMap = new HashMap<>(); - aktoDataTypeMap.put("JWT", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()))); - aktoDataTypeMap.put("PHONE_NUMBER", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()))); - aktoDataTypeMap.put("CREDIT_CARD", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()))); - aktoDataTypeMap.put("IP_ADDRESS", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()))); - aktoDataTypeMap.put("EMAIL", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()))); - aktoDataTypeMap.put("SSN", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()))); - aktoDataTypeMap.put("UUID", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()))); + aktoDataTypeMap.put("JWT", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()), false, true)); + aktoDataTypeMap.put("PHONE_NUMBER", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()), false, true)); + aktoDataTypeMap.put("CREDIT_CARD", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()), false, true)); + aktoDataTypeMap.put("IP_ADDRESS", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()), false, true)); + aktoDataTypeMap.put("EMAIL", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()), false, true)); + aktoDataTypeMap.put("SSN", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()), false, true)); + aktoDataTypeMap.put("UUID", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()), false, true)); AccountDataTypesInfo info = SingleTypeInfo.getAccountToDataTypesInfo().get(ACCOUNT_ID); if (info == null) { info = new AccountDataTypesInfo(); diff --git a/libs/dao/src/test/java/com/akto/dto/type/TestSubType.java b/libs/dao/src/test/java/com/akto/dto/type/TestSubType.java index 5411bc6ab5..17022a5b7f 100644 --- a/libs/dao/src/test/java/com/akto/dto/type/TestSubType.java +++ b/libs/dao/src/test/java/com/akto/dto/type/TestSubType.java @@ -17,10 +17,10 @@ public class TestSubType { public void testInitializer(){ Context.accountId.set(ACCOUNT_ID); Map aktoDataTypeMap = new HashMap<>(); - aktoDataTypeMap.put("JWT", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()))); - aktoDataTypeMap.put("PHONE_NUMBER", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()))); - aktoDataTypeMap.put("CREDIT_CARD", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()))); - aktoDataTypeMap.put("IP_ADDRESS", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()))); + aktoDataTypeMap.put("JWT", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()), false, true)); + aktoDataTypeMap.put("PHONE_NUMBER", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()), false, true)); + aktoDataTypeMap.put("CREDIT_CARD", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()), false, true)); + aktoDataTypeMap.put("IP_ADDRESS", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()), false, true)); AccountDataTypesInfo info = SingleTypeInfo.getAccountToDataTypesInfo().get(ACCOUNT_ID); if (info == null) { info = new AccountDataTypesInfo(); From e3a5628d4da69ea117c4506a8474fafb6ae2e186 Mon Sep 17 00:00:00 2001 From: aktoboy Date: Wed, 10 Jan 2024 12:19:58 +0530 Subject: [PATCH 16/39] Removed original constructor for CustomDataType --- .../com/akto/listener/InitializerListener.java | 4 +++- .../com/akto/action/TestCustomDataTypeAction.java | 8 ++++---- .../main/java/com/akto/dto/CustomDataType.java | 15 --------------- .../java/com/akto/dao/TestSingleTypeInfoDao.java | 6 +++--- .../akto/dto/data_types/TestCustomDataType.java | 2 +- .../test/java/com/akto/dto/type/TestKeyTypes.java | 4 ++-- 6 files changed, 13 insertions(+), 26 deletions(-) diff --git a/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java b/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java index 6d6c52e3f0..e95c3fa9d5 100644 --- a/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java +++ b/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java @@ -565,7 +565,9 @@ private static CustomDataType getCustomDataTypeFromPiiType(PIISource piiSource, (piiType.getOnKey() ? conditions : null), (piiType.getOnKey() ? null : conditions), Operator.OR, - ignoreData + ignoreData, + false, + true ); return ret; diff --git a/apps/dashboard/src/test/java/com/akto/action/TestCustomDataTypeAction.java b/apps/dashboard/src/test/java/com/akto/action/TestCustomDataTypeAction.java index 850edcce93..06dd8dd555 100644 --- a/apps/dashboard/src/test/java/com/akto/action/TestCustomDataTypeAction.java +++ b/apps/dashboard/src/test/java/com/akto/action/TestCustomDataTypeAction.java @@ -179,17 +179,17 @@ public void testGenerateCustomDataTypeNewInvalidValueMap() { @Test public void testFetchDataTypes() { IgnoreData ignoreData = new IgnoreData(new HashMap<>(), new HashSet<>()); - CustomDataType customDataType1 = new CustomDataType("name1", true, Collections.emptyList(), 1, true, null,null, Conditions.Operator.AND,ignoreData); + CustomDataType customDataType1 = new CustomDataType("name1", true, Collections.emptyList(), 1, true, null,null, Conditions.Operator.AND,ignoreData, false, true); User user1 = new User(); user1.setId(1); user1.setName("user1"); - CustomDataType customDataType2 = new CustomDataType("name2", true, Collections.emptyList(), 2,false, null,null, Conditions.Operator.AND,ignoreData); + CustomDataType customDataType2 = new CustomDataType("name2", true, Collections.emptyList(), 2,false, null,null, Conditions.Operator.AND,ignoreData, false, true); User user2 = new User(); user2.setId(2); user2.setName("user1"); - CustomDataType customDataType3 = new CustomDataType("name3", true, Collections.emptyList(), 3, true, null,null, Conditions.Operator.AND,ignoreData); + CustomDataType customDataType3 = new CustomDataType("name3", true, Collections.emptyList(), 3, true, null,null, Conditions.Operator.AND,ignoreData, false, true); User user3 = new User(); user3.setId(3); user3.setName("user1"); @@ -226,7 +226,7 @@ public void testToggleDataTypeActiveParam() { Context.accountId.set(1_000_000); CustomDataTypeDao.instance.getMCollection().drop(); IgnoreData ignoreData = new IgnoreData(new HashMap<>(), new HashSet<>()); - CustomDataType customDataType = new CustomDataType("NAME1", true, Collections.emptyList(), 1, true, null,null, Conditions.Operator.AND,ignoreData); + CustomDataType customDataType = new CustomDataType("NAME1", true, Collections.emptyList(), 1, true, null,null, Conditions.Operator.AND,ignoreData, false, true); CustomDataTypeDao.instance.insertOne(customDataType); CustomDataTypeAction customDataTypeAction = new CustomDataTypeAction(); diff --git a/libs/dao/src/main/java/com/akto/dto/CustomDataType.java b/libs/dao/src/main/java/com/akto/dto/CustomDataType.java index 412031726c..917d5c29c2 100644 --- a/libs/dao/src/main/java/com/akto/dto/CustomDataType.java +++ b/libs/dao/src/main/java/com/akto/dto/CustomDataType.java @@ -38,21 +38,6 @@ public class CustomDataType { public CustomDataType() { } - public CustomDataType(String name, boolean sensitiveAlways, List sensitivePosition, int creatorId, boolean active, Conditions keyConditions, Conditions valueConditions, Conditions.Operator operator, IgnoreData ignoreData) { - this.name = name; - this.sensitiveAlways = sensitiveAlways; - this.sensitivePosition = sensitivePosition; - this.creatorId = creatorId; - this.timestamp = Context.now(); - this.active = active; - this.keyConditions = keyConditions; - this.valueConditions = valueConditions; - this.operator = operator; - this.ignoreData = ignoreData; - this.redacted = false; - this.sampleDataFixed = true; - } - public CustomDataType(String name, boolean sensitiveAlways, List sensitivePosition, int creatorId, boolean active, Conditions keyConditions, Conditions valueConditions, Conditions.Operator operator, IgnoreData ignoreData, boolean redacted, boolean sampleDataFixed) { this.name = name; this.sensitiveAlways = sensitiveAlways; diff --git a/libs/dao/src/test/java/com/akto/dao/TestSingleTypeInfoDao.java b/libs/dao/src/test/java/com/akto/dao/TestSingleTypeInfoDao.java index 794ca8e2cc..27a1dd842d 100644 --- a/libs/dao/src/test/java/com/akto/dao/TestSingleTypeInfoDao.java +++ b/libs/dao/src/test/java/com/akto/dao/TestSingleTypeInfoDao.java @@ -68,7 +68,7 @@ public void testInsertAndFetchCustom() { Conditions valueConditions = new Conditions(Collections.singletonList(new EndsWithPredicate("something")), Conditions.Operator.OR); CustomDataType customDataType = new CustomDataType( "custom1", false, Arrays.asList(SingleTypeInfo.Position.REQUEST_PAYLOAD, SingleTypeInfo.Position.RESPONSE_PAYLOAD), - 0,true, keyConditions, valueConditions, Conditions.Operator.OR,ignoreData + 0,true, keyConditions, valueConditions, Conditions.Operator.OR,ignoreData, false, true ); CustomDataTypeDao.instance.insertOne(customDataType); @@ -115,8 +115,8 @@ public void testFilterForSensitiveParamsExcludingUserMarkedSensitive() { bulkWrites.add(createSingleTypeInfoUpdate("C", "GET", SingleTypeInfo.JWT, 0,200)); bulkWrites.add(createSingleTypeInfoUpdate("D", "POST", SingleTypeInfo.JWT, 0,-1)); IgnoreData ignoreData = new IgnoreData(new HashMap<>(), new HashSet<>()); - CustomDataType customDataType1 = new CustomDataType("CUSTOM_DATA_1", true, Collections.emptyList(), 0,true, null,null, Conditions.Operator.AND,ignoreData); - CustomDataType customDataType2 = new CustomDataType("CUSTOM_DATA_2",false, Collections.emptyList(), 0,true, null,null, Conditions.Operator.AND,ignoreData); + CustomDataType customDataType1 = new CustomDataType("CUSTOM_DATA_1", true, Collections.emptyList(), 0,true, null,null, Conditions.Operator.AND,ignoreData, false, true); + CustomDataType customDataType2 = new CustomDataType("CUSTOM_DATA_2",false, Collections.emptyList(), 0,true, null,null, Conditions.Operator.AND,ignoreData, false, true); CustomDataTypeDao.instance.insertMany(Arrays.asList(customDataType1, customDataType2)); bulkWrites.add(createSingleTypeInfoUpdate("E", "POST",customDataType1.toSubType(), 0,200)); diff --git a/libs/dao/src/test/java/com/akto/dto/data_types/TestCustomDataType.java b/libs/dao/src/test/java/com/akto/dto/data_types/TestCustomDataType.java index 89cf1c540c..bf7bb7ff28 100644 --- a/libs/dao/src/test/java/com/akto/dto/data_types/TestCustomDataType.java +++ b/libs/dao/src/test/java/com/akto/dto/data_types/TestCustomDataType.java @@ -35,7 +35,7 @@ public void testValidate() { IgnoreData ignoreData = new IgnoreData(new HashMap<>(), new HashSet<>()); CustomDataType customDataType = new CustomDataType("name", true, Collections.emptyList(), - 0,true, conditions2, conditions1, Conditions.Operator.AND,ignoreData); + 0,true, conditions2, conditions1, Conditions.Operator.AND,ignoreData, false, true); assertTrue(customDataType.validate("w13rd_something_a55", "Dope_shit")); assertFalse(customDataType.validate("w13rd_something_a55", "BDope_shit")); diff --git a/libs/dao/src/test/java/com/akto/dto/type/TestKeyTypes.java b/libs/dao/src/test/java/com/akto/dto/type/TestKeyTypes.java index f365163098..617e861e14 100644 --- a/libs/dao/src/test/java/com/akto/dto/type/TestKeyTypes.java +++ b/libs/dao/src/test/java/com/akto/dto/type/TestKeyTypes.java @@ -53,9 +53,9 @@ public void testProcess() { HashMap customDataTypeMap = new HashMap<>(); IgnoreData ignoreData = new IgnoreData(new HashMap<>(), new HashSet<>()); CustomDataType customDataType1 = new CustomDataType("SHIPPING", true, Collections.emptyList(), - 1, true, new Conditions(Collections.singletonList(new StartsWithPredicate("ship")), Conditions.Operator.AND), null, Conditions.Operator.AND, ignoreData); + 1, true, new Conditions(Collections.singletonList(new StartsWithPredicate("ship")), Conditions.Operator.AND), null, Conditions.Operator.AND, ignoreData, false, true); CustomDataType customDataType2 = new CustomDataType("CAPTAIN", false, Collections.emptyList(), - 1, true, new Conditions(Collections.singletonList(new StartsWithPredicate("captain")), Conditions.Operator.AND), null, Conditions.Operator.AND, ignoreData); + 1, true, new Conditions(Collections.singletonList(new StartsWithPredicate("captain")), Conditions.Operator.AND), null, Conditions.Operator.AND, ignoreData, false, true); customDataTypeMap.put("SHIPPING", customDataType1); customDataTypeMap.put("CAPTAIN", customDataType2); From 3398abbc2ea5c149aba2dfd3bf259798593e2cca Mon Sep 17 00:00:00 2001 From: aktoboy Date: Wed, 10 Jan 2024 15:21:42 +0530 Subject: [PATCH 17/39] Removed duplicate code from RedactSampleData, removed original constructor for Apicollection object --- .../java/com/akto/runtime/APICatalogSync.java | 2 +- .../src/main/java/com/akto/runtime/Main.java | 2 +- .../akto/runtime/merge/MergeOnHostOnly.java | 2 +- .../java/com/akto/utils/RedactSampleData.java | 133 +++++------------- .../java/com/akto/parsers/TestDBSync.java | 8 +- .../akto/runtime/TestMainSubFunctions.java | 4 +- .../runtime/merge/TestMergeOnHostOnly.java | 10 +- .../com/akto/utils/TestRedactSampleData.java | 4 +- .../java/com/akto/action/AccountAction.java | 2 +- .../com/akto/action/ApiCollectionsAction.java | 4 +- .../com/akto/listener/RuntimeListener.java | 4 +- .../akto/action/TestApiCollectionsAction.java | 6 +- .../action/observe/TestInventoryAction.java | 2 +- .../listener/TestInitializerListener.java | 6 +- .../main/java/com/akto/dto/ApiCollection.java | 13 +- .../com/akto/dao/TestApiCollectionsDao.java | 8 +- .../test/java/com/akto/dao/VCDemoData.java | 6 +- 17 files changed, 68 insertions(+), 148 deletions(-) diff --git a/apps/api-runtime/src/main/java/com/akto/runtime/APICatalogSync.java b/apps/api-runtime/src/main/java/com/akto/runtime/APICatalogSync.java index 7163471dac..ef2ba65828 100644 --- a/apps/api-runtime/src/main/java/com/akto/runtime/APICatalogSync.java +++ b/apps/api-runtime/src/main/java/com/akto/runtime/APICatalogSync.java @@ -1077,7 +1077,7 @@ public DbUpdateReturn getDBUpdatesForParams(APICatalog currentDelta, APICatalog for (Object example : deltaInfo.getExamples()) { try{ String exampleStr = (String) example; - String s = RedactSampleData.redactMarkedDataTypes(exampleStr); + String s = RedactSampleData.redactDataTypes(exampleStr); updatedSampleData.add(s); } catch (Exception e) { ; diff --git a/apps/api-runtime/src/main/java/com/akto/runtime/Main.java b/apps/api-runtime/src/main/java/com/akto/runtime/Main.java index c06b4e2470..65532dd0a1 100644 --- a/apps/api-runtime/src/main/java/com/akto/runtime/Main.java +++ b/apps/api-runtime/src/main/java/com/akto/runtime/Main.java @@ -347,7 +347,7 @@ public static void initializeRuntimeHelper() { for(SingleTypeInfo singleTypeInfo: SingleTypeInfoDao.instance.fetchAll()) { urls.add(singleTypeInfo.getUrl()); } - ApiCollectionsDao.instance.insertOne(new ApiCollection(0, "Default", Context.now(), urls, null, 0)); + ApiCollectionsDao.instance.insertOne(new ApiCollection(0, "Default", Context.now(), urls, null, 0, false, true)); } } diff --git a/apps/api-runtime/src/main/java/com/akto/runtime/merge/MergeOnHostOnly.java b/apps/api-runtime/src/main/java/com/akto/runtime/merge/MergeOnHostOnly.java index 296c0e2a78..32f023feed 100644 --- a/apps/api-runtime/src/main/java/com/akto/runtime/merge/MergeOnHostOnly.java +++ b/apps/api-runtime/src/main/java/com/akto/runtime/merge/MergeOnHostOnly.java @@ -171,7 +171,7 @@ public void mergeHostUtil(String host, List apiCollectionIds) { old.setId(newApiCollectionId); try { - ApiCollectionsDao.instance.insertOne(new ApiCollection(newApiCollectionId, null, old.getStartTs(), new HashSet<>(), host, 0)); + ApiCollectionsDao.instance.insertOne(new ApiCollection(newApiCollectionId, null, old.getStartTs(), new HashSet<>(), host, 0, false, true)); } catch (Exception e) { return; } diff --git a/apps/api-runtime/src/main/java/com/akto/utils/RedactSampleData.java b/apps/api-runtime/src/main/java/com/akto/utils/RedactSampleData.java index df1cbe9814..29a00e3641 100644 --- a/apps/api-runtime/src/main/java/com/akto/utils/RedactSampleData.java +++ b/apps/api-runtime/src/main/java/com/akto/utils/RedactSampleData.java @@ -26,70 +26,18 @@ public static String redactIfRequired(String sample, boolean accountLevelRedact, HttpResponseParams httpResponseParams = HttpCallParser.parseKafkaMessage(sample); HttpResponseParams.Source source = httpResponseParams.getSource(); // if(source.equals(HttpResponseParams.Source.HAR) || source.equals(HttpResponseParams.Source.PCAP)) return sample; - if(accountLevelRedact || apiCollectionLevelRedact) return redact(httpResponseParams); - return redactMarkedDataTypes(httpResponseParams); + return redact(httpResponseParams, accountLevelRedact || apiCollectionLevelRedact); } - public static String redact(String sample) throws Exception { - HttpResponseParams httpResponseParams = HttpCallParser.parseKafkaMessage(sample); - return redact(httpResponseParams); + public static String redactDataTypes(String sample) throws Exception{ + return redact(HttpCallParser.parseKafkaMessage(sample), false); } - public static String redactMarkedDataTypes(String sample) throws Exception { - HttpResponseParams httpResponseParams = HttpCallParser.parseKafkaMessage(sample); - return redactMarkedDataTypes(httpResponseParams); - } - - public static String redactMarkedDataTypes(HttpResponseParams httpResponseParams) throws Exception{ - - Map> responseHeaders = httpResponseParams.getHeaders(); - if (responseHeaders == null) responseHeaders = new HashMap<>(); - handleHeaders(responseHeaders); - - String responsePayload = httpResponseParams.getPayload(); - if (responsePayload == null) responsePayload = "{}"; - try { - JsonParser jp = factory.createParser(responsePayload); - JsonNode node = mapper.readTree(jp); - changeRedactedDataType(null, node, redactValue); - if (node != null) { - responsePayload = node.toString(); - } else { - responsePayload = "{}"; - } - } catch (Exception e) { - responsePayload = "{}"; - } - httpResponseParams.setPayload(responsePayload); - - Map> requestHeaders = httpResponseParams.requestParams.getHeaders(); - if (requestHeaders == null) requestHeaders = new HashMap<>(); - handleHeaders(requestHeaders); - - String requestPayload = httpResponseParams.requestParams.getPayload(); - if (requestPayload == null) requestPayload = "{}"; - try { - JsonParser jp = factory.createParser(requestPayload); - JsonNode node = mapper.readTree(jp); - changeRedactedDataType(null, node, redactValue); - if (node != null) { - requestPayload= node.toString(); - } else { - requestPayload = "{}"; - } - } catch (Exception e) { - requestPayload = "{}"; + private static void handleHeaders(Map> responseHeaders, boolean redactAll) { + if(redactAll){ + responseHeaders.replaceAll((n, v) -> Collections.singletonList(redactValue)); + return; } - - httpResponseParams.requestParams.setPayload(requestPayload); - - //TODO confirm if this is required or not - httpResponseParams.setSourceIP(redactValue); - - return convertHttpRespToOriginalString(httpResponseParams); - } - - private static void handleHeaders(Map> responseHeaders) { Set>> entries = responseHeaders.entrySet(); for(Map.Entry> entry : entries){ String key = entry.getKey(); @@ -102,11 +50,11 @@ private static void handleHeaders(Map> responseHeaders) { } // never use this function directly. This alters the httpResponseParams - public static String redact(HttpResponseParams httpResponseParams) throws Exception { + public static String redact(HttpResponseParams httpResponseParams, final boolean redactAll) throws Exception { // response headers Map> responseHeaders = httpResponseParams.getHeaders(); if (responseHeaders == null) responseHeaders = new HashMap<>(); - responseHeaders.replaceAll((n, v) -> Collections.singletonList(redactValue)); + handleHeaders(responseHeaders, redactAll); // response payload String responsePayload = httpResponseParams.getPayload(); @@ -114,7 +62,7 @@ public static String redact(HttpResponseParams httpResponseParams) throws Except try { JsonParser jp = factory.createParser(responsePayload); JsonNode node = mapper.readTree(jp); - change(node, redactValue); + change(null, node, redactValue, redactAll); if (node != null) { responsePayload = node.toString(); } else { @@ -129,7 +77,7 @@ public static String redact(HttpResponseParams httpResponseParams) throws Except // request headers Map> requestHeaders = httpResponseParams.requestParams.getHeaders(); if (requestHeaders == null) requestHeaders = new HashMap<>(); - requestHeaders.replaceAll((n, v) -> Collections.singletonList(redactValue)); + handleHeaders(requestHeaders, redactAll); // request payload String requestPayload = httpResponseParams.requestParams.getPayload(); @@ -137,7 +85,7 @@ public static String redact(HttpResponseParams httpResponseParams) throws Except try { JsonParser jp = factory.createParser(requestPayload); JsonNode node = mapper.readTree(jp); - change(node, redactValue); + change(null, node, redactValue, redactAll); if (node != null) { requestPayload= node.toString(); } else { @@ -150,13 +98,15 @@ public static String redact(HttpResponseParams httpResponseParams) throws Except httpResponseParams.requestParams.setPayload(requestPayload); // ip - httpResponseParams.setSourceIP(redactValue); + if(redactAll) { + httpResponseParams.setSourceIP(redactValue); + } return convertHttpRespToOriginalString(httpResponseParams); } - public static void change(JsonNode parent, String newValue) { + public static void change(String parentName, JsonNode parent, String newValue, boolean redactAll) { if (parent == null) return; if (parent.isArray()) { @@ -164,40 +114,16 @@ public static void change(JsonNode parent, String newValue) { for(int i = 0; i < arrayNode.size(); i++) { JsonNode arrayElement = arrayNode.get(i); if (arrayElement.isValueNode()) { - arrayNode.set(i, new TextNode(newValue)); - } else { - change(arrayElement, newValue); - } - } - } else { - Iterator fieldNames = parent.fieldNames(); - while(fieldNames.hasNext()) { - String f = fieldNames.next(); - JsonNode fieldValue = parent.get(f); - if (fieldValue.isValueNode()) { - ((ObjectNode) parent).put(f, newValue); - } else { - change(fieldValue, newValue); - } - } - } - - } - - public static void changeRedactedDataType(String parentName, JsonNode parent, String newValue) { - if (parent == null) return; - - if (parent.isArray()) { - ArrayNode arrayNode = (ArrayNode) parent; - for(int i = 0; i < arrayNode.size(); i++) { - JsonNode arrayElement = arrayNode.get(i); - if (arrayElement.isValueNode()) { - SingleTypeInfo.SubType subType = KeyTypes.findSubType(arrayElement.asText(), parentName, null); - if(SingleTypeInfo.isRedacted(subType.getName())){ - arrayNode.set(i, new TextNode(newValue)); + if(redactAll){ + arrayNode.set(i, new TextNode(redactValue)); + } else{ + SingleTypeInfo.SubType subType = KeyTypes.findSubType(arrayElement.asText(), parentName, null); + if(SingleTypeInfo.isRedacted(subType.getName())){ + arrayNode.set(i, new TextNode(newValue)); + } } } else { - changeRedactedDataType(parentName, arrayElement, newValue); + change(parentName, arrayElement, newValue, redactAll); } } } else { @@ -206,13 +132,18 @@ public static void changeRedactedDataType(String parentName, JsonNode parent, St String f = fieldNames.next(); JsonNode fieldValue = parent.get(f); if (fieldValue.isValueNode()) { - SingleTypeInfo.SubType subType = KeyTypes.findSubType(fieldValue.asText(), f, null); - if(SingleTypeInfo.isRedacted(subType.getName())){ + if(redactAll){ ((ObjectNode) parent).put(f, newValue); } + else { + SingleTypeInfo.SubType subType = KeyTypes.findSubType(fieldValue.asText(), f, null); + if (SingleTypeInfo.isRedacted(subType.getName())) { + ((ObjectNode) parent).put(f, newValue); + } + } } else { - changeRedactedDataType(f, fieldValue, newValue); + change(f, fieldValue, newValue, redactAll); } } } diff --git a/apps/api-runtime/src/test/java/com/akto/parsers/TestDBSync.java b/apps/api-runtime/src/test/java/com/akto/parsers/TestDBSync.java index 0253f82b88..1fd6a9b471 100644 --- a/apps/api-runtime/src/test/java/com/akto/parsers/TestDBSync.java +++ b/apps/api-runtime/src/test/java/com/akto/parsers/TestDBSync.java @@ -293,7 +293,7 @@ public void testFilterHttpResponseParamsWithoutHost() { int vxlanId1 = 1; String domain1 = "domain1.com"; - ApiCollectionsDao.instance.insertOne(new ApiCollection(vxlanId1, groupName1, 0, new HashSet<>(), null, 0)); + ApiCollectionsDao.instance.insertOne(new ApiCollection(vxlanId1, groupName1, 0, new HashSet<>(), null, 0, false, true)); HttpResponseParams h1 = new HttpResponseParams(); h1.requestParams = new HttpRequestParams(); @@ -354,7 +354,7 @@ public void testFilterResponseParamsWithHost() { int vxlanId1 = 1; String domain1 = "domain1.com"; - ApiCollectionsDao.instance.insertOne(new ApiCollection(vxlanId1, groupName1, 0, new HashSet<>(), null, 0)); + ApiCollectionsDao.instance.insertOne(new ApiCollection(vxlanId1, groupName1, 0, new HashSet<>(), null, 0, false, true)); HttpResponseParams h1 = new HttpResponseParams(); h1.requestParams = new HttpRequestParams(); @@ -443,7 +443,7 @@ public void testFilterResponseParamsWithHost() { // before processing inserting apiCollection with same id but different vxlanId and host int dupId = domain4.hashCode(); ApiCollectionsDao.instance.insertOne( - new ApiCollection(dupId,"something", 0, new HashSet<>(), "hostRandom", 1234) + new ApiCollection(dupId,"something", 0, new HashSet<>(), "hostRandom", 1234, false, true) ); httpCallParser.getHostNameToIdMap().put("hostRandom 1234", dupId); @@ -461,7 +461,7 @@ public void testFilterResponseParamsWithHost() { @Test public void testCollisionHostNameCollection() { ApiCollectionsDao.instance.getMCollection().drop(); - ApiCollectionsDao.instance.insertOne(new ApiCollection(0, "domain", 0, new HashSet<>(), null, 0)); + ApiCollectionsDao.instance.insertOne(new ApiCollection(0, "domain", 0, new HashSet<>(), null, 0, false, true)); HttpResponseParams h1 = new HttpResponseParams(); h1.requestParams = new HttpRequestParams(); h1.requestParams.setHeaders(new HashMap<>()); diff --git a/apps/api-runtime/src/test/java/com/akto/runtime/TestMainSubFunctions.java b/apps/api-runtime/src/test/java/com/akto/runtime/TestMainSubFunctions.java index 55f03bb829..5f7dc32ecf 100644 --- a/apps/api-runtime/src/test/java/com/akto/runtime/TestMainSubFunctions.java +++ b/apps/api-runtime/src/test/java/com/akto/runtime/TestMainSubFunctions.java @@ -38,7 +38,7 @@ public void testTryForCollectionNameWithoutCidr() throws JsonProcessingException String group_name = "bb"; ApiCollectionsDao.instance.insertOne(new ApiCollection( - vxlan_id, null, 0, new HashSet<>(), null, vxlan_id + vxlan_id, null, 0, new HashSet<>(), null, vxlan_id, false, true )); Map m = new HashMap<>(); @@ -65,7 +65,7 @@ public void testTryForCollectionNameWithCidr() throws JsonProcessingException { List vpc_cidr = Arrays.asList("192.1.1.1/16", "193.1.1.1/16"); ApiCollectionsDao.instance.insertOne(new ApiCollection( - vxlan_id, null, 0, new HashSet<>(), null, vxlan_id + vxlan_id, null, 0, new HashSet<>(), null, vxlan_id, false, true )); Map m = new HashMap<>(); diff --git a/apps/api-runtime/src/test/java/com/akto/runtime/merge/TestMergeOnHostOnly.java b/apps/api-runtime/src/test/java/com/akto/runtime/merge/TestMergeOnHostOnly.java index 121e6baaa3..865bfddd9b 100644 --- a/apps/api-runtime/src/test/java/com/akto/runtime/merge/TestMergeOnHostOnly.java +++ b/apps/api-runtime/src/test/java/com/akto/runtime/merge/TestMergeOnHostOnly.java @@ -61,12 +61,12 @@ public void test1(){ Set url1 = new HashSet<>(); Set url2 = new HashSet<>(); - ApiCollection ap1 = new ApiCollection(1, "something-red", 0, url1, "http://www.akto.io", 1); - ApiCollection ap2 = new ApiCollection(2, "something-blue", 0, url2, "http://www.akto.io", 2); + ApiCollection ap1 = new ApiCollection(1, "something-red", 0, url1, "http://www.akto.io", 1, false, true); + ApiCollection ap2 = new ApiCollection(2, "something-blue", 0, url2, "http://www.akto.io", 2, false, true); - ApiCollection ap3 = new ApiCollection(3, "something-red", 0, url1, "http://www.notakto.io", 1); - ApiCollection ap4 = new ApiCollection(4, "something-blue", 0, url2, "http://www.notakto.io", 2); - ApiCollection ap5 = new ApiCollection(5, "something-yellow", 0, url2, "http://www.notakto.io", 2); + ApiCollection ap3 = new ApiCollection(3, "something-red", 0, url1, "http://www.notakto.io", 1, false, true); + ApiCollection ap4 = new ApiCollection(4, "something-blue", 0, url2, "http://www.notakto.io", 2, false, true); + ApiCollection ap5 = new ApiCollection(5, "something-yellow", 0, url2, "http://www.notakto.io", 2, false, true); ApiCollectionsDao.instance.getMCollection().insertOne(ap1); ApiCollectionsDao.instance.getMCollection().insertOne(ap2); ApiCollectionsDao.instance.getMCollection().insertOne(ap3); diff --git a/apps/api-runtime/src/test/java/com/akto/utils/TestRedactSampleData.java b/apps/api-runtime/src/test/java/com/akto/utils/TestRedactSampleData.java index b053b91d43..377a57d7eb 100644 --- a/apps/api-runtime/src/test/java/com/akto/utils/TestRedactSampleData.java +++ b/apps/api-runtime/src/test/java/com/akto/utils/TestRedactSampleData.java @@ -205,7 +205,7 @@ public void redactNonJsonPayload() throws Exception { "type", 200, "OK", respHeaders, respPayload, httpRequestParams, 0, "1000000", false, HttpResponseParams.Source.MIRRORING, "orig","172.0.0.1" ); - String redactedValue = RedactSampleData.redact(httpResponseParams); + String redactedValue = RedactSampleData.redact(httpResponseParams, true); HttpResponseParams redactedHttpResponseParams = HttpCallParser.parseKafkaMessage(redactedValue); @@ -273,7 +273,7 @@ public void happy() throws Exception { HttpResponseParams originalHttpResponseParams = HttpCallParser.parseKafkaMessage(originalString); - String redactedValue = RedactSampleData.redact(httpResponseParams); + String redactedValue = RedactSampleData.redact(httpResponseParams, true); HttpResponseParams redactedHttpResponseParams = HttpCallParser.parseKafkaMessage(redactedValue); diff --git a/apps/dashboard/src/main/java/com/akto/action/AccountAction.java b/apps/dashboard/src/main/java/com/akto/action/AccountAction.java index 9b26783aeb..fb489f5cae 100644 --- a/apps/dashboard/src/main/java/com/akto/action/AccountAction.java +++ b/apps/dashboard/src/main/java/com/akto/action/AccountAction.java @@ -292,7 +292,7 @@ private static void intializeCollectionsForTheAccount(int newAccountId) { executorService.schedule(new Runnable() { public void run() { Context.accountId.set(newAccountId); - ApiCollectionsDao.instance.insertOne(new ApiCollection(0, "Default", Context.now(), new HashSet<>(), null, 0)); + ApiCollectionsDao.instance.insertOne(new ApiCollection(0, "Default", Context.now(), new HashSet<>(), null, 0, false, true)); BackwardCompatibility backwardCompatibility = BackwardCompatibilityDao.instance.findOne(new BasicDBObject()); if (backwardCompatibility == null) { backwardCompatibility = new BackwardCompatibility(); diff --git a/apps/dashboard/src/main/java/com/akto/action/ApiCollectionsAction.java b/apps/dashboard/src/main/java/com/akto/action/ApiCollectionsAction.java index e02b7cb137..2910762089 100644 --- a/apps/dashboard/src/main/java/com/akto/action/ApiCollectionsAction.java +++ b/apps/dashboard/src/main/java/com/akto/action/ApiCollectionsAction.java @@ -114,7 +114,7 @@ public String createCollection() { } // do not change hostName or vxlanId here - ApiCollection apiCollection = new ApiCollection(Context.now(), collectionName,Context.now(),new HashSet<>(), null, 0); + ApiCollection apiCollection = new ApiCollection(Context.now(), collectionName,Context.now(),new HashSet<>(), null, 0, false, true); ApiCollectionsDao.instance.insertOne(apiCollection); this.apiCollections = new ArrayList<>(); this.apiCollections.add(apiCollection); @@ -127,7 +127,7 @@ public String createCollection() { public String deleteCollection() { this.apiCollections = new ArrayList<>(); - this.apiCollections.add(new ApiCollection(apiCollectionId, null, 0, null, null, 0)); + this.apiCollections.add(new ApiCollection(apiCollectionId, null, 0, null, null, 0, false, true)); return this.deleteMultipleCollections(); } diff --git a/apps/dashboard/src/main/java/com/akto/listener/RuntimeListener.java b/apps/dashboard/src/main/java/com/akto/listener/RuntimeListener.java index 80a2ccd8fa..1ee359d42f 100644 --- a/apps/dashboard/src/main/java/com/akto/listener/RuntimeListener.java +++ b/apps/dashboard/src/main/java/com/akto/listener/RuntimeListener.java @@ -150,7 +150,7 @@ public static void addSampleData() { ApiCollection sameNameCollection = ApiCollectionsDao.instance.findByName(VULNERABLE_API_COLLECTION_NAME); if (sameNameCollection == null){ - ApiCollection apiCollection = new ApiCollection(VULNERABLE_API_COLLECTION_ID, VULNERABLE_API_COLLECTION_NAME, Context.now(),new HashSet<>(), null, VULNERABLE_API_COLLECTION_ID); + ApiCollection apiCollection = new ApiCollection(VULNERABLE_API_COLLECTION_ID, VULNERABLE_API_COLLECTION_NAME, Context.now(),new HashSet<>(), null, VULNERABLE_API_COLLECTION_ID, false, true); ApiCollectionsDao.instance.insertOne(apiCollection); } @@ -232,7 +232,7 @@ public static void addLlmSampleData(int accountId) { loggerMaker.infoAndAddToDb("adding llm sample data for account" + accountId, LoggerMaker.LogDb.DASHBOARD); ApiCollection sameNameCollection = ApiCollectionsDao.instance.findByName(LLM_API_COLLECTION_NAME); if (sameNameCollection == null){ - ApiCollection apiCollection = new ApiCollection(LLM_API_COLLECTION_ID, LLM_API_COLLECTION_NAME, Context.now(),new HashSet<>(), null, LLM_API_COLLECTION_ID); + ApiCollection apiCollection = new ApiCollection(LLM_API_COLLECTION_ID, LLM_API_COLLECTION_NAME, Context.now(),new HashSet<>(), null, LLM_API_COLLECTION_ID, false, true); ApiCollectionsDao.instance.insertOne(apiCollection); } diff --git a/apps/dashboard/src/test/java/com/akto/action/TestApiCollectionsAction.java b/apps/dashboard/src/test/java/com/akto/action/TestApiCollectionsAction.java index 2a0bd46dc2..005c13182e 100644 --- a/apps/dashboard/src/test/java/com/akto/action/TestApiCollectionsAction.java +++ b/apps/dashboard/src/test/java/com/akto/action/TestApiCollectionsAction.java @@ -84,15 +84,15 @@ public void fetchAllCollections() { // mirroring collection with host Set urls1 = new HashSet<>(Arrays.asList("1", "2", "3", "4", "5", "6")); - apiCollectionList.add(new ApiCollection(1000, "one", 1000, urls1, "one.com", 1000)); + apiCollectionList.add(new ApiCollection(1000, "one", 1000, urls1, "one.com", 1000, false, true)); // mirroring collections without hosts Set urls2 = new HashSet<>(Arrays.asList("1", "2", "3")); - apiCollectionList.add(new ApiCollection(2000, "two", 2000, urls2, null,2000)); + apiCollectionList.add(new ApiCollection(2000, "two", 2000, urls2, null,2000, false, true)); // manually created collections Set urls3 = new HashSet<>(Arrays.asList("1", "2", "3", "4")); - apiCollectionList.add(new ApiCollection(3000, "three", 3000, urls3, null,0)); + apiCollectionList.add(new ApiCollection(3000, "three", 3000, urls3, null,0, false, true)); ApiCollectionsDao.instance.insertMany(apiCollectionList); diff --git a/apps/dashboard/src/test/java/com/akto/action/observe/TestInventoryAction.java b/apps/dashboard/src/test/java/com/akto/action/observe/TestInventoryAction.java index fcc215d2a9..4febbf1fe5 100644 --- a/apps/dashboard/src/test/java/com/akto/action/observe/TestInventoryAction.java +++ b/apps/dashboard/src/test/java/com/akto/action/observe/TestInventoryAction.java @@ -37,7 +37,7 @@ public void testFetchEndpointsBasedOnHostName() { ApiCollectionsDao.instance.getMCollection().drop(); - ApiCollection apiCollection = new ApiCollection(0, "petstore-lb",0, new HashSet<>(), "petstore.com", 0); + ApiCollection apiCollection = new ApiCollection(0, "petstore-lb",0, new HashSet<>(), "petstore.com", 0, false, true); ApiCollectionsDao.instance.insertOne(apiCollection); SingleTypeInfo sti1 = buildHostSti("url1", "GET", "host", 0); diff --git a/apps/dashboard/src/test/java/com/akto/listener/TestInitializerListener.java b/apps/dashboard/src/test/java/com/akto/listener/TestInitializerListener.java index bed96e9fae..a639aedb0b 100644 --- a/apps/dashboard/src/test/java/com/akto/listener/TestInitializerListener.java +++ b/apps/dashboard/src/test/java/com/akto/listener/TestInitializerListener.java @@ -41,9 +41,9 @@ private static SingleTypeInfo generateSti(String url, int apiCollectionId, boole @Test public void testChangesInfo() { - ApiCollection apiCollection1 = new ApiCollection(0, "coll1", Context.now(), new HashSet<>(), "akto.io", 1); - ApiCollection apiCollection2 = new ApiCollection(1, "coll2", Context.now(), new HashSet<>(), "app.akto.io", 2); - ApiCollection apiCollection3 = new ApiCollection(2, "coll3", Context.now(), new HashSet<>(), null, 3); + ApiCollection apiCollection1 = new ApiCollection(0, "coll1", Context.now(), new HashSet<>(), "akto.io", 1, false, true); + ApiCollection apiCollection2 = new ApiCollection(1, "coll2", Context.now(), new HashSet<>(), "app.akto.io", 2, false, true); + ApiCollection apiCollection3 = new ApiCollection(2, "coll3", Context.now(), new HashSet<>(), null, 3, false, true); ApiCollectionsDao.instance.insertMany(Arrays.asList(apiCollection1, apiCollection2, apiCollection3)); SingleTypeInfo sti1 = generateSti("/api/books", 0, false); diff --git a/libs/dao/src/main/java/com/akto/dto/ApiCollection.java b/libs/dao/src/main/java/com/akto/dto/ApiCollection.java index 351604afa3..dd0ea75e6d 100644 --- a/libs/dao/src/main/java/com/akto/dto/ApiCollection.java +++ b/libs/dao/src/main/java/com/akto/dto/ApiCollection.java @@ -36,17 +36,6 @@ public class ApiCollection { public ApiCollection() { } - public ApiCollection(int id, String name, int startTs, Set urls, String hostName, int vxlanId) { - this.id = id; - this.name = name; - this.startTs = startTs; - this.urls = urls; - this.hostName = hostName; - this.vxlanId = vxlanId; - this.redact = false; - this.sampleCollectionsDropped = true; - } - public ApiCollection(int id, String name, int startTs, Set urls, String hostName, int vxlanId, boolean redact, boolean sampleCollectionsDropped) { this.id = id; this.name = name; @@ -144,7 +133,7 @@ public String getDisplayName() { // To be called if you are creating a collection that is not from mirroring public static ApiCollection createManualCollection(int id, String name){ - return new ApiCollection(id, name, Context.now() , new HashSet<>(), null, 0); + return new ApiCollection(id, name, Context.now() , new HashSet<>(), null, 0, false, true); } public boolean getRedact() { diff --git a/libs/dao/src/test/java/com/akto/dao/TestApiCollectionsDao.java b/libs/dao/src/test/java/com/akto/dao/TestApiCollectionsDao.java index 4bd669342e..4b632e7d21 100644 --- a/libs/dao/src/test/java/com/akto/dao/TestApiCollectionsDao.java +++ b/libs/dao/src/test/java/com/akto/dao/TestApiCollectionsDao.java @@ -18,13 +18,13 @@ public void testFetchNonTrafficApiCollectionsIds() { ApiCollectionsDao.instance.getMCollection().drop(); // mirroring collection with host - ApiCollection apiCollection1 = new ApiCollection(1000, "mirroring_with_host", 1000, new HashSet<>(), "akto.io", 123); + ApiCollection apiCollection1 = new ApiCollection(1000, "mirroring_with_host", 1000, new HashSet<>(), "akto.io", 123, false, true); // mirroring collections without hosts - ApiCollection apiCollection2 = new ApiCollection(2000, "mirroring_without_host", 2000, new HashSet<>(), null, 456); + ApiCollection apiCollection2 = new ApiCollection(2000, "mirroring_without_host", 2000, new HashSet<>(), null, 456, false, true); // manually created collections with vxlanid == 0 - ApiCollection apiCollection3 = new ApiCollection(3000, "burp1", 3000, new HashSet<>(),null, 0); + ApiCollection apiCollection3 = new ApiCollection(3000, "burp1", 3000, new HashSet<>(),null, 0, false, true); // manually created collections with vxlanid != 0 - ApiCollection apiCollection4 = new ApiCollection(4000, "burp2", 4000, new HashSet<>(),null, 4000); + ApiCollection apiCollection4 = new ApiCollection(4000, "burp2", 4000, new HashSet<>(),null, 4000, false, true); ApiCollectionsDao.instance.insertMany(Arrays.asList(apiCollection1, apiCollection2, apiCollection3, apiCollection4)); diff --git a/libs/dao/src/test/java/com/akto/dao/VCDemoData.java b/libs/dao/src/test/java/com/akto/dao/VCDemoData.java index d3a3618582..0fb8c445a9 100644 --- a/libs/dao/src/test/java/com/akto/dao/VCDemoData.java +++ b/libs/dao/src/test/java/com/akto/dao/VCDemoData.java @@ -26,13 +26,13 @@ public static void main(String[] args) { // ApiCollectionsDao.instance.insertOne(new ApiCollection(0, "Default", Context.now(), new HashSet<>(), null,0)); Set urls1 = new HashSet<>(); for (int i=0; i<8; i++) urls1.add(i+""); - ApiCollectionsDao.instance.insertOne(new ApiCollection(1, null, Context.now(), urls1,"api.akto.io",0)); + ApiCollectionsDao.instance.insertOne(new ApiCollection(1, null, Context.now(), urls1,"api.akto.io",0, false, true)); Set urls2 = new HashSet<>(); for (int i=0; i<10; i++) urls2.add(i+""); - ApiCollectionsDao.instance.insertOne(new ApiCollection(2, null, Context.now(), urls2,"staging.akto.io",0)); + ApiCollectionsDao.instance.insertOne(new ApiCollection(2, null, Context.now(), urls2,"staging.akto.io",0, false, true)); Set urls3 = new HashSet<>(); for (int i=0; i<9; i++) urls3.add(i+""); - ApiCollectionsDao.instance.insertOne(new ApiCollection(3, null, Context.now(), urls3,"prod.akto.io",0)); + ApiCollectionsDao.instance.insertOne(new ApiCollection(3, null, Context.now(), urls3,"prod.akto.io",0, false, true)); List singleTypeInfoList = new ArrayList<>(); From 6e50136a4d91c6677be7befea65d56f630947f1a Mon Sep 17 00:00:00 2001 From: aktoboy Date: Wed, 10 Jan 2024 15:32:45 +0530 Subject: [PATCH 18/39] Fixed retro data fix for collection redaction --- .../java/com/akto/action/ApiCollectionsAction.java | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/apps/dashboard/src/main/java/com/akto/action/ApiCollectionsAction.java b/apps/dashboard/src/main/java/com/akto/action/ApiCollectionsAction.java index 2910762089..d2c8fd4c1a 100644 --- a/apps/dashboard/src/main/java/com/akto/action/ApiCollectionsAction.java +++ b/apps/dashboard/src/main/java/com/akto/action/ApiCollectionsAction.java @@ -154,21 +154,21 @@ public String deleteMultipleCollections() { public static void dropSampleDataForApiCollection() { List apiCollections = ApiCollectionsDao.instance.findAll(Filters.eq(ApiCollection.SAMPLE_COLLECTIONS_DROPPED, false)); if(apiCollections.isEmpty()) { - loggerMaker.infoAndAddToDb("No api collections to drop sample data for", LoggerMaker.LogDb.DASHBOARD); + loggerMaker.infoAndAddToDb("No api collections to fix sample data for", LoggerMaker.LogDb.DASHBOARD); return; } - loggerMaker.infoAndAddToDb(String.format("Dropping sample data for %d api collections", apiCollections.size()), LoggerMaker.LogDb.DASHBOARD); + loggerMaker.infoAndAddToDb(String.format("Fixing sample data for %d api collections", apiCollections.size()), LoggerMaker.LogDb.DASHBOARD); for (ApiCollection apiCollection: apiCollections) { int apiCollectionId = apiCollection.getId(); UpdateResult updateResult = SampleDataDao.instance.updateManyNoUpsert(Filters.eq("_id.apiCollectionId", apiCollectionId), Updates.set("samples", Collections.emptyList())); - loggerMaker.infoAndAddToDb(String.format("Dropped %d sample data for api collection %d", updateResult.getModifiedCount(), apiCollectionId), LoggerMaker.LogDb.DASHBOARD); + loggerMaker.infoAndAddToDb(String.format("Fixed %d sample data for api collection %d", updateResult.getModifiedCount(), apiCollectionId), LoggerMaker.LogDb.DASHBOARD); updateResult = SensitiveSampleDataDao.instance.updateManyNoUpsert(Filters.eq("_id.apiCollectionId", apiCollectionId), Updates.set("sampleData", Collections.emptyList())); - loggerMaker.infoAndAddToDb(String.format("Dropped %d sensitive sample data for api collection %d", updateResult.getModifiedCount(), apiCollectionId), LoggerMaker.LogDb.DASHBOARD); - updateResult = SensitiveSampleDataDao.instance.updateManyNoUpsert(Filters.eq("_id.apiCollectionId", apiCollectionId), Updates.set("values.elements", Collections.emptyList())); - loggerMaker.infoAndAddToDb(String.format("Dropped %d sensitive sample data for api collection %d", updateResult.getModifiedCount(), apiCollectionId), LoggerMaker.LogDb.DASHBOARD); + loggerMaker.infoAndAddToDb(String.format("Fixed %d sensitive sample data for api collection %d", updateResult.getModifiedCount(), apiCollectionId), LoggerMaker.LogDb.DASHBOARD); + updateResult = SingleTypeInfoDao.instance.updateManyNoUpsert(Filters.and(Filters.eq("apiCollectionId", apiCollectionId), Filters.exists("values", true)), Updates.set("values.elements", Collections.emptyList())); + loggerMaker.infoAndAddToDb(String.format("Fixed %d sti for api collection %d", updateResult.getModifiedCount(), apiCollectionId), LoggerMaker.LogDb.DASHBOARD); ApiCollectionsDao.instance.updateOneNoUpsert(Filters.eq("_id", apiCollectionId), Updates.set(ApiCollection.SAMPLE_COLLECTIONS_DROPPED, true)); } - loggerMaker.infoAndAddToDb(String.format("Dropped sample data for %d api collections", apiCollections.size()), LoggerMaker.LogDb.DASHBOARD); + loggerMaker.infoAndAddToDb(String.format("Fixed sample data for %d api collections", apiCollections.size()), LoggerMaker.LogDb.DASHBOARD); } public String redactCollection() { From 743e9bd4505a76ba1b76429ec14ddba23c2abbef Mon Sep 17 00:00:00 2001 From: aktoboy Date: Wed, 10 Jan 2024 15:34:56 +0530 Subject: [PATCH 19/39] Fixed ui pr comments --- .../polaris_web/web/src/apps/dashboard/pages/observe/api.js | 3 +-- .../src/apps/dashboard/pages/observe/data_types/transform.js | 1 - 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api.js b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api.js index 1dc7049e87..0941013d75 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api.js +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api.js @@ -223,8 +223,7 @@ export default { url: '/api/redactCollection', method: 'post', data:{ - apiCollectionId: apiCollectionId, - redacted: redacted + apiCollectionId,redacted } }) }, diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/data_types/transform.js b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/data_types/transform.js index 582a76b171..9ab9944de9 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/data_types/transform.js +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/data_types/transform.js @@ -48,7 +48,6 @@ const func = { }, fillInitialState: function({dataObj, type}){ - console.log("name:" + dataObj.name + ",redacted:" +dataObj.redacted) let initialObj = {...func.initialObj}; initialObj.id = dataObj.id; initialObj.name = dataObj.name From dd25edb858f2e175e8cccbfee8d63db9396f4a57 Mon Sep 17 00:00:00 2001 From: aktoboy Date: Wed, 10 Jan 2024 16:16:34 +0530 Subject: [PATCH 20/39] Updated status code --- apps/dashboard/src/main/resources/struts.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/dashboard/src/main/resources/struts.xml b/apps/dashboard/src/main/resources/struts.xml index 7b7a788aa4..36d1de2823 100644 --- a/apps/dashboard/src/main/resources/struts.xml +++ b/apps/dashboard/src/main/resources/struts.xml @@ -600,7 +600,7 @@ - 401 + 422 From 120b9dc6839f5146959be4199525833f7ce98e7d Mon Sep 17 00:00:00 2001 From: aktoboy Date: Wed, 10 Jan 2024 16:20:06 +0530 Subject: [PATCH 21/39] Reverted runtime push --- .github/workflows/staging.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/staging.yml b/.github/workflows/staging.yml index db3c316964..0116ed8b2f 100644 --- a/.github/workflows/staging.yml +++ b/.github/workflows/staging.yml @@ -76,8 +76,6 @@ jobs: docker buildx build --platform linux/arm64/v8,linux/amd64 -t $ECR_REGISTRY/akto-billing:$IMAGE_TAG . --push cd ../internal docker buildx build --platform linux/arm64/v8,linux/amd64 -t $ECR_REGISTRY/akto-internal:$IMAGE_TAG . --push - cd ../api-runtime - docker buildx build --platform linux/arm64/v8,linux/amd64 -t $ECR_REGISTRY/akto-api-security-runtime:$IMAGE_TAG . --push - name: Set up JDK 11 uses: actions/setup-java@v1 From ba405ed37a69e31273d398c51752fd256206df80 Mon Sep 17 00:00:00 2001 From: aktoboy Date: Wed, 10 Jan 2024 16:28:57 +0530 Subject: [PATCH 22/39] Disabled redaction for HAR uploads --- .../src/main/java/com/akto/utils/RedactSampleData.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/api-runtime/src/main/java/com/akto/utils/RedactSampleData.java b/apps/api-runtime/src/main/java/com/akto/utils/RedactSampleData.java index 29a00e3641..00352859bc 100644 --- a/apps/api-runtime/src/main/java/com/akto/utils/RedactSampleData.java +++ b/apps/api-runtime/src/main/java/com/akto/utils/RedactSampleData.java @@ -25,7 +25,7 @@ public class RedactSampleData { public static String redactIfRequired(String sample, boolean accountLevelRedact, boolean apiCollectionLevelRedact) throws Exception { HttpResponseParams httpResponseParams = HttpCallParser.parseKafkaMessage(sample); HttpResponseParams.Source source = httpResponseParams.getSource(); -// if(source.equals(HttpResponseParams.Source.HAR) || source.equals(HttpResponseParams.Source.PCAP)) return sample; + if(source.equals(HttpResponseParams.Source.HAR) || source.equals(HttpResponseParams.Source.PCAP)) return sample; return redact(httpResponseParams, accountLevelRedact || apiCollectionLevelRedact); } From 6df579c40014e48b32a9b813383da9499a8b238d Mon Sep 17 00:00:00 2001 From: aktoboy Date: Wed, 10 Jan 2024 16:42:52 +0530 Subject: [PATCH 23/39] Fixed redactCollection api --- apps/dashboard/src/main/resources/struts.xml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/apps/dashboard/src/main/resources/struts.xml b/apps/dashboard/src/main/resources/struts.xml index 36d1de2823..5ed2450b43 100644 --- a/apps/dashboard/src/main/resources/struts.xml +++ b/apps/dashboard/src/main/resources/struts.xml @@ -599,8 +599,10 @@ - - 422 + + 422 + false + ^actionErrors.* From e494273655be611be1f7c0bdc1332eeb37f74e89 Mon Sep 17 00:00:00 2001 From: Ark2307 Date: Thu, 11 Jan 2024 16:36:32 +0530 Subject: [PATCH 24/39] WIP --- .../java/com/akto/runtime/APICatalogSync.java | 48 ++++++++++++++----- .../java/com/akto/runtime/URLAggregator.java | 21 -------- 2 files changed, 35 insertions(+), 34 deletions(-) diff --git a/apps/api-runtime/src/main/java/com/akto/runtime/APICatalogSync.java b/apps/api-runtime/src/main/java/com/akto/runtime/APICatalogSync.java index 5ede05868e..7e2aa64dd9 100644 --- a/apps/api-runtime/src/main/java/com/akto/runtime/APICatalogSync.java +++ b/apps/api-runtime/src/main/java/com/akto/runtime/APICatalogSync.java @@ -45,7 +45,7 @@ public class APICatalogSync { public Map dbState; public Map delta; public Map sensitiveParamInfoBooleanMap; - public static boolean mergeAsyncOutside = false; + public static boolean mergeAsyncOutside = true; public APICatalogSync(String userIdentifier,int thresh) { this.thresh = thresh; @@ -56,7 +56,7 @@ public APICatalogSync(String userIdentifier,int thresh) { try { String instanceType = System.getenv("AKTO_INSTANCE_TYPE"); if (instanceType != null && "RUNTIME".equalsIgnoreCase(instanceType)) { - mergeAsyncOutside = AccountSettingsDao.instance.findOne(AccountSettingsDao.generateFilter()).getMergeAsyncOutside(); + mergeAsyncOutside = true; } } catch (Exception e) { @@ -564,10 +564,6 @@ public static boolean areBothUuidUrls(URLStatic newUrl, URLStatic deltaUrl, URLT public static boolean areBothMatchingUrls(URLStatic newUrl, URLStatic deltaUrl, URLTemplate mergedTemplate, Boolean urlRegexMatchingEnabled) { - if (!urlRegexMatchingEnabled) { - return false; - } - String[] n = tokenize(newUrl.getUrl()); String[] o = tokenize(deltaUrl.getUrl()); SuperType[] b = mergedTemplate.getTypes(); @@ -575,7 +571,12 @@ public static boolean areBothMatchingUrls(URLStatic newUrl, URLStatic deltaUrl, SuperType c = b[idx]; if (Objects.equals(c, SuperType.STRING) && o.length > idx) { String val = n[idx]; - if(!isAlphanumericString(val) || !isAlphanumericString(o[idx])) { + SubType newSubType = getTokenSubType(n[idx]); + SubType oldSubType = getTokenSubType(o[idx]); + boolean isAlphaNumeric = isAlphanumericString(val) && isAlphanumericString(o[idx]); + boolean isUserRegex = isValidSubtype(oldSubType) && isValidSubtype(newSubType); + + if(!isAlphaNumeric && !isUserRegex) { return false; } } @@ -603,6 +604,19 @@ public static boolean isAlphanumericString(String s) { return (intCount >= 3 && charCount >= 1); } + private static SubType getTokenSubType(Object token){ + for (CustomDataType customDataType: SingleTypeInfo.getCustomDataTypesSortedBySensitivity(Context.accountId.get())) { + if (!customDataType.isActive()) continue; + boolean result = customDataType.validateTokenData(token); + if (result) return customDataType.toSubType(); + } + return KeyTypes.findSubType(token, "", null); + } + + private static Boolean isValidSubtype(SubType subType){ + return !(subType.getName().equals(SingleTypeInfo.GENERIC.getName()) || subType.getName().equals(SingleTypeInfo.OTHER.getName())); + } + public static URLTemplate tryMergeUrls(URLStatic dbUrl, URLStatic newUrl) { if (dbUrl.getMethod() != newUrl.getMethod()) { @@ -623,6 +637,9 @@ public static URLTemplate tryMergeUrls(URLStatic dbUrl, URLStatic newUrl) { String tempToken = newTokens[i]; String dbToken = dbTokens[i]; + SubType dbSubType = getTokenSubType(dbTokens[i]); + SubType tempSubType = getTokenSubType(newTokens[i]); + if (tempToken.equalsIgnoreCase(dbToken)) { continue; } @@ -633,7 +650,12 @@ public static URLTemplate tryMergeUrls(URLStatic dbUrl, URLStatic newUrl) { } else if(pattern.matcher(tempToken).matches() && pattern.matcher(dbToken).matches()){ newTypes[i] = SuperType.STRING; newTokens[i] = null; - } else { + } + else if(isValidSubtype(tempSubType) && isValidSubtype(dbSubType) && (dbSubType.getName().equals(tempSubType.getName()))){ + newTypes[i] = SuperType.STRING; + newTokens[i] = null; + } + else { newTypes[i] = SuperType.STRING; newTokens[i] = null; templatizedStrTokens++; @@ -683,10 +705,10 @@ public static void mergeUrlsAndSave(int apiCollectionId, Boolean urlRegexMatchin for (int i = 0; i < urlTemplate.getTypes().length; i++) { SuperType superType = urlTemplate.getTypes()[i]; if (superType == null) continue; - + String word = delEndpoint.split("/")[i]; SingleTypeInfo.ParamId stiId = new SingleTypeInfo.ParamId(newTemplateUrl, delMethod.name(), -1, false, i+"", SingleTypeInfo.GENERIC, apiCollectionId, true); - SubType subType = KeyTypes.findSubType(i, i+"",stiId); - stiId.setSubType(subType); + SubType tokenSubType = getTokenSubType(word);; + stiId.setSubType(tokenSubType); SingleTypeInfo sti = new SingleTypeInfo( stiId, new HashSet<>(), new HashSet<>(), 0, Context.now(), 0, CappedSet.create(i+""), SingleTypeInfo.Domain.ENUM, SingleTypeInfo.ACCEPTED_MIN_VALUE, SingleTypeInfo.ACCEPTED_MAX_VALUE); @@ -1170,10 +1192,10 @@ public void buildFromDB(boolean calcDiff, boolean fetchAllSTI) { loggerMaker.infoAndAddToDb("Started building from dB", LogDb.RUNTIME); if (mergeAsyncOutside) { - if (Context.now() - lastMergeAsyncOutsideTs > 600) { + if (true) { this.lastMergeAsyncOutsideTs = Context.now(); - boolean gotDibs = Cluster.callDibs(Cluster.RUNTIME_MERGER, 1800, 60); + boolean gotDibs =true; if (gotDibs) { BackwardCompatibility backwardCompatibility = BackwardCompatibilityDao.instance.findOne(new BasicDBObject()); if (backwardCompatibility.getMergeOnHostInit() == 0) { diff --git a/apps/api-runtime/src/main/java/com/akto/runtime/URLAggregator.java b/apps/api-runtime/src/main/java/com/akto/runtime/URLAggregator.java index fbf8dcb4c5..a6dde2c75a 100644 --- a/apps/api-runtime/src/main/java/com/akto/runtime/URLAggregator.java +++ b/apps/api-runtime/src/main/java/com/akto/runtime/URLAggregator.java @@ -48,15 +48,6 @@ private static SingleTypeInfo.SuperType getTokenSupertype(String tempToken, Patt return defaultResult; } - private static SubType getTokenSubType(Object token){ - for (CustomDataType customDataType: SingleTypeInfo.getCustomDataTypesSortedBySensitivity(Context.accountId.get())) { - if (!customDataType.isActive()) continue; - boolean result = customDataType.validateTokenData(token); - if (result) return customDataType.toSubType(); - } - return KeyTypes.findSubType(token, "", null); - } - public static URLStatic getBaseURL(String url, String method) { if (url == null) { @@ -77,18 +68,6 @@ public static URLStatic getBaseURL(String url, String method) { if (tempToken.length() == 0) { continue; } - - SubType subType = getTokenSubType(tempToken); - - if(!(subType == SingleTypeInfo.GENERIC || subType == SingleTypeInfo.OTHER)){ - SingleTypeInfo.SuperType superType = subType.getSuperType();; - if(superType == SingleTypeInfo.SuperType.CUSTOM){ - superType = getTokenSupertype(tempToken,pattern, true); - } - newTypes[i] = superType; - urlTokens[i] = null; - continue; - } if(getTokenSupertype(tempToken, pattern, false) != null){ newTypes[i] = getTokenSupertype(tempToken, pattern, false); From 59b1c897586c88880aad9bf29907c6f8a6075ace Mon Sep 17 00:00:00 2001 From: Ark2307 Date: Fri, 12 Jan 2024 10:29:50 +0530 Subject: [PATCH 25/39] modified apicatalogsync --- .../src/main/java/com/akto/runtime/APICatalogSync.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/api-runtime/src/main/java/com/akto/runtime/APICatalogSync.java b/apps/api-runtime/src/main/java/com/akto/runtime/APICatalogSync.java index 0a494a5dde..e85199733e 100644 --- a/apps/api-runtime/src/main/java/com/akto/runtime/APICatalogSync.java +++ b/apps/api-runtime/src/main/java/com/akto/runtime/APICatalogSync.java @@ -1243,7 +1243,7 @@ public void buildFromDB(boolean calcDiff, boolean fetchAllSTI) { if (Context.now() - lastMergeAsyncOutsideTs > 600) { this.lastMergeAsyncOutsideTs = Context.now(); - boolean gotDibs =true; + boolean gotDibs = Cluster.callDibs(Cluster.RUNTIME_MERGER, 1800, 60); if (gotDibs) { mergingCalled = true; BackwardCompatibility backwardCompatibility = BackwardCompatibilityDao.instance.findOne(new BasicDBObject()); From be92e188a3110014a6e30320be0d8fe254a0529d Mon Sep 17 00:00:00 2001 From: Ark2307 Date: Fri, 12 Jan 2024 14:24:13 +0530 Subject: [PATCH 26/39] Changes made, feature phase 1 completed --- .../pages/observe/SingleRequest/SingleRequest.jsx | 11 +++++++++-- .../dashboard/pages/observe/data_types/DataTypes.jsx | 4 +--- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/SingleRequest/SingleRequest.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/SingleRequest/SingleRequest.jsx index 6406c82275..8de524d83c 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/SingleRequest/SingleRequest.jsx +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/SingleRequest/SingleRequest.jsx @@ -48,8 +48,15 @@ function SingleRequest(){ await api.loadSensitiveParameters(apiCollectionId, url, method, subType).then((res) => { setEndpointData(transform.prepareEndpointData(collectionsMap, res)); }) - await api.fetchSensitiveSampleData(url, apiCollectionId, method).then((res) => { - setSampleData(transform.prepareSampleData(res, subType)) + await api.fetchSensitiveSampleData(url, apiCollectionId, method).then(async(res) => { + if(res.sampleDataList && Object.keys(res.sampleDataList).length > 0){ + setSampleData(transform.prepareSampleData(res, subType)) + }else{ + await api.fetchSampleData(url, apiCollectionId, method).then((resp) => { + const commonMessages = transform.getCommonSamples(resp.sampleDataList[0].samples,resp) + setSampleData(commonMessages) + }) + } }) } fetchData(); diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/data_types/DataTypes.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/data_types/DataTypes.jsx index 7da1a4e488..1efebad085 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/data_types/DataTypes.jsx +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/data_types/DataTypes.jsx @@ -287,9 +287,7 @@ function DataTypes() {
) - - console.log(currState) - + let components = (!isNew && currState.dataType === 'Akto') ? [descriptionCard, requestCard] : [descriptionCard, conditionsCard, urlConditionsCard,requestCard] return ( From 429cdd4a3d978cc46615028496db0a095d4754b6 Mon Sep 17 00:00:00 2001 From: Ark2307 Date: Fri, 12 Jan 2024 15:10:28 +0530 Subject: [PATCH 27/39] Fixed query param bug in editor --- .../components/shared/SampleDataComponent.jsx | 10 +++------- .../components/shared/customDiffEditor.js | 17 ++++------------- .../web/polaris_web/web/src/util/func.js | 11 +++++++---- 3 files changed, 14 insertions(+), 24 deletions(-) diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/shared/SampleDataComponent.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/shared/SampleDataComponent.jsx index ac1386b2d7..de478d3abf 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/shared/SampleDataComponent.jsx +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/shared/SampleDataComponent.jsx @@ -18,11 +18,7 @@ function formatData(data,style){ if(style === "http" && data && Object.keys(data).length > 0){ if(data.json){ Object.keys(data?.json).forEach((element)=> { - if(element.includes("query")){ - if(data.json[element]){ - localFirstLine = localFirstLine + func.convertQueryParamsToUrl(data?.json[element]) - } - }else if(element.includes("Header")){ + if(element.includes("Header")){ if(data.json[element]){ Object.keys(data?.json[element]).forEach((key) => { finalData = finalData + key + ': ' + data.json[element][key] + "\n" @@ -69,8 +65,8 @@ function SampleDataComponent(props) { let originalRequestJson = func.requestJson(originalParsed, sampleData?.highlightPaths) if(isNewDiff){ - let lineReqObj = transform.getFirstLine(originalRequestJson?.firstLine,requestJson?.firstLine,originalRequestJson?.json?.queryParams,requestJson?.json?.queryParams) - let lineResObj = transform.getFirstLine(originalResponseJson?.firstLine,responseJson?.firstLine,originalResponseJson?.json?.queryParams,responseJson?.json?.queryParams) + let lineReqObj = transform.getFirstLine(originalRequestJson?.firstLine,requestJson?.firstLine) + let lineResObj = transform.getFirstLine(originalResponseJson?.firstLine,responseJson?.firstLine) let requestHeaderObj = transform.compareJsonKeys(originalRequestJson?.json?.requestHeaders,requestJson?.json?.requestHeaders) let responseHeaderObj = transform.compareJsonKeys(originalResponseJson?.json?.responseHeaders,responseJson?.json?.responseHeaders) diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/shared/customDiffEditor.js b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/shared/customDiffEditor.js index 9446452410..5438398aed 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/shared/customDiffEditor.js +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/shared/customDiffEditor.js @@ -52,20 +52,11 @@ const transform = { } }, - getFirstLine(original,current,originalParams,currentParams){ - let ogFirstLine = original - let firstLine = current - if(originalParams){ - ogFirstLine = ogFirstLine + func.convertQueryParamsToUrl(originalParams) - } - if(currentParams){ - firstLine = firstLine + func.convertQueryParamsToUrl(currentParams) - } - + getFirstLine(original,current){ return{ - original: ogFirstLine, - firstLine: firstLine, - isUpdated: firstLine !== ogFirstLine + original: original, + firstLine: current, + isUpdated: original !== current } }, diff --git a/apps/dashboard/web/polaris_web/web/src/util/func.js b/apps/dashboard/web/polaris_web/web/src/util/func.js index 5e885e16f6..3f7aa8c8f3 100644 --- a/apps/dashboard/web/polaris_web/web/src/util/func.js +++ b/apps/dashboard/web/polaris_web/web/src/util/func.js @@ -401,7 +401,7 @@ prettifyEpoch(epoch) { result["json"] = { "queryParams": queryParams, "requestHeaders": requestHeaders, "requestPayload": requestPayload } result["highlightPaths"] = {} - result['firstLine'] = func.requestFirstLine(message) + result['firstLine'] = func.requestFirstLine(message, queryParams) for (const x of highlightPaths) { if (x["responseCode"] === -1) { let keys = [] @@ -466,12 +466,12 @@ prettifyEpoch(epoch) { } return result }, - requestFirstLine(message) { + requestFirstLine(message, queryParams) { if (message["request"]) { let url = message["request"]["url"] - return message["request"]["method"] + " " + url + " " + message["request"]["type"] + return message["request"]["method"] + " " + url + func.convertQueryParamsToUrl(queryParams) + " " + message["request"]["type"] } else { - return message.method + " " + message.path.split("?")[0] + " " + message.type + return message.method + " " + message.path.split("?")[0] + func.convertQueryParamsToUrl(queryParams) + " " + message.type } }, responseFirstLine(message) { @@ -1244,6 +1244,9 @@ mapCollectionIdToHostName(apiCollections){ return count === 1 ? "" : "s" }, convertQueryParamsToUrl(queryParams) { + if(!queryParams){ + return ""; + } let url = "" let first = true; let joiner = "?" From ebba0cbb5b4f5d2096bfcbf9749bc3d203ff78af Mon Sep 17 00:00:00 2001 From: Ark2307 Date: Tue, 6 Feb 2024 10:54:33 +0530 Subject: [PATCH 28/39] changes made --- .../java/com/akto/runtime/APICatalogSync.java | 10 +++++-- .../com/akto/action/CustomDataTypeAction.java | 27 +------------------ .../pages/observe/data_types/DataTypes.jsx | 23 +++------------- .../pages/observe/data_types/transform.js | 7 ----- .../java/com/akto/dto/CustomDataType.java | 23 ---------------- 5 files changed, 12 insertions(+), 78 deletions(-) diff --git a/apps/api-runtime/src/main/java/com/akto/runtime/APICatalogSync.java b/apps/api-runtime/src/main/java/com/akto/runtime/APICatalogSync.java index e85199733e..acf2ddd04e 100644 --- a/apps/api-runtime/src/main/java/com/akto/runtime/APICatalogSync.java +++ b/apps/api-runtime/src/main/java/com/akto/runtime/APICatalogSync.java @@ -4,6 +4,7 @@ import com.akto.dao.context.Context; import com.akto.dto.*; import com.akto.dto.HttpResponseParams.Source; +import com.akto.dto.data_types.Conditions; import com.akto.dto.traffic.Key; import com.akto.dto.traffic.SampleData; import com.akto.dto.traffic.TrafficInfo; @@ -617,7 +618,12 @@ public static boolean isAlphanumericString(String s) { private static SubType getTokenSubType(Object token){ for (CustomDataType customDataType: SingleTypeInfo.getCustomDataTypesSortedBySensitivity(Context.accountId.get())) { if (!customDataType.isActive()) continue; - boolean result = customDataType.validateTokenData(token); + boolean result = false; + Conditions valueConditions = customDataType.getValueConditions(); + + if(valueConditions !=null){ + result = valueConditions.validate(token); + } if (result) return customDataType.toSubType(); } return KeyTypes.findSubType(token, "", null); @@ -1244,7 +1250,7 @@ public void buildFromDB(boolean calcDiff, boolean fetchAllSTI) { this.lastMergeAsyncOutsideTs = Context.now(); boolean gotDibs = Cluster.callDibs(Cluster.RUNTIME_MERGER, 1800, 60); - if (gotDibs) { + if (gotDibs) { mergingCalled = true; BackwardCompatibility backwardCompatibility = BackwardCompatibilityDao.instance.findOne(new BasicDBObject()); if (backwardCompatibility == null || backwardCompatibility.getMergeOnHostInit() == 0) { diff --git a/apps/dashboard/src/main/java/com/akto/action/CustomDataTypeAction.java b/apps/dashboard/src/main/java/com/akto/action/CustomDataTypeAction.java index 20abb1e501..fe3927fb16 100644 --- a/apps/dashboard/src/main/java/com/akto/action/CustomDataTypeAction.java +++ b/apps/dashboard/src/main/java/com/akto/action/CustomDataTypeAction.java @@ -46,8 +46,6 @@ public class CustomDataTypeAction extends UserAction{ private String valueOperator; private List valueConditionFromUsers; - private String tokenOperator; - private List tokenConditionFromUsers; public static class ConditionFromUser { Predicate.Type type; Map valueMap; @@ -488,14 +486,8 @@ public CustomDataType generateCustomDataType(int userId) throws AktoCustomExcept throw new AktoCustomException(e.getMessage()); } - Conditions tokenConditions = null; - try { - tokenConditions = getValidConditionsFromUserConditions(tokenConditionFromUsers, tokenOperator, "urlToken"); - } catch (Exception e) { - throw new AktoCustomException(e.getMessage()); - } - if (isInvalidCondition(keyConditions) && isInvalidCondition(valueConditions) && isInvalidCondition(tokenConditions)) { + if (isInvalidCondition(keyConditions) && isInvalidCondition(valueConditions)){ throw new AktoCustomException("All key, value and token conditions can't be empty"); } @@ -517,7 +509,6 @@ public CustomDataType generateCustomDataType(int userId) throws AktoCustomExcept IgnoreData ignoreData = new IgnoreData(); CustomDataType customDataType = new CustomDataType(name, sensitiveAlways, sensitivePositions, userId, true,keyConditions,valueConditions, mainOperator,ignoreData); - customDataType.setTokenConditions(tokenConditions); return customDataType; } @@ -743,20 +734,4 @@ public List getSensitivePosition() { public void setSensitivePosition(List sensitivePosition) { this.sensitivePosition = sensitivePosition; } - - public List getTokenConditionFromUsers() { - return tokenConditionFromUsers; - } - - public void setTokenConditionFromUsers(List tokenConditionFromUsers) { - this.tokenConditionFromUsers = tokenConditionFromUsers; - } - - public String getTokenOperator() { - return tokenOperator; - } - - public void setTokenOperator(String tokenOperator) { - this.tokenOperator = tokenOperator; - } } diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/data_types/DataTypes.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/data_types/DataTypes.jsx index 1efebad085..207b04b8aa 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/data_types/DataTypes.jsx +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/data_types/DataTypes.jsx @@ -195,7 +195,8 @@ function DataTypes() { { pageTitle === "Add data type" ? handleChange({ name: val }) : {} }} /> + {...pageTitle === "Add data type" ? {onChange: (val) => handleChange({name: val})} : {}} + /> {currState.dataType === 'Custom' ? { handleChange({ active: val }) }} @@ -248,24 +249,6 @@ function DataTypes() { ) - const urlConditionsCard = ( - - URL token conditions - } - > - { handleDispatch(val, "tokenConditions") }} - /> - - ) - const compareFunc = () => { return !change } @@ -288,7 +271,7 @@ function DataTypes() { ) - let components = (!isNew && currState.dataType === 'Akto') ? [descriptionCard, requestCard] : [descriptionCard, conditionsCard, urlConditionsCard,requestCard] + let components = (!isNew && currState.dataType === 'Akto') ? [descriptionCard, requestCard] : [descriptionCard, conditionsCard,requestCard] return ( Date: Tue, 6 Feb 2024 15:28:18 +0530 Subject: [PATCH 29/39] removed reduntant code which was used earlier, stabilising branch --- .../java/com/akto/runtime/APICatalogSync.java | 2 +- .../java/com/akto/runtime/URLAggregator.java | 36 ++++---- .../com/akto/action/CustomDataTypeAction.java | 87 ++++++++++--------- .../java/com/akto/dto/CustomDataType.java | 2 +- .../main/java/com/akto/dto/type/KeyTypes.java | 6 +- 5 files changed, 65 insertions(+), 68 deletions(-) diff --git a/apps/api-runtime/src/main/java/com/akto/runtime/APICatalogSync.java b/apps/api-runtime/src/main/java/com/akto/runtime/APICatalogSync.java index acf2ddd04e..b4d71bc116 100644 --- a/apps/api-runtime/src/main/java/com/akto/runtime/APICatalogSync.java +++ b/apps/api-runtime/src/main/java/com/akto/runtime/APICatalogSync.java @@ -626,7 +626,7 @@ private static SubType getTokenSubType(Object token){ } if (result) return customDataType.toSubType(); } - return KeyTypes.findSubType(token, "", null); + return KeyTypes.findSubType(token, "", null,true); } private static Boolean isValidSubtype(SubType subType){ diff --git a/apps/api-runtime/src/main/java/com/akto/runtime/URLAggregator.java b/apps/api-runtime/src/main/java/com/akto/runtime/URLAggregator.java index a6dde2c75a..34ad52101b 100644 --- a/apps/api-runtime/src/main/java/com/akto/runtime/URLAggregator.java +++ b/apps/api-runtime/src/main/java/com/akto/runtime/URLAggregator.java @@ -30,30 +30,13 @@ public class URLAggregator { ConcurrentMap> urls; - private static SingleTypeInfo.SuperType getTokenSupertype(String tempToken, Pattern pattern, Boolean isCustom){ - String numToken = tempToken; - SingleTypeInfo.SuperType defaultResult = isCustom ? SingleTypeInfo.SuperType.STRING : null; - if (tempToken.charAt(0) == '+') { - numToken = tempToken.substring(1); - } - - if (NumberUtils.isParsable(numToken)) { - return SingleTypeInfo.SuperType.INTEGER; - - } else if(pattern.matcher(tempToken).matches()){ - return SingleTypeInfo.SuperType.STRING; - } else if (isAlphanumericString(tempToken)) { - return SingleTypeInfo.SuperType.STRING; - } - return defaultResult; - } - public static URLStatic getBaseURL(String url, String method) { if (url == null) { return null; } + String urlStr = url.split("\\?")[0]; String[] urlTokens = tokenize(urlStr); @@ -68,9 +51,20 @@ public static URLStatic getBaseURL(String url, String method) { if (tempToken.length() == 0) { continue; } - - if(getTokenSupertype(tempToken, pattern, false) != null){ - newTypes[i] = getTokenSupertype(tempToken, pattern, false); + + String numToken = tempToken; + if (tempToken.charAt(0) == '+') { + numToken = tempToken.substring(1); + } + + if (NumberUtils.isParsable(numToken)) { + newTypes[i] = SingleTypeInfo.SuperType.INTEGER; + urlTokens[i] = null; + } else if(pattern.matcher(tempToken).matches()){ + newTypes[i] = SingleTypeInfo.SuperType.STRING; + urlTokens[i] = null; + } else if (isAlphanumericString(tempToken)) { + newTypes[i] = SingleTypeInfo.SuperType.STRING; urlTokens[i] = null; } } diff --git a/apps/dashboard/src/main/java/com/akto/action/CustomDataTypeAction.java b/apps/dashboard/src/main/java/com/akto/action/CustomDataTypeAction.java index fe3927fb16..604f495fb5 100644 --- a/apps/dashboard/src/main/java/com/akto/action/CustomDataTypeAction.java +++ b/apps/dashboard/src/main/java/com/akto/action/CustomDataTypeAction.java @@ -428,68 +428,72 @@ public boolean forPayload(String payload, CustomDataType customDataType, Key api } - private Conditions getValidConditionsFromUserConditions(List conditionFromUsers, String operator, String key) throws AktoCustomException { - if (conditionFromUsers != null && operator != null) { + public CustomDataType generateCustomDataType(int userId) throws AktoCustomException { + if (name == null || name.length() == 0) throw new AktoCustomException("Name cannot be empty"); + int maxChars = 25; + if (name.length() > maxChars) throw new AktoCustomException("Maximum length allowed is "+maxChars+" characters"); + name = name.trim(); + name = name.toUpperCase(); + if (!(name.matches("[A-Z_0-9 ]+"))) throw new AktoCustomException("Name can only contain alphabets, spaces, numbers and underscores"); + + if (SingleTypeInfo.subTypeMap.containsKey(name)) { + throw new AktoCustomException("Data type name reserved"); + } + + + Conditions keyConditions = null; + if (keyConditionFromUsers != null && keyOperator != null) { Conditions.Operator kOperator; try { - kOperator = Conditions.Operator.valueOf(operator); + kOperator = Conditions.Operator.valueOf(keyOperator); } catch (Exception ignored) { - throw new AktoCustomException("Invalid " + key + " operator"); + throw new AktoCustomException("Invalid key operator"); } List predicates = new ArrayList<>(); - for (ConditionFromUser conditionFromUser: conditionFromUsers) { + for (ConditionFromUser conditionFromUser: keyConditionFromUsers) { Predicate predicate = Predicate.generatePredicate(conditionFromUser.type, conditionFromUser.valueMap); if (predicate == null) { - throw new AktoCustomException("Invalid " + key + " conditions"); + throw new AktoCustomException("Invalid key conditions"); } else { predicates.add(predicate); } } if (predicates.size() > 0) { - return new Conditions(predicates, kOperator); + keyConditions = new Conditions(predicates, kOperator); } } - return null; - } - - private static Boolean isInvalidCondition(Conditions conditions){ - return (conditions == null || conditions.getPredicates() == null || conditions.getPredicates().size() == 0); - } - - public CustomDataType generateCustomDataType(int userId) throws AktoCustomException { - if (name == null || name.length() == 0) throw new AktoCustomException("Name cannot be empty"); - int maxChars = 25; - if (name.length() > maxChars) throw new AktoCustomException("Maximum length allowed is "+maxChars+" characters"); - name = name.trim(); - name = name.toUpperCase(); - if (!(name.matches("[A-Z_0-9 ]+"))) throw new AktoCustomException("Name can only contain alphabets, spaces, numbers and underscores"); - - if (SingleTypeInfo.subTypeMap.containsKey(name)) { - throw new AktoCustomException("Data type name reserved"); - } + Conditions valueConditions = null; + if (valueConditionFromUsers != null && valueOperator != null) { + Conditions.Operator vOperator; + try { + vOperator = Conditions.Operator.valueOf(valueOperator); + } catch (Exception ignored) { + throw new AktoCustomException("Invalid value operator"); + } - Conditions keyConditions = null; - try { - keyConditions = getValidConditionsFromUserConditions(keyConditionFromUsers, keyOperator , "key"); - } catch (Exception e) { - throw new AktoCustomException(e.getMessage()); - } + List predicates = new ArrayList<>(); + for (ConditionFromUser conditionFromUser: valueConditionFromUsers) { + Predicate predicate = Predicate.generatePredicate(conditionFromUser.type, conditionFromUser.valueMap); + if (predicate == null) { + throw new AktoCustomException("Invalid value conditions"); + } else { + predicates.add(predicate); + } + } - Conditions valueConditions = null; - try { - valueConditions = getValidConditionsFromUserConditions(valueConditionFromUsers, valueOperator, "value"); - } catch (Exception e) { - throw new AktoCustomException(e.getMessage()); + if (predicates.size() > 0) { + valueConditions = new Conditions(predicates, vOperator); + } } + if ((keyConditions == null || keyConditions.getPredicates() == null || keyConditions.getPredicates().size() == 0) && + (valueConditions == null || valueConditions.getPredicates() ==null || valueConditions.getPredicates().size() == 0)) { - if (isInvalidCondition(keyConditions) && isInvalidCondition(valueConditions)){ - - throw new AktoCustomException("All key, value and token conditions can't be empty"); + throw new AktoCustomException("Both key and value conditions can't be empty"); } Conditions.Operator mainOperator; @@ -507,9 +511,8 @@ public CustomDataType generateCustomDataType(int userId) throws AktoCustomExcept } IgnoreData ignoreData = new IgnoreData(); - CustomDataType customDataType = new CustomDataType(name, sensitiveAlways, sensitivePositions, userId, + return new CustomDataType(name, sensitiveAlways, sensitivePositions, userId, true,keyConditions,valueConditions, mainOperator,ignoreData); - return customDataType; } public void setCreateNew(boolean createNew) { @@ -734,4 +737,4 @@ public List getSensitivePosition() { public void setSensitivePosition(List sensitivePosition) { this.sensitivePosition = sensitivePosition; } -} +} \ No newline at end of file diff --git a/libs/dao/src/main/java/com/akto/dto/CustomDataType.java b/libs/dao/src/main/java/com/akto/dto/CustomDataType.java index 2225d9b761..c7290284ac 100644 --- a/libs/dao/src/main/java/com/akto/dto/CustomDataType.java +++ b/libs/dao/src/main/java/com/akto/dto/CustomDataType.java @@ -88,7 +88,7 @@ public boolean validateRaw(Object value, Object key) throws Exception { } } - + public ObjectId getId() { return id; } diff --git a/libs/dao/src/main/java/com/akto/dto/type/KeyTypes.java b/libs/dao/src/main/java/com/akto/dto/type/KeyTypes.java index 6ad021b9c4..250b8b54d4 100644 --- a/libs/dao/src/main/java/com/akto/dto/type/KeyTypes.java +++ b/libs/dao/src/main/java/com/akto/dto/type/KeyTypes.java @@ -186,11 +186,11 @@ private static SubType getSubtype(Object o,String key, boolean checkForSubtypes) return SingleTypeInfo.OTHER; } - public static SubType findSubType(Object o,String key, ParamId paramId, Boolean executeCheckForSubtypes){ + public static SubType findSubType(Object o,String key, ParamId paramId, boolean executeCheckForSubtypes){ if(executeCheckForSubtypes){ - return findSubType(o, key, paramId); - }else{ return getSubtype(o, key, true); + }else{ + return findSubType(o, key, paramId); } } From 737d5c7808481d2e8e06a4212f9769a8d1c331c0 Mon Sep 17 00:00:00 2001 From: Ark2307 Date: Wed, 7 Feb 2024 09:44:04 +0530 Subject: [PATCH 30/39] fixing comments on PR for ApicatalogSync --- .../java/com/akto/runtime/APICatalogSync.java | 29 +++++-------------- 1 file changed, 8 insertions(+), 21 deletions(-) diff --git a/apps/api-runtime/src/main/java/com/akto/runtime/APICatalogSync.java b/apps/api-runtime/src/main/java/com/akto/runtime/APICatalogSync.java index b4d71bc116..75afad71e7 100644 --- a/apps/api-runtime/src/main/java/com/akto/runtime/APICatalogSync.java +++ b/apps/api-runtime/src/main/java/com/akto/runtime/APICatalogSync.java @@ -573,7 +573,7 @@ public static boolean areBothUuidUrls(URLStatic newUrl, URLStatic deltaUrl, URLT return true; } - public static boolean areBothMatchingUrls(URLStatic newUrl, URLStatic deltaUrl, URLTemplate mergedTemplate, Boolean urlRegexMatchingEnabled) { + public static boolean areBothMatchingUrls(URLStatic newUrl, URLStatic deltaUrl, URLTemplate mergedTemplate, boolean urlRegexMatchingEnabled) { String[] n = tokenize(newUrl.getUrl()); String[] o = tokenize(deltaUrl.getUrl()); @@ -582,10 +582,10 @@ public static boolean areBothMatchingUrls(URLStatic newUrl, URLStatic deltaUrl, SuperType c = b[idx]; if (Objects.equals(c, SuperType.STRING) && o.length > idx) { String val = n[idx]; - SubType newSubType = getTokenSubType(n[idx]); - SubType oldSubType = getTokenSubType(o[idx]); + SubType newSubType = KeyTypes.findSubType(n[idx], "", null,true); + SubType oldSubType = KeyTypes.findSubType(o[idx], "", null,true); boolean isAlphaNumeric = isAlphanumericString(val) && isAlphanumericString(o[idx]); - boolean isUserRegex = isValidSubtype(oldSubType) && isValidSubtype(newSubType); + boolean isUserRegex = isValidSubtype(oldSubType) && isValidSubtype(newSubType) && oldSubType.equals(newSubType); if(!isAlphaNumeric && !isUserRegex) { return false; @@ -615,21 +615,8 @@ public static boolean isAlphanumericString(String s) { return (intCount >= 3 && charCount >= 1); } - private static SubType getTokenSubType(Object token){ - for (CustomDataType customDataType: SingleTypeInfo.getCustomDataTypesSortedBySensitivity(Context.accountId.get())) { - if (!customDataType.isActive()) continue; - boolean result = false; - Conditions valueConditions = customDataType.getValueConditions(); - - if(valueConditions !=null){ - result = valueConditions.validate(token); - } - if (result) return customDataType.toSubType(); - } - return KeyTypes.findSubType(token, "", null,true); - } - private static Boolean isValidSubtype(SubType subType){ + private static boolean isValidSubtype(SubType subType){ return !(subType.getName().equals(SingleTypeInfo.GENERIC.getName()) || subType.getName().equals(SingleTypeInfo.OTHER.getName())); } @@ -653,8 +640,8 @@ public static URLTemplate tryMergeUrls(URLStatic dbUrl, URLStatic newUrl) { String tempToken = newTokens[i]; String dbToken = dbTokens[i]; - SubType dbSubType = getTokenSubType(dbTokens[i]); - SubType tempSubType = getTokenSubType(newTokens[i]); + SubType dbSubType = KeyTypes.findSubType(dbTokens[i], "", null,true); + SubType tempSubType = KeyTypes.findSubType(newTokens[i], "", null,true); if (tempToken.equalsIgnoreCase(dbToken)) { continue; @@ -723,7 +710,7 @@ public static void mergeUrlsAndSave(int apiCollectionId, Boolean urlRegexMatchin if (superType == null) continue; String word = delEndpoint.split("/")[i]; SingleTypeInfo.ParamId stiId = new SingleTypeInfo.ParamId(newTemplateUrl, delMethod.name(), -1, false, i+"", SingleTypeInfo.GENERIC, apiCollectionId, true); - SubType tokenSubType = getTokenSubType(word);; + SubType tokenSubType = KeyTypes.findSubType(word, "", null,true); stiId.setSubType(tokenSubType); SingleTypeInfo sti = new SingleTypeInfo( stiId, new HashSet<>(), new HashSet<>(), 0, Context.now(), 0, CappedSet.create(i+""), From d99d9fd7b3cab35a63b3a81340650c0325a9dc6c Mon Sep 17 00:00:00 2001 From: Ark2307 Date: Wed, 7 Feb 2024 09:50:47 +0530 Subject: [PATCH 31/39] removing unused import --- .../src/main/java/com/akto/runtime/APICatalogSync.java | 1 - 1 file changed, 1 deletion(-) diff --git a/apps/api-runtime/src/main/java/com/akto/runtime/APICatalogSync.java b/apps/api-runtime/src/main/java/com/akto/runtime/APICatalogSync.java index 0c6bae2987..9370881100 100644 --- a/apps/api-runtime/src/main/java/com/akto/runtime/APICatalogSync.java +++ b/apps/api-runtime/src/main/java/com/akto/runtime/APICatalogSync.java @@ -4,7 +4,6 @@ import com.akto.dao.context.Context; import com.akto.dto.*; import com.akto.dto.HttpResponseParams.Source; -import com.akto.dto.data_types.Conditions; import com.akto.dto.traffic.Key; import com.akto.dto.traffic.SampleData; import com.akto.dto.traffic.TrafficInfo; From 5c1def2450f5961afef70860ed4a73f0a70da9da Mon Sep 17 00:00:00 2001 From: Ark2307 Date: Wed, 7 Feb 2024 11:57:08 +0530 Subject: [PATCH 32/39] fixed url aggregator and finding subtypes method --- .../java/com/akto/runtime/APICatalogSync.java | 8 ++-- .../java/com/akto/runtime/URLAggregator.java | 38 +------------------ 2 files changed, 5 insertions(+), 41 deletions(-) diff --git a/apps/api-runtime/src/main/java/com/akto/runtime/APICatalogSync.java b/apps/api-runtime/src/main/java/com/akto/runtime/APICatalogSync.java index 9370881100..b6dca9896e 100644 --- a/apps/api-runtime/src/main/java/com/akto/runtime/APICatalogSync.java +++ b/apps/api-runtime/src/main/java/com/akto/runtime/APICatalogSync.java @@ -584,8 +584,8 @@ public static boolean areBothMatchingUrls(URLStatic newUrl, URLStatic deltaUrl, SubType newSubType = KeyTypes.findSubType(n[idx], "", null,true); SubType oldSubType = KeyTypes.findSubType(o[idx], "", null,true); boolean isAlphaNumeric = isAlphanumericString(val) && isAlphanumericString(o[idx]); - boolean isUserRegex = isValidSubtype(oldSubType) && isValidSubtype(newSubType) && oldSubType.equals(newSubType); - + boolean isUserRegex = isValidSubtype(oldSubType) && isValidSubtype(newSubType) && oldSubType.getName().equals(newSubType.getName()); + if(!isAlphaNumeric && !isUserRegex) { return false; } @@ -641,7 +641,6 @@ public static URLTemplate tryMergeUrls(URLStatic dbUrl, URLStatic newUrl) { SubType dbSubType = KeyTypes.findSubType(dbTokens[i], "", null,true); SubType tempSubType = KeyTypes.findSubType(newTokens[i], "", null,true); - int minCount = dbUrl.getUrl().startsWith("http") && newUrl.getUrl().startsWith("http") ? 3 : 0; if (tempToken.equalsIgnoreCase(dbToken) || i < minCount) { @@ -719,7 +718,8 @@ public static void mergeUrlsAndSave(int apiCollectionId, Boolean urlRegexMatchin for (int i = 0; i < urlTemplate.getTypes().length; i++) { SuperType superType = urlTemplate.getTypes()[i]; if (superType == null) continue; - String word = delEndpoint.split("/")[i]; + int idx = delEndpoint.startsWith("http") ? i:i+1; + String word = delEndpoint.split("/")[idx]; SingleTypeInfo.ParamId stiId = new SingleTypeInfo.ParamId(newTemplateUrl, delMethod.name(), -1, false, i+"", SingleTypeInfo.GENERIC, apiCollectionId, true); SubType tokenSubType = KeyTypes.findSubType(word, "", null,true); stiId.setSubType(tokenSubType); diff --git a/apps/api-runtime/src/main/java/com/akto/runtime/URLAggregator.java b/apps/api-runtime/src/main/java/com/akto/runtime/URLAggregator.java index 34ad52101b..1e106cbf67 100644 --- a/apps/api-runtime/src/main/java/com/akto/runtime/URLAggregator.java +++ b/apps/api-runtime/src/main/java/com/akto/runtime/URLAggregator.java @@ -36,43 +36,7 @@ public static URLStatic getBaseURL(String url, String method) { return null; } - - String urlStr = url.split("\\?")[0]; - - String[] urlTokens = tokenize(urlStr); - - Pattern pattern = patternToSubType.get(SingleTypeInfo.UUID); - - SingleTypeInfo.SuperType[] newTypes = new SingleTypeInfo.SuperType[urlTokens.length]; - - for(int i = 0; i < urlTokens.length; i ++) { - String tempToken = urlTokens[i]; - - if (tempToken.length() == 0) { - continue; - } - - String numToken = tempToken; - if (tempToken.charAt(0) == '+') { - numToken = tempToken.substring(1); - } - - if (NumberUtils.isParsable(numToken)) { - newTypes[i] = SingleTypeInfo.SuperType.INTEGER; - urlTokens[i] = null; - } else if(pattern.matcher(tempToken).matches()){ - newTypes[i] = SingleTypeInfo.SuperType.STRING; - urlTokens[i] = null; - } else if (isAlphanumericString(tempToken)) { - newTypes[i] = SingleTypeInfo.SuperType.STRING; - urlTokens[i] = null; - } - } - - Method m = Method.valueOf(method); - URLStatic ret = new URLStatic(new URLTemplate(urlTokens, newTypes, m).getTemplateString(), m); - - return ret; + return new URLStatic(url.split("\\?")[0], Method.fromString(method)); } From 5b3f38be4fac9d327e2564480e66dfbdea2fc3e1 Mon Sep 17 00:00:00 2001 From: Ark2307 Date: Thu, 8 Feb 2024 12:41:44 +0530 Subject: [PATCH 33/39] updated merging logic for url --- .../java/com/akto/runtime/APICatalogSync.java | 143 +++++++++++++----- .../java/com/akto/parsers/TestMergingNew.java | 28 ++-- .../apps/dashboard/pages/observe/transform.js | 1 + .../com/akto/dto/type/SingleTypeInfo.java | 2 +- 4 files changed, 118 insertions(+), 56 deletions(-) diff --git a/apps/api-runtime/src/main/java/com/akto/runtime/APICatalogSync.java b/apps/api-runtime/src/main/java/com/akto/runtime/APICatalogSync.java index b6dca9896e..2941a6747c 100644 --- a/apps/api-runtime/src/main/java/com/akto/runtime/APICatalogSync.java +++ b/apps/api-runtime/src/main/java/com/akto/runtime/APICatalogSync.java @@ -27,6 +27,7 @@ import org.apache.commons.lang3.math.NumberUtils; import org.bson.conversions.Bson; import org.bson.json.JsonParseException; +import org.bson.types.ObjectId; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -205,12 +206,30 @@ public void computeDelta(URLAggregator origAggregator, boolean triggerTemplateGe } else { for (URLStatic pending: pendingRequests.keySet()) { RequestTemplate pendingTemplate = pendingRequests.get(pending); - RequestTemplate rt = deltaCatalog.getStrictURLToMethods().get(pending); - if (rt != null) { - rt.mergeFrom(pendingTemplate); - } else { - deltaCatalog.getStrictURLToMethods().put(pending, pendingTemplate); + + URLTemplate parameterisedTemplate = tryParamteresingUrl(pending); + + if(parameterisedTemplate != null){ + RequestTemplate rt = deltaCatalog.getTemplateURLToMethods().get(parameterisedTemplate); + if (rt != null) { + rt.mergeFrom(pendingTemplate); + } else { + deltaCatalog.getTemplateURLToMethods().put(parameterisedTemplate, pendingTemplate); + } + + rt = deltaCatalog.getTemplateURLToMethods().get(parameterisedTemplate); + rt.fillUrlParams(tokenize(pending.getUrl()), parameterisedTemplate, apiCollectionId); + + }else{ + RequestTemplate rt = deltaCatalog.getStrictURLToMethods().get(pending); + if (rt != null) { + rt.mergeFrom(pendingTemplate); + } else { + deltaCatalog.getStrictURLToMethods().put(pending, pendingTemplate); + } } + + } } } @@ -581,12 +600,9 @@ public static boolean areBothMatchingUrls(URLStatic newUrl, URLStatic deltaUrl, SuperType c = b[idx]; if (Objects.equals(c, SuperType.STRING) && o.length > idx) { String val = n[idx]; - SubType newSubType = KeyTypes.findSubType(n[idx], "", null,true); - SubType oldSubType = KeyTypes.findSubType(o[idx], "", null,true); boolean isAlphaNumeric = isAlphanumericString(val) && isAlphanumericString(o[idx]); - boolean isUserRegex = isValidSubtype(oldSubType) && isValidSubtype(newSubType) && oldSubType.getName().equals(newSubType.getName()); - - if(!isAlphaNumeric && !isUserRegex) { + + if(!isAlphaNumeric) { return false; } } @@ -619,6 +635,48 @@ private static boolean isValidSubtype(SubType subType){ return !(subType.getName().equals(SingleTypeInfo.GENERIC.getName()) || subType.getName().equals(SingleTypeInfo.OTHER.getName())); } + public static URLTemplate tryParamteresingUrl(URLStatic newUrl){ + String[] tokens = tokenize(newUrl.getUrl()); + if(tokens.length < 2){ + return null; + } + Pattern pattern = patternToSubType.get(SingleTypeInfo.UUID); + boolean allNull = true; + SuperType[] newTypes = new SuperType[tokens.length]; + for(int i = 0; i < tokens.length; i ++) { + String tempToken = tokens[i]; + + if (NumberUtils.isParsable(tempToken)) { + newTypes[i] = SuperType.INTEGER; + tokens[i] = null; + } else if(ObjectId.isValid(tempToken)){ + newTypes[i] = SuperType.OBJECT_ID; + tokens[i] = null; + }else if(pattern.matcher(tempToken).matches()){ + newTypes[i] = SuperType.STRING; + tokens[i] = null; + } + + if(tokens[i] != null){ + SubType tempSubType = KeyTypes.findSubType(tokens[i], ""+i, null,true); + if(isValidSubtype(tempSubType)){ + newTypes[i] = SuperType.STRING; + tokens[i] = null; + }else if(isAlphanumericString(tempToken)){ + newTypes[i] = SuperType.STRING; + tokens[i] = null; + } + } + + if(newTypes[i] != null){ + allNull = false; + } + } + + if (allNull) return null; + return new URLTemplate(tokens, newTypes, newUrl.getMethod()); + } + public static URLTemplate tryMergeUrls(URLStatic dbUrl, URLStatic newUrl) { if (dbUrl.getMethod() != newUrl.getMethod()) { @@ -639,9 +697,6 @@ public static URLTemplate tryMergeUrls(URLStatic dbUrl, URLStatic newUrl) { String tempToken = newTokens[i]; String dbToken = dbTokens[i]; - SubType dbSubType = KeyTypes.findSubType(dbTokens[i], "", null,true); - SubType tempSubType = KeyTypes.findSubType(newTokens[i], "", null,true); - int minCount = dbUrl.getUrl().startsWith("http") && newUrl.getUrl().startsWith("http") ? 3 : 0; if (tempToken.equalsIgnoreCase(dbToken) || i < minCount) { continue; @@ -653,11 +708,7 @@ public static URLTemplate tryMergeUrls(URLStatic dbUrl, URLStatic newUrl) { } else if(pattern.matcher(tempToken).matches() && pattern.matcher(dbToken).matches()){ newTypes[i] = SuperType.STRING; newTokens[i] = null; - } - else if(isValidSubtype(tempSubType) && isValidSubtype(dbSubType) && (dbSubType.getName().equals(tempSubType.getName()))){ - newTypes[i] = SuperType.STRING; - newTokens[i] = null; - } else if(isAlphanumericString(tempToken) && isAlphanumericString(dbToken)){ + }else if(isAlphanumericString(tempToken) && isAlphanumericString(dbToken)){ newTypes[i] = SuperType.STRING; newTokens[i] = null; } else { @@ -682,6 +733,29 @@ else if(isValidSubtype(tempSubType) && isValidSubtype(dbSubType) && (dbSubType.g } + public static List getSTIListForUrlTemplate(URLTemplate urlTemplate, String orginalUrl, int apiCollectionId){ + + List stiList = new ArrayList<>(); + + Method extractedMethod = Method.fromString(orginalUrl.split(" ")[0]); + String extractedUrl = orginalUrl.split(" ")[1]; + String newTemplateUrl = urlTemplate.getTemplateString(); + for (int i = 0; i < urlTemplate.getTypes().length; i++) { + SuperType superType = urlTemplate.getTypes()[i]; + if (superType == null) continue; + int idx = extractedUrl.startsWith("http") ? i:i+1; + String word = extractedUrl.split("/")[idx]; + SingleTypeInfo.ParamId stiId = new SingleTypeInfo.ParamId(newTemplateUrl, extractedMethod.name(), -1, false, i+"", SingleTypeInfo.GENERIC, apiCollectionId, true); + SubType tokenSubType = KeyTypes.findSubType(word, "", null,true); + stiId.setSubType(tokenSubType); + SingleTypeInfo sti = new SingleTypeInfo( + stiId, new HashSet<>(), new HashSet<>(), 0, Context.now(), 0, CappedSet.create(i+""), + SingleTypeInfo.Domain.ENUM, SingleTypeInfo.ACCEPTED_MIN_VALUE, SingleTypeInfo.ACCEPTED_MAX_VALUE); + + stiList.add(sti); + } + return stiList; + } public static void mergeUrlsAndSave(int apiCollectionId, Boolean urlRegexMatchingEnabled) { @@ -699,44 +773,31 @@ public static void mergeUrlsAndSave(int apiCollectionId, Boolean urlRegexMatchin boolean isFirst = true; for (String matchedURL: matchStaticURLs) { Method delMethod = Method.fromString(matchedURL.split(" ")[0]); - String delEndpoint = matchedURL.split(" ")[1]; + String originalUrl = matchedURL.split(" ")[1]; Bson filterQ = Filters.and( Filters.eq("apiCollectionId", apiCollectionId), Filters.eq("method", delMethod.name()), - Filters.eq("url", delEndpoint) + Filters.eq("url", originalUrl) ); Bson filterQSampleData = Filters.and( Filters.eq("_id.apiCollectionId", apiCollectionId), Filters.eq("_id.method", delMethod.name()), - Filters.eq("_id.url", delEndpoint) + Filters.eq("_id.url", originalUrl) ); if (isFirst) { - String newTemplateUrl = urlTemplate.getTemplateString(); - for (int i = 0; i < urlTemplate.getTypes().length; i++) { - SuperType superType = urlTemplate.getTypes()[i]; - if (superType == null) continue; - int idx = delEndpoint.startsWith("http") ? i:i+1; - String word = delEndpoint.split("/")[idx]; - SingleTypeInfo.ParamId stiId = new SingleTypeInfo.ParamId(newTemplateUrl, delMethod.name(), -1, false, i+"", SingleTypeInfo.GENERIC, apiCollectionId, true); - SubType tokenSubType = KeyTypes.findSubType(word, "", null,true); - stiId.setSubType(tokenSubType); - SingleTypeInfo sti = new SingleTypeInfo( - stiId, new HashSet<>(), new HashSet<>(), 0, Context.now(), 0, CappedSet.create(i+""), - SingleTypeInfo.Domain.ENUM, SingleTypeInfo.ACCEPTED_MIN_VALUE, SingleTypeInfo.ACCEPTED_MAX_VALUE); - - - // SingleTypeInfoDao.instance.insertOne(sti); - bulkUpdatesForSti.add(new InsertOneModel<>(sti)); + List urlTemplateStis = getSTIListForUrlTemplate(urlTemplate, matchedURL, apiCollectionId); + for(SingleTypeInfo sti: urlTemplateStis) { + bulkUpdatesForSti.add(new InsertOneModel<>(sti)); } - // SingleTypeInfoDao.instance.getMCollection().updateMany(filterQ, Updates.set("url", newTemplateUrl)); + String newTemplateUrl = urlTemplate.getTemplateString(); + SingleTypeInfoDao.instance.getMCollection().updateMany(filterQ, Updates.set("url", newTemplateUrl)); bulkUpdatesForSti.add(new UpdateManyModel<>(filterQ, Updates.set("url", newTemplateUrl), new UpdateOptions())); - SampleData sd = SampleDataDao.instance.findOne(filterQSampleData); if (sd != null) { sd.getId().url = newTemplateUrl; @@ -765,12 +826,12 @@ public static void mergeUrlsAndSave(int apiCollectionId, Boolean urlRegexMatchin Bson filterForDependencyNode = Filters.or( Filters.and( Filters.eq(DependencyNode.API_COLLECTION_ID_REQ, apiCollectionId+""), - Filters.eq(DependencyNode.URL_REQ, delEndpoint), + Filters.eq(DependencyNode.URL_REQ, originalUrl), Filters.eq(DependencyNode.METHOD_REQ, delMethod.name()) ), Filters.and( Filters.eq(DependencyNode.API_COLLECTION_ID_RESP, apiCollectionId+""), - Filters.eq(DependencyNode.URL_RESP, delEndpoint), + Filters.eq(DependencyNode.URL_RESP, originalUrl), Filters.eq(DependencyNode.METHOD_RESP,delMethod.name()) ) ); diff --git a/apps/api-runtime/src/test/java/com/akto/parsers/TestMergingNew.java b/apps/api-runtime/src/test/java/com/akto/parsers/TestMergingNew.java index 2a370e4dd3..53fefa3d07 100644 --- a/apps/api-runtime/src/test/java/com/akto/parsers/TestMergingNew.java +++ b/apps/api-runtime/src/test/java/com/akto/parsers/TestMergingNew.java @@ -120,7 +120,7 @@ public void testmultipleUUIDForceMerge(){ parser.apiCatalogSync.syncWithDB(false,true); APICatalogSync.mergeUrlsAndSave(123,true); parser.apiCatalogSync.buildFromDB(false, true); - assertEquals(1, getStaticURLsSize(parser)); + assertEquals(0, getStaticURLsSize(parser)); parser.syncFunction(responseParams.subList(1,2), false,true); parser.apiCatalogSync.syncWithDB(false,true); @@ -179,7 +179,7 @@ public void testUUIDForceMerge() { parser.apiCatalogSync.syncWithDB(false, true); APICatalogSync.mergeUrlsAndSave(123,true); parser.apiCatalogSync.buildFromDB(false, true); - assertEquals(1, getStaticURLsSize(parser)); + assertEquals(0, getStaticURLsSize(parser)); parser.syncFunction(responseParams.subList(1,2), false, true); parser.apiCatalogSync.syncWithDB(false, true); @@ -216,7 +216,7 @@ public void testParameterizedURLsTestString() { SingleTypeInfoDao.instance.getMCollection().drop(); ApiCollectionsDao.instance.getMCollection().drop(); HttpCallParser parser = new HttpCallParser("userIdentifier", 1, 1, 1, true); - String url = "link/"; + String url = "/link/"; List responseParams = new ArrayList<>(); List urls = new ArrayList<>(); for (String x: Arrays.asList("A", "B", "C", "D", "E", "F", "G", "H")) { @@ -290,7 +290,7 @@ public void testEmptyResponsePayload() throws Exception { SingleTypeInfoDao.instance.getMCollection().drop(); ApiCollectionsDao.instance.getMCollection().drop(); - String a = "{\"path\": \"http://killdill.mpl.internal:8080/1.0/kb/paymentGateways/hosted/form/bbb-bbbb-bbb?paymentMethodId=qq-qqq-qqq\", \"method\": \"POST\", \"type\": \"HTTP/1.1\", \"requestHeaders\": \"{\\\"X-Killbill-ApiKey\\\": \\\"mplgaming\\\", \\\"Authorization\\\": \\\"Basic somerandom=\\\", \\\"X-Killbill-ApiSecret\\\": \\\"something\\\", \\\"Accept\\\": \\\"application/json\\\", \\\"X-MPL-COUNTRYCODE\\\": \\\"IN\\\", \\\"X-Killbill-CreatedBy\\\": \\\"test-payment\\\", \\\"Content-type\\\": \\\"application/json\\\"}\", \"requestPayload\": \"{\\\"formFields\\\":[{\\\"key\\\":\\\"amount\\\",\\\"value\\\":\\\"125.000\\\"},{\\\"key\\\":\\\"netAmount\\\",\\\"value\\\":\\\"125.000\\\"},{\\\"key\\\":\\\"currency\\\",\\\"value\\\":\\\"INR\\\"},{\\\"key\\\":\\\"orderId\\\",\\\"value\\\":\\\"ASGARD\\\"},{\\\"key\\\":\\\"paymentMethodId\\\",\\\"value\\\":\\\"zzzz-zzz-zzz-zzzz-zzzz\\\"},{\\\"key\\\":\\\"mobileNumber\\\",\\\"value\\\":\\\"+917021916328\\\"},{\\\"key\\\":\\\"countryCode\\\",\\\"value\\\":\\\"IN\\\"},{\\\"key\\\":\\\"chargeDetails\\\",\\\"value\\\":\\\"{\\\\\\\"charges\\\\\\\":[],\\\\\\\"totalCharges\\\\\\\":0,\\\\\\\"totalChargesLC\\\\\\\":0}\\\"},{\\\"key\\\":\\\"pegRate\\\",\\\"value\\\":\\\"1.0000\\\"},{\\\"key\\\":\\\"extraInfo\\\",\\\"value\\\":\\\"{\\\\\\\"paymentMode\\\\\\\":\\\\\\\"NB_ICICI\\\\\\\",\\\\\\\"additionalPluginInfo\\\\\\\":\\\\\\\"{\\\\\\\\\\\\\\\"merchantId\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"mpl_qa\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"clientId\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"mplgaming\\\\\\\\\\\\\\\"}\\\\\\\",\\\\\\\"paymentModePluginInfo\\\\\\\":\\\\\\\"{\\\\\\\\\\\\\\\"code\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"NB_ICICI\\\\\\\\\\\\\\\"}\\\\\\\",\\\\\\\"paymentMethodType\\\\\\\":\\\\\\\"netbanking\\\\\\\",\\\\\\\"paymentFlow\\\\\\\":\\\\\\\"JP2_AT_R4\\\\\\\"}\\\"},{\\\"key\\\":\\\"appVersion\\\",\\\"value\\\":\\\"1000174\\\"},{\\\"key\\\":\\\"savedPaymentDetails\\\",\\\"value\\\":\\\"{}\\\"},{\\\"key\\\":\\\"appType\\\",\\\"value\\\":\\\"CASH\\\"},{\\\"key\\\":\\\"savedPaymentDetails\\\",\\\"value\\\":\\\"{}\\\"}]}\", \"statusCode\": \"200\", \"responseHeaders\": \"{\\\"Date\\\": \\\"Mon, 18 Apr 2022 13:05:16 GMT\\\", \\\"Content-Type\\\": \\\"application/json\\\", \\\"Transfer-Encoding\\\": \\\"chunked\\\", \\\"Connection\\\": \\\"keep-alive\\\", \\\"Server\\\": \\\"Apache-Coyote/1.1\\\", \\\"Access-Control-Allow-Origin\\\": \\\"*\\\", \\\"Access-Control-Allow-Methods\\\": \\\"GET, POST, DELETE, PUT, OPTIONS\\\", \\\"Access-Control-Allow-Headers\\\": \\\"Authorization,Content-Type,Location,X-Killbill-ApiKey,X-Killbill-ApiSecret,X-Killbill-Comment,X-Killbill-CreatedBy,X-Killbill-Pagination-CurrentOffset,X-Killbill-Pagination-MaxNbRecords,X-Killbill-Pagination-NextOffset,X-Killbill-Pagination-NextPageUri,X-Killbill-Pagination-TotalNbRecords,X-Killbill-Reason\\\", \\\"Access-Control-Expose-Headers\\\": \\\"Authorization,Content-Type,Location,X-Killbill-ApiKey,X-Killbill-ApiSecret,X-Killbill-Comment,X-Killbill-CreatedBy,X-Killbill-Pagination-CurrentOffset,X-Killbill-Pagination-MaxNbRecords,X-Killbill-Pagination-NextOffset,X-Killbill-Pagination-NextPageUri,X-Killbill-Pagination-TotalNbRecords,X-Killbill-Reason\\\", \\\"Access-Control-Allow-Credentials\\\": \\\"true\\\"}\", \"status\": \"OK\", \"responsePayload\": \"\", \"ip\": \"\", \"time\": \"1650287116\", \"akto_account_id\": \"1000000\", \"akto_vxlan_id\": 123, \"source\": \"OTHER\"}"; + String a = "{\"path\": \"http://killdill.mpl.internal:8080/1/kb/paymentGateways/hosted/form/bbb-bbbb-bbb?paymentMethodId=qq-qqq-qqq\", \"method\": \"POST\", \"type\": \"HTTP/1.1\", \"requestHeaders\": \"{\\\"X-Killbill-ApiKey\\\": \\\"mplgaming\\\", \\\"Authorization\\\": \\\"Basic somerandom=\\\", \\\"X-Killbill-ApiSecret\\\": \\\"something\\\", \\\"Accept\\\": \\\"application/json\\\", \\\"X-MPL-COUNTRYCODE\\\": \\\"IN\\\", \\\"X-Killbill-CreatedBy\\\": \\\"test-payment\\\", \\\"Content-type\\\": \\\"application/json\\\"}\", \"requestPayload\": \"{\\\"formFields\\\":[{\\\"key\\\":\\\"amount\\\",\\\"value\\\":\\\"125.000\\\"},{\\\"key\\\":\\\"netAmount\\\",\\\"value\\\":\\\"125.000\\\"},{\\\"key\\\":\\\"currency\\\",\\\"value\\\":\\\"INR\\\"},{\\\"key\\\":\\\"orderId\\\",\\\"value\\\":\\\"ASGARD\\\"},{\\\"key\\\":\\\"paymentMethodId\\\",\\\"value\\\":\\\"zzzz-zzz-zzz-zzzz-zzzz\\\"},{\\\"key\\\":\\\"mobileNumber\\\",\\\"value\\\":\\\"+917021916328\\\"},{\\\"key\\\":\\\"countryCode\\\",\\\"value\\\":\\\"IN\\\"},{\\\"key\\\":\\\"chargeDetails\\\",\\\"value\\\":\\\"{\\\\\\\"charges\\\\\\\":[],\\\\\\\"totalCharges\\\\\\\":0,\\\\\\\"totalChargesLC\\\\\\\":0}\\\"},{\\\"key\\\":\\\"pegRate\\\",\\\"value\\\":\\\"1.0000\\\"},{\\\"key\\\":\\\"extraInfo\\\",\\\"value\\\":\\\"{\\\\\\\"paymentMode\\\\\\\":\\\\\\\"NB_ICICI\\\\\\\",\\\\\\\"additionalPluginInfo\\\\\\\":\\\\\\\"{\\\\\\\\\\\\\\\"merchantId\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"mpl_qa\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"clientId\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"mplgaming\\\\\\\\\\\\\\\"}\\\\\\\",\\\\\\\"paymentModePluginInfo\\\\\\\":\\\\\\\"{\\\\\\\\\\\\\\\"code\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"NB_ICICI\\\\\\\\\\\\\\\"}\\\\\\\",\\\\\\\"paymentMethodType\\\\\\\":\\\\\\\"netbanking\\\\\\\",\\\\\\\"paymentFlow\\\\\\\":\\\\\\\"JP2_AT_R4\\\\\\\"}\\\"},{\\\"key\\\":\\\"appVersion\\\",\\\"value\\\":\\\"1000174\\\"},{\\\"key\\\":\\\"savedPaymentDetails\\\",\\\"value\\\":\\\"{}\\\"},{\\\"key\\\":\\\"appType\\\",\\\"value\\\":\\\"CASH\\\"},{\\\"key\\\":\\\"savedPaymentDetails\\\",\\\"value\\\":\\\"{}\\\"}]}\", \"statusCode\": \"200\", \"responseHeaders\": \"{\\\"Date\\\": \\\"Mon, 18 Apr 2022 13:05:16 GMT\\\", \\\"Content-Type\\\": \\\"application/json\\\", \\\"Transfer-Encoding\\\": \\\"chunked\\\", \\\"Connection\\\": \\\"keep-alive\\\", \\\"Server\\\": \\\"Apache-Coyote/1.1\\\", \\\"Access-Control-Allow-Origin\\\": \\\"*\\\", \\\"Access-Control-Allow-Methods\\\": \\\"GET, POST, DELETE, PUT, OPTIONS\\\", \\\"Access-Control-Allow-Headers\\\": \\\"Authorization,Content-Type,Location,X-Killbill-ApiKey,X-Killbill-ApiSecret,X-Killbill-Comment,X-Killbill-CreatedBy,X-Killbill-Pagination-CurrentOffset,X-Killbill-Pagination-MaxNbRecords,X-Killbill-Pagination-NextOffset,X-Killbill-Pagination-NextPageUri,X-Killbill-Pagination-TotalNbRecords,X-Killbill-Reason\\\", \\\"Access-Control-Expose-Headers\\\": \\\"Authorization,Content-Type,Location,X-Killbill-ApiKey,X-Killbill-ApiSecret,X-Killbill-Comment,X-Killbill-CreatedBy,X-Killbill-Pagination-CurrentOffset,X-Killbill-Pagination-MaxNbRecords,X-Killbill-Pagination-NextOffset,X-Killbill-Pagination-NextPageUri,X-Killbill-Pagination-TotalNbRecords,X-Killbill-Reason\\\", \\\"Access-Control-Allow-Credentials\\\": \\\"true\\\"}\", \"status\": \"OK\", \"responsePayload\": \"\", \"ip\": \"\", \"time\": \"1650287116\", \"akto_account_id\": \"1000000\", \"akto_vxlan_id\": 123, \"source\": \"OTHER\"}"; List responseParams = new ArrayList<>(); for (int i = 0; i < 30; i++) { HttpResponseParams httpResponseParams = HttpCallParser.parseKafkaMessage(a); @@ -315,7 +315,7 @@ public void testEmptyResponsePayload() throws Exception { Map urlTemplateMap = parser.apiCatalogSync.getDbState(0).getTemplateURLToMethods(); Map urlStaticMap = parser.apiCatalogSync.getDbState(0).getStrictURLToMethods(); - assertEquals(urlTemplateMap.size(), 1); + assertEquals(urlTemplateMap.size(), 30); assertEquals(urlStaticMap.size(), 0); } @@ -326,7 +326,7 @@ public void testStrictIntoTemplate() { SingleTypeInfoDao.instance.getMCollection().drop(); ApiCollectionsDao.instance.getMCollection().drop(); HttpCallParser parser = new HttpCallParser("userIdentifier", 1, 1, 1, true); - String url = "api/books/"; + String url = "/api/books/"; List responseParams = new ArrayList<>(); List urls = new ArrayList<>(); for (int i=0; i< 50; i++) { @@ -366,7 +366,7 @@ public void test20percentCondition() { SingleTypeInfoDao.instance.getMCollection().drop(); ApiCollectionsDao.instance.getMCollection().drop(); HttpCallParser parser = new HttpCallParser("userIdentifier", 1, 1, 1, true); - String url = "api/"; + String url = "/api/"; List responseParams = new ArrayList<>(); List urls = new ArrayList<>(); for (int i=0; i< 50; i++) { @@ -569,7 +569,7 @@ public void testUrlParamSingleTypeInfoAndValues() { SingleTypeInfoDao.instance.getMCollection().drop(); ApiCollectionsDao.instance.getMCollection().drop(); HttpCallParser parser = new HttpCallParser("userIdentifier", 1, 1, 1, true); - String url = "api/"; + String url = "/api/"; List responseParams = new ArrayList<>(); List urls = new ArrayList<>(); for (int i=0; i< 300; i++) { @@ -587,7 +587,7 @@ public void testUrlParamSingleTypeInfoAndValues() { parser.apiCatalogSync.syncWithDB(false, true); // dbState doesn't have any template URLs initially so no urlParams are considered - testSampleSizeAndDomainOfSti(parser,0, 10, SingleTypeInfo.Domain.ENUM, SingleTypeInfo.Domain.ENUM); + testSampleSizeAndDomainOfSti(parser,10, 10, SingleTypeInfo.Domain.ENUM, SingleTypeInfo.Domain.ENUM); parser.syncFunction(responseParams.subList(10,55), false, true); parser.apiCatalogSync.syncWithDB(false, true); @@ -596,14 +596,14 @@ public void testUrlParamSingleTypeInfoAndValues() { // Now dbState has template URLs so urlParam values are now stored assertEquals(0,getStaticURLsSize(parser)); - testSampleSizeAndDomainOfSti(parser,1, 1, SingleTypeInfo.Domain.ENUM, SingleTypeInfo.Domain.ENUM); + testSampleSizeAndDomainOfSti(parser,55, 55, SingleTypeInfo.Domain.ENUM, SingleTypeInfo.Domain.ENUM); parser.apiCatalogSync.syncWithDB(false, true); parser.apiCatalogSync.syncWithDB(false, true); APICatalogSync.mergeUrlsAndSave(123,true); parser.apiCatalogSync.buildFromDB(false, true); - testSampleSizeAndDomainOfSti(parser, 1, 1, SingleTypeInfo.Domain.ENUM, SingleTypeInfo.Domain.ENUM); + testSampleSizeAndDomainOfSti(parser, 55, 55, SingleTypeInfo.Domain.ENUM, SingleTypeInfo.Domain.ENUM); // changing the parser symbolizes instance restart // using the new or old parser shouldn't change the result @@ -623,7 +623,7 @@ public void testUrlParamSingleTypeInfoAndValues() { parserNew.apiCatalogSync.buildFromDB(false, true); // both now range - testSampleSizeAndDomainOfSti(parserNew, 50, 50, SingleTypeInfo.Domain.RANGE, SingleTypeInfo.Domain.ANY); + testSampleSizeAndDomainOfSti(parserNew, 0, 0, SingleTypeInfo.Domain.RANGE, SingleTypeInfo.Domain.ANY); } @@ -649,7 +649,7 @@ public void testMinMaxAndLastSeenNew() { SingleTypeInfoDao.instance.getMCollection().drop(); ApiCollectionsDao.instance.getMCollection().drop(); HttpCallParser parser = new HttpCallParser("userIdentifier", 1, 1, 1, true); - String url = "api/"; + String url = "/api/"; // test for 1 url HttpResponseParams httpResponseParams1 = createHttpResponseForMinMax(url+"books1", 23.4F,-98F ); @@ -818,7 +818,7 @@ public void testSampleDataUpdate() throws Exception { httpCallParser.apiCatalogSync.syncWithDB(false, true); Bson filter2 = Filters.and( - Filters.eq("_id.url", "https://petstore.swagger.io/v2/books/1"), + Filters.eq("_id.url", "https://petstore.swagger.io/v2/books/INTEGER"), Filters.eq("_id.method", "POST") ); diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/transform.js b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/transform.js index 14205e13fc..2ff640d36e 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/transform.js +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/transform.js @@ -189,6 +189,7 @@ const transform = { prepareSampleData: (res, subType) => { let paths = [] for (const c in res.sensitiveSampleData) { + if(c === "") continue let paramInfoList = res.sensitiveSampleData[c] if (!paramInfoList) { paramInfoList = [] diff --git a/libs/dao/src/main/java/com/akto/dto/type/SingleTypeInfo.java b/libs/dao/src/main/java/com/akto/dto/type/SingleTypeInfo.java index 83a90a7939..9eccb2102e 100644 --- a/libs/dao/src/main/java/com/akto/dto/type/SingleTypeInfo.java +++ b/libs/dao/src/main/java/com/akto/dto/type/SingleTypeInfo.java @@ -124,7 +124,7 @@ public static void fetchCustomAuthTypes(int accountId) { } public enum SuperType { - BOOLEAN, INTEGER, FLOAT, STRING, NULL, OTHER, CUSTOM + BOOLEAN, INTEGER, FLOAT, STRING, NULL, OTHER, CUSTOM, OBJECT_ID } public enum Position { From d9bd3da40639d531f5713d636e7477a7c7bee2d0 Mon Sep 17 00:00:00 2001 From: Ark2307 Date: Thu, 8 Feb 2024 14:40:23 +0530 Subject: [PATCH 34/39] added collection name on export html page --- .../src/apps/dashboard/pages/Dashboard.jsx | 3 +- .../pages/testing/ExportHtml/ExportHtml.jsx | 32 +++++++++++++------ .../pages/testing/ExportHtml/styles.css | 1 + 3 files changed, 25 insertions(+), 11 deletions(-) diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/Dashboard.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/Dashboard.jsx index 0237b61930..d4bdcb8a20 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/Dashboard.jsx +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/Dashboard.jsx @@ -21,6 +21,7 @@ function Dashboard() { const setHostNameMap = PersistStore(state => state.setHostNameMap) const allCollections = PersistStore(state => state.allCollections) + const collectionsMap = PersistStore(state => state.collectionsMap) const fetchAllCollections = async () => { let apiCollections = await homeFunctions.getAllCollections() @@ -32,7 +33,7 @@ function Dashboard() { } useEffect(() => { - if(allCollections && allCollections.length === 0){ + if((allCollections && allCollections.length === 0) || (Object.keys(collectionsMap).length === 0)){ fetchAllCollections() } transform.setTestMetadata(); diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/ExportHtml/ExportHtml.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/ExportHtml/ExportHtml.jsx index 7b30503376..0b2f4ff5e2 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/ExportHtml/ExportHtml.jsx +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/ExportHtml/ExportHtml.jsx @@ -3,9 +3,8 @@ import { useParams } from 'react-router-dom' import issuesApi from '../../issues/api'; import api from '../api'; import PersistStore from '../../../../main/PersistStore'; -import { Avatar, Box, Button,Frame, HorizontalGrid, HorizontalStack, LegacyCard, Text, TopBar, VerticalStack, Icon, Badge, List, Link } from '@shopify/polaris' +import { Avatar, Box, Frame, HorizontalGrid, HorizontalStack, LegacyCard, Text, TopBar, VerticalStack, Icon} from '@shopify/polaris' import {FlagMajor, CollectionsMajor, ResourcesMajor, InfoMinor, CreditCardSecureMajor, FraudProtectMajor} from "@shopify/polaris-icons" -import func from '@/util/func' import './styles.css' import transform from '../transform'; @@ -48,8 +47,8 @@ function ExportHtml() { const [vulnerableResultsMap, setVulnerableResultsMap] = useState([]) ; const [dataToCurlObj, setDataToCurlObj] = useState({}); - const [infoState, setInfoState] = useState(moreInfoSections) const [severitiesCount,setSeveritiesCount] = useState({HIGH: 0, MEDIUM: 0, LOW: 0}) ; + const collectionsMap = PersistStore(state => state.collectionsMap) const subCategoryMap = PersistStore(state => state.subCategoryMap) @@ -275,7 +274,7 @@ function ExportHtml() { sections={fillContent(item)} item={item} dataToCurlObj={dataToCurlObj} - + collectionsMap={collectionsMap} /> @@ -289,6 +288,7 @@ function ExportHtml() { } function MoreInformationComponent(props) { + const getTruncatedString = (str) => { if (str && str.length > 3000) { return str.substr(0, 3000) + ' .........'; @@ -334,12 +334,24 @@ function MoreInformationComponent(props) { {props.item?.category?.vulnerableTestingRunResults?.map((testingRun, index)=> (
- - Vulnerable endpoint : - - - { testingRun.apiInfoKey.url } - + + + Vulnerable endpoint: + + + { testingRun.apiInfoKey.url } + + + + + + Collection name: {" "} + + + { props.collectionsMap[testingRun.apiInfoKey.apiCollectionId]} + + +
{testingRun?.testResults?.map((testRun, index1) => (
diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/ExportHtml/styles.css b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/ExportHtml/styles.css index 43fbbb2e4f..a632979338 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/ExportHtml/styles.css +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/ExportHtml/styles.css @@ -13,6 +13,7 @@ background: #ebf0f4; gap: 10px; padding: 0 8px; + justify-content: space-between; } .row-div-1 .title-name{ border-right: none !important; From 03729ab34348caf15141d4f1c804a4e9268abcf3 Mon Sep 17 00:00:00 2001 From: Ark2307 Date: Thu, 8 Feb 2024 15:30:33 +0530 Subject: [PATCH 35/39] fixing typo in test library page --- .../apps/dashboard/pages/settings/test_library/TestLibrary.jsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/settings/test_library/TestLibrary.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/settings/test_library/TestLibrary.jsx index 23baf54901..0124d945bd 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/settings/test_library/TestLibrary.jsx +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/settings/test_library/TestLibrary.jsx @@ -184,7 +184,7 @@ function TestLibrary() { components={components} title={ - API Collections + Test library } primaryAction={} From 1cd45b0171d1401fb5339eb210062e276e6ef4e7 Mon Sep 17 00:00:00 2001 From: aktoboy Date: Thu, 8 Feb 2024 15:50:13 +0530 Subject: [PATCH 36/39] Fixed fetchEndpoints() overloading --- .../java/com/akto/dao/SingleTypeInfoDao.java | 50 ++++++------------- 1 file changed, 15 insertions(+), 35 deletions(-) diff --git a/libs/dao/src/main/java/com/akto/dao/SingleTypeInfoDao.java b/libs/dao/src/main/java/com/akto/dao/SingleTypeInfoDao.java index 121803574b..039f7c6098 100644 --- a/libs/dao/src/main/java/com/akto/dao/SingleTypeInfoDao.java +++ b/libs/dao/src/main/java/com/akto/dao/SingleTypeInfoDao.java @@ -16,6 +16,7 @@ import com.mongodb.client.MongoCursor; import com.mongodb.client.model.*; +import org.apache.commons.lang3.StringUtils; import org.bson.conversions.Bson; public class SingleTypeInfoDao extends AccountsContextDao { @@ -269,10 +270,10 @@ public void resetCount() { // to get results irrespective of collections use negative value for apiCollectionId public List fetchEndpointsInCollection(int apiCollectionId) { Bson filter = null; - if (apiCollectionId != -1) { + if (apiCollectionId < 0) { filter = Filters.in(SingleTypeInfo._COLLECTION_IDS, apiCollectionId); } - return fetchEndpointsInCollection(filter, -1); + return fetchEndpoints(filter, null, -1); } public List fetchEndpointsInCollection(Method method) { @@ -281,44 +282,23 @@ public List fetchEndpointsInCollection(Method method) { // the filter obtained uses index. filter = MethodCondition.createMethodFilter(method); } - return fetchEndpointsInCollection(filter, 50); + return fetchEndpoints(filter, null, 50); } - public List fetchEndpointsInCollection(int apiCollectionId) { - Bson matchCriteria = null; - if (apiCollectionId != -1) { - matchCriteria = Filters.eq("apiCollectionId", apiCollectionId); - } - return fetchEndpoints(matchCriteria, "startTs"); - } public List fetchEndpointsBySubType(SingleTypeInfo.SubType subType, int skip, int limit) { - return fetchEndpoints(Filters.eq("subType", subType.getName()), "timestamp", skip, limit); - } - - public List fetchEndpointsInCollection(int apiCollectionId) { - Bson matchCriteria = null; - if (apiCollectionId != -1) { - matchCriteria = Filters.eq("apiCollectionId", apiCollectionId); - } - return fetchEndpoints(matchCriteria, "startTs"); + return fetchEndpoints(Filters.eq("subType", subType.getName()), "timestamp", limit, skip); } - public List fetchEndpointsBySubType(SingleTypeInfo.SubType subType, int skip, int limit) { - return fetchEndpoints(Filters.eq("subType", subType.getName()), "timestamp", skip, limit); - } - - - - private List fetchEndpointsInCollection(Bson filter, int limit) { + private List fetchEndpoints(Bson matchCriteria, String sortField, int... limitSkip) { List pipeline = new ArrayList<>(); BasicDBObject groupedId = new BasicDBObject("apiCollectionId", "$apiCollectionId") .append("url", "$url") .append("method", "$method"); - if(filter != null){ - pipeline.add(Aggregates.match(filter)); + if(matchCriteria != null) { + pipeline.add(Aggregates.match(matchCriteria)); } Bson projections = Projections.fields( @@ -327,15 +307,15 @@ private List fetchEndpointsInCollection(Bson filter, int lim pipeline.add(Aggregates.project(projections)); pipeline.add(Aggregates.group(groupedId)); - pipeline.add(Aggregates.sort(Sorts.descending(sortField))); - - if (skipLimit.length == 2) { - pipeline.add(Aggregates.limit(skipLimit[1])); - pipeline.add(Aggregates.skip(skipLimit[0])); + if(StringUtils.isNotEmpty(sortField)) { + pipeline.add(Aggregates.sort(Sorts.descending(sortField))); } - if (limit > 0) { - pipeline.add(Aggregates.limit(limit)); + if (limitSkip.length == 2) { + pipeline.add(Aggregates.limit(limitSkip[0])); + pipeline.add(Aggregates.skip(limitSkip[1])); + } else if(limitSkip.length == 1){ + pipeline.add(Aggregates.limit(limitSkip[0])); } MongoCursor endpointsCursor = instance.getMCollection().aggregate(pipeline, BasicDBObject.class).cursor(); From cdf0e276297da883c2bded830802aff2d80f26a0 Mon Sep 17 00:00:00 2001 From: Ark2307 Date: Fri, 9 Feb 2024 14:38:03 +0530 Subject: [PATCH 37/39] fixed merging for subtypes in url --- .../java/com/akto/runtime/APICatalogSync.java | 15 +++- .../observe/api_collections/ApiEndpoints.jsx | 89 ++++--------------- .../pages/observe/data_types/DataTypes.jsx | 5 +- .../pages/observe/data_types/transform.js | 5 +- 4 files changed, 39 insertions(+), 75 deletions(-) diff --git a/apps/api-runtime/src/main/java/com/akto/runtime/APICatalogSync.java b/apps/api-runtime/src/main/java/com/akto/runtime/APICatalogSync.java index 49ccfd37ad..d71887542b 100644 --- a/apps/api-runtime/src/main/java/com/akto/runtime/APICatalogSync.java +++ b/apps/api-runtime/src/main/java/com/akto/runtime/APICatalogSync.java @@ -205,7 +205,10 @@ public void computeDelta(URLAggregator origAggregator, boolean triggerTemplateGe for (URLStatic pending: pendingRequests.keySet()) { RequestTemplate pendingTemplate = pendingRequests.get(pending); - URLTemplate parameterisedTemplate = tryParamteresingUrl(pending); + URLTemplate parameterisedTemplate = null; + if((apiCollectionId != VULNERABLE_API_COLLECTION_ID) && (apiCollectionId != LLM_API_COLLECTION_ID)){ + parameterisedTemplate = tryParamteresingUrl(pending); + } if(parameterisedTemplate != null){ RequestTemplate rt = deltaCatalog.getTemplateURLToMethods().get(parameterisedTemplate); @@ -388,6 +391,16 @@ private static ApiMergerResult tryMergingWithKnownStrictURLs( continue; } + URLStatic newStaticUrl = new URLStatic(newEndpoint, newMethod); + + URLTemplate tempUrlTemplate = tryParamteresingUrl(newStaticUrl); + if(tempUrlTemplate != null){ + Set matchedStaticURLs = templateToStaticURLs.getOrDefault(tempUrlTemplate, new HashSet<>()); + matchedStaticURLs.add(newUrl); + templateToStaticURLs.put(tempUrlTemplate, matchedStaticURLs); + } + + int countSimilarURLs = 0; Map>> potentialMerges = new HashMap<>(); for(String aUrl: pendingRequests.keySet()) { diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/ApiEndpoints.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/ApiEndpoints.jsx index 5710525bbe..e36f9079f7 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/ApiEndpoints.jsx +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/ApiEndpoints.jsx @@ -559,6 +559,23 @@ function ApiEndpoints() { return ret; } + let modal = ( + setShowRedactModal(false)} + title="Note!" + primaryAction={{ + content: 'Enable', + onAction: redactCollection + }} + key="redact-modal" + > + + When enabled, existing sample payload values for this collection will be deleted, and data in all the future payloads for this collection will be redacted. Please note that your API Inventory, Sensitive data etc. will be intact. We will simply be deleting the sample payload values. + + + ) + const components = [ loading ? [] : ( showWorkflowTests ? [ @@ -624,7 +641,8 @@ function ApiEndpoints() { operation={actionOperation} currentApiGroupName={pageTitle} fetchData={fetchData} - /> + />, + modal ] ) ] @@ -640,75 +658,6 @@ function ApiEndpoints() { secondaryActions={secondaryActionsComponent} components={components} /> - let modal = ( - setShowRedactModal(false)} - title="Note!" - primaryAction={{ - content: 'Enable', - onAction: redactCollection - }} - > - - When enabled, existing sample payload values for this collection will be deleted, and data in all the future payloads for this collection will be redacted. Please note that your API Inventory, Sensitive data etc. will be intact. We will simply be deleting the sample payload values. - - - ) - - return ( - <> - - - - } - backUrl="/dashboard/observe/inventory" - secondaryActions={secondaryActionsComponent} - components={ - loading ? [] : - [ -
- { return "warning" }} - selected={selected} - onRowClick={handleRowClick} - onSelect={handleSelectedTab} - getFilteredItems={getFilteredItems} - mode={IndexFiltersMode.Default} - headings={headings} - useNewRow={true} - condensedHeight={true} - tableTabs={tableTabs} - /> - setIsGptScreenActive(false)} title="Akto GPT"> - - setIsGptScreenActive(false)}/> - - -
, - { return "warning" }} - isGptActive={isGptActive} - /> - ]} - /> - {modal} - ) } diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/data_types/DataTypes.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/data_types/DataTypes.jsx index a8ba3dea68..fa0850d7fc 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/data_types/DataTypes.jsx +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/data_types/DataTypes.jsx @@ -154,10 +154,13 @@ function DataTypes() { }, [currState]) const saveAction = async () => { + console.log(currState) if (currState.dataType === 'Akto') { let obj = { name: currState.name, - ...transform.convertToSensitiveData(currState) + redacted:currState.redacted, + ...transform.convertToSensitiveData(currState.sensitiveState), + } api.saveAktoDataType(obj).then((response) => { func.setToast(true, false, "Data type updated successfully"); diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/data_types/transform.js b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/data_types/transform.js index a8779b3098..f3b69d0f47 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/data_types/transform.js +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/data_types/transform.js @@ -11,11 +11,11 @@ const func = { redacted: 'false' }, - convertToSensitiveData: function(obj) { + convertToSensitiveData: function(state) { let sensitiveAlways = false; let sensitivePosition = [] ; - switch (obj.sensitiveState){ + switch (state){ case '1': sensitivePosition = ["REQUEST_PAYLOAD", "REQUEST_HEADER"]; break; @@ -34,7 +34,6 @@ const func = { let resultObj= { sensitiveAlways: sensitiveAlways, sensitivePosition: sensitivePosition, - redacted: obj.redacted } return resultObj; From f64ca77ca8d151bc5da6e63baa4e544293bf22be Mon Sep 17 00:00:00 2001 From: aktoboy Date: Fri, 9 Feb 2024 17:34:21 +0530 Subject: [PATCH 38/39] Fixed failing test --- libs/dao/src/main/java/com/akto/dao/SingleTypeInfoDao.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libs/dao/src/main/java/com/akto/dao/SingleTypeInfoDao.java b/libs/dao/src/main/java/com/akto/dao/SingleTypeInfoDao.java index 039f7c6098..3bb8e6235a 100644 --- a/libs/dao/src/main/java/com/akto/dao/SingleTypeInfoDao.java +++ b/libs/dao/src/main/java/com/akto/dao/SingleTypeInfoDao.java @@ -270,10 +270,10 @@ public void resetCount() { // to get results irrespective of collections use negative value for apiCollectionId public List fetchEndpointsInCollection(int apiCollectionId) { Bson filter = null; - if (apiCollectionId < 0) { + if (apiCollectionId != -1) { filter = Filters.in(SingleTypeInfo._COLLECTION_IDS, apiCollectionId); } - return fetchEndpoints(filter, null, -1); + return fetchEndpoints(filter, null); } public List fetchEndpointsInCollection(Method method) { From fc0f6e5ef0fac91fd21df109f5eac266fe067a8e Mon Sep 17 00:00:00 2001 From: Ark2307 Date: Fri, 9 Feb 2024 21:45:35 +0530 Subject: [PATCH 39/39] fixing url merging to remove host and fixing test --- .../src/main/java/com/akto/runtime/APICatalogSync.java | 4 +++- .../src/test/java/com/akto/parsers/TestDBSync.java | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/apps/api-runtime/src/main/java/com/akto/runtime/APICatalogSync.java b/apps/api-runtime/src/main/java/com/akto/runtime/APICatalogSync.java index d71887542b..0c56e75e59 100644 --- a/apps/api-runtime/src/main/java/com/akto/runtime/APICatalogSync.java +++ b/apps/api-runtime/src/main/java/com/akto/runtime/APICatalogSync.java @@ -654,7 +654,9 @@ public static URLTemplate tryParamteresingUrl(URLStatic newUrl){ Pattern pattern = patternToSubType.get(SingleTypeInfo.UUID); boolean allNull = true; SuperType[] newTypes = new SuperType[tokens.length]; - for(int i = 0; i < tokens.length; i ++) { + + int start = newUrl.getUrl().startsWith("http") ? 3 : 0; + for(int i = start; i < tokens.length; i ++) { String tempToken = tokens[i]; if (NumberUtils.isParsable(tempToken)) { diff --git a/apps/api-runtime/src/test/java/com/akto/parsers/TestDBSync.java b/apps/api-runtime/src/test/java/com/akto/parsers/TestDBSync.java index 4977d2e8a9..6083f9b1b4 100644 --- a/apps/api-runtime/src/test/java/com/akto/parsers/TestDBSync.java +++ b/apps/api-runtime/src/test/java/com/akto/parsers/TestDBSync.java @@ -208,7 +208,7 @@ public void testInvalidMergeParameterizedURL() { APICatalogSync sync = new APICatalogSync("access-token", 1, true); for (int i = 1; i <= 30; i ++ ) { - aggr.addURL(TestDump2.createSampleParams("user"+i, "payment/id"+i)); + aggr.addURL(TestDump2.createSampleParams("user"+i, "/payment/id"+i)); } sync.computeDelta(aggr, true, 123); sync.syncWithDB(false, true); @@ -217,7 +217,7 @@ public void testInvalidMergeParameterizedURL() { assertEquals(30, sync.getDbState(123).getStrictURLToMethods().size()); assertEquals(0, sync.getDbState(123).getTemplateURLToMethods().size()); - HttpResponseParams resp2 = TestDump2.createSampleParams("user1", "payment/history"); + HttpResponseParams resp2 = TestDump2.createSampleParams("user1", "/payment/history"); ArrayList newHeader = new ArrayList<>(); newHeader.add("hnew"); resp2.getHeaders().put("new header", newHeader);