From e67278ee346e949fe49c88d91e601ff5fb2bed9b Mon Sep 17 00:00:00 2001 From: Avneesh Hota Date: Thu, 7 Mar 2024 13:27:07 +0530 Subject: [PATCH 01/16] basic implementation of using dependency graph in testing --- .../main/java/com/akto/action/HarAction.java | 155 ++++++- .../src/main/java/com/akto/utils/Build.java | 66 +-- .../src/main/java/com/akto/utils/Memory.java | 235 ++++++++++ .../com/akto/test_editor/execution/Build.java | 424 ++++++++++++++++++ .../akto/test_editor/execution/Memory.java | 242 ++++++++++ .../YamlNodeExecutor.java | 21 +- .../akto/dto/dependency_flow/TreeHelper.java | 5 +- 7 files changed, 1105 insertions(+), 43 deletions(-) create mode 100644 apps/dashboard/src/main/java/com/akto/utils/Memory.java create mode 100644 apps/testing/src/main/java/com/akto/test_editor/execution/Build.java create mode 100644 apps/testing/src/main/java/com/akto/test_editor/execution/Memory.java diff --git a/apps/dashboard/src/main/java/com/akto/action/HarAction.java b/apps/dashboard/src/main/java/com/akto/action/HarAction.java index bca822d26d..7956b6abc6 100644 --- a/apps/dashboard/src/main/java/com/akto/action/HarAction.java +++ b/apps/dashboard/src/main/java/com/akto/action/HarAction.java @@ -1,18 +1,29 @@ package com.akto.action; +import com.akto.DaoInit; +import com.akto.action.test_editor.SaveTestEditorAction; import com.akto.dao.ApiCollectionsDao; import com.akto.dao.BurpPluginInfoDao; +import com.akto.dao.SampleDataDao; import com.akto.dao.context.Context; import com.akto.dao.file.FilesDao; import com.akto.dto.ApiCollection; +import com.akto.dto.ApiInfo; import com.akto.dto.HttpResponseParams; +import com.akto.dto.testing.TestingRunResult; +import com.akto.dto.testing.YamlTestResult; +import com.akto.dto.traffic.SampleData; +import com.akto.dto.type.URLMethods; import com.akto.har.HAR; import com.akto.log.LoggerMaker; import com.akto.dto.ApiToken.Utility; +import com.akto.test_editor.execution.Executor; +import com.akto.testing.yaml_tests.YamlTestTemplate; import com.akto.util.DashboardMode; import com.akto.utils.GzipUtils; import com.akto.utils.Utils; import com.mongodb.BasicDBObject; +import com.mongodb.ConnectionString; import com.mongodb.client.model.Filters; import com.opensymphony.xwork2.Action; import org.apache.commons.io.FileUtils; @@ -20,10 +31,7 @@ import java.io.File; import java.io.IOException; -import java.util.Arrays; -import java.util.Collection; -import java.util.List; -import java.util.UUID; +import java.util.*; public class HarAction extends UserAction { private String harString; @@ -36,6 +44,145 @@ public class HarAction extends UserAction { private byte[] tcpContent; private static final LoggerMaker loggerMaker = new LoggerMaker(HarAction.class); + public static void main(String[] args) { + DaoInit.init(new ConnectionString("mongodb://localhost:27017/admini")); + Context.accountId.set(1_000_000); + + ApiInfo.ApiInfoKey apiInfoKey = new ApiInfo.ApiInfoKey(1709612022, "https://juiceshop.akto.io/rest/products/reviews", URLMethods.Method.PATCH); + + SampleData sampleData = SampleDataDao.instance.findOne( + Filters.and( + Filters.eq("_id.apiCollectionId", apiInfoKey.getApiCollectionId()), + Filters.eq("_id.method",apiInfoKey.getMethod().name()), + Filters.eq("_id.url",apiInfoKey.getUrl()) + ) + ); + + BasicDBObject apiInfoKeyObj = new BasicDBObject() + .append("apiCollectionId", apiInfoKey.getApiCollectionId()) + .append("url", apiInfoKey.getUrl()) + .append("method", apiInfoKey.getMethod().name()); + + SaveTestEditorAction saveTestEditorAction = new SaveTestEditorAction(); + saveTestEditorAction.setApiInfoKey(apiInfoKeyObj); + saveTestEditorAction.setContent(testContent); + saveTestEditorAction.setSampleDataList(Collections.singletonList(sampleData)); + String result = saveTestEditorAction.runTestForGivenTemplate(); + System.out.println(result); + + TestingRunResult testingRunResult = saveTestEditorAction.getTestingRunResult(); + System.out.println(testingRunResult.getTestResults().size()); + for (TestingRunResult.TestLog testLog: testingRunResult.getTestLogs()) { + System.out.println(testLog.getMessage()); + } + } + + + public static String testContent = "id: REPORT_GENERATION_DOS\n" + + "info:\n" + + " name: \"Denial of Service Test on Report Generation Endpoint\"\n" + + " description: \"A Denial of Service (DoS) test on a report generation endpoint involves \n" + + " overwhelming the system with a high volume of requests to assess its \n" + + " resilience under heavy load. By bombarding the endpoint with \n" + + " numerous simultaneous requests for report generation, testers \n" + + " evaluate how well the system handles the load and whether it \n" + + " remains responsive. This testing helps identify potential bottlenecks \n" + + " or vulnerabilities in the report generation process, enabling \n" + + " proactive measures to fortify the system's defenses against \n" + + " DoS attacks targeting this endpoint.\"\n" + + " details: \"In this test, the report generation endpoint is bombarded with an \n" + + " excessive number of requests, aiming to simulate real-world peak \n" + + " loads and stress the system. Testers assess how the endpoint \n" + + " responds to this influx of requests, evaluating its ability \n" + + " to maintain responsiveness and generate reports efficiently. \n" + + " Through this process, potential weaknesses in scalability \n" + + " and performance are identified, enabling organizations to \n" + + " fortify their systems against Denial of Service (DoS) attacks \n" + + " on report generation functionalities.\"\n" + + " impact: \"A successful Denial of Service (DoS) test on a report generation endpoint \n" + + " can have significant consequences. It may lead to system slowdowns, \n" + + " unavailability, or crashes, hindering users' access to vital reports \n" + + " and disrupting business operations. Additionally, prolonged \n" + + " service disruptions can tarnish the organization's reputation, \n" + + " eroding user trust and potentially resulting in financial \n" + + " losses. Identifying and addressing vulnerabilities in the \n" + + " report generation process is crucial for maintaining system \n" + + " reliability and resilience against DoS attacks.\"\n" + + " category:\n" + + " name: RL\n" + + " shortName: Lack of Resources & Rate Limiting\n" + + " displayName: Lack of Resources & Rate Limiting (RL)\n" + + " subCategory: REPORT_GENERATION_DOS\n" + + " severity: HIGH\n" + + " tags:\n" + + " - Business logic\n" + + " - OWASP top 10\n" + + " - HackerOne top 10\n" + + " references:\n" + + " - \"https://github.com/OWASP/API-Security/blob/master/2019/en/src/0xa4-lack-of-resources-and-rate-limiting.md#scenario-2\"\n" + + " cwe:\n" + + " - CWE-400\n" + + " cve:\n" + + " - CVE-2023-4647\n" + + " - CVE-2023-38254\n" + + "api_selection_filters:\n" + + " response_code:\n" + + " gte: 200\n" + + " lt: 300\n" + + " url:\n" + + " contains_either:\n" + + " - a\n" + + "wordLists:\n" + + " dummyHeaders:\n" + + " - a\n" + + "\n" + + "execute:\n" + + " type: multiple\n" + + " requests:\n" + + " - req:\n" + + " - add_header:\n" + + " dummy_Header_Key: \"dummyValue\"\n" + + " - validate:\n" + + " percentage_match:\n" + + " gte: 90\n" + + " - success: x2\n" + + " - failure: exit\n" + + "\n" + + " - req:\n" + + " - add_header:\n" + + " dummy_Header_Key: \"dummyValue\"\n" + + " - validate:\n" + + " percentage_match:\n" + + " gte: 90\n" + + " - success: x3\n" + + " - failure: exit\n" + + "\n" + + " - req:\n" + + " - add_header:\n" + + " dummy_Header_Key: \"dummyValue\"\n" + + " - validate:\n" + + " percentage_match:\n" + + " gte: 90\n" + + " - success: x4\n" + + " - failure: exit\n" + + "\n" + + " - req:\n" + + " - add_header:\n" + + " dummy_Header_Key: \"dummyValue\"\n" + + " - validate:\n" + + " percentage_match:\n" + + " gte: 90\n" + + " - success: x5\n" + + " - failure: exit\n" + + "\n" + + "validate:\n" + + " and:\n" + + " - compare_greater:\n" + + " - ${x2.response.stats.median_response_time}\n" + + " - 3001\n" + + " - compare_greater:\n" + + " - ${x2.response.stats.median_response_time}\n" + + " - ${x1.response.stats.median_response_time} * 3"; @Override public String execute() throws IOException { ApiCollection apiCollection = null; diff --git a/apps/dashboard/src/main/java/com/akto/utils/Build.java b/apps/dashboard/src/main/java/com/akto/utils/Build.java index 1f7f981fd9..b94387504c 100644 --- a/apps/dashboard/src/main/java/com/akto/utils/Build.java +++ b/apps/dashboard/src/main/java/com/akto/utils/Build.java @@ -1,9 +1,6 @@ package com.akto.utils; -import com.akto.DaoInit; import com.akto.dao.DependencyFlowNodesDao; -import com.akto.dao.ModifyHostDetailsDao; -import com.akto.dao.ReplaceDetailsDao; import com.akto.dao.SampleDataDao; import com.akto.dao.context.Context; import com.akto.dto.*; @@ -13,7 +10,6 @@ import com.akto.dto.traffic.SampleData; import com.akto.dto.type.URLMethods; import com.akto.log.LoggerMaker; -import com.akto.parsers.HttpCallParser; import com.akto.runtime.policies.AuthPolicy; import com.akto.test_editor.execution.Operations; import com.akto.testing.ApiExecutor; @@ -24,7 +20,6 @@ import com.akto.util.modifier.SetValueModifier; import com.fasterxml.jackson.core.JsonFactory; import com.fasterxml.jackson.databind.ObjectMapper; -import com.mongodb.ConnectionString; import com.mongodb.client.model.Filters; import joptsimple.internal.Strings; import org.bson.conversions.Bson; @@ -41,8 +36,7 @@ public class Build { private static final LoggerMaker loggerMaker = new LoggerMaker(Build.class); - private void buildParentToChildMap(List nodes) { - parentToChildMap = new HashMap<>(); + public static void buildParentToChildMap(Collection nodes, Map parentToChildMap) { for (Node node: nodes) { if (node.getConnections() == null) continue; for (Connection connection: node.getConnections().values()) { @@ -69,7 +63,7 @@ private void buildParentToChildMap(List nodes) { } - private Map> buildLevelsToSampleDataMap(List nodes) { + public static Map> buildLevelsToSampleDataMap(List nodes) { // divide them into levels Map> levelsToSampleDataMap = new HashMap<>(); @@ -162,7 +156,7 @@ public void setIsSuccess(boolean success) { } Set apisReplayedSet = new HashSet<>(); - public List runPerLevel(List sdList, Map modifyHostDetailMap, Map replaceDetailsMap) { + public static List runPerLevel(List sdList, Map modifyHostDetailMap, Map replaceDetailsMap, Map parentToChildMap, Set apisReplayedSet) { List runResults = new ArrayList<>(); for (SampleData sampleData: sdList) { Key id = sampleData.getId(); @@ -249,7 +243,7 @@ public List run(List apiCollectionsIds, List nodes = DependencyFlowNodesDao.instance.findNodesForCollectionIds(apiCollectionsIds,false,0, 10_000); - buildParentToChildMap(nodes); + buildParentToChildMap(nodes, parentToChildMap); Map> levelsToSampleDataMap = buildLevelsToSampleDataMap(nodes); List runResults = new ArrayList<>(); @@ -261,7 +255,7 @@ public List run(List apiCollectionsIds, List runResultsPerLevel = runPerLevel(sdList, modifyHostDetailMap, replaceDetailsMap); + List runResultsPerLevel = runPerLevel(sdList, modifyHostDetailMap, replaceDetailsMap, parentToChildMap, apisReplayedSet); runResults.addAll(runResultsPerLevel); loggerMaker.infoAndAddToDb("Finished running level " + level, LoggerMaker.LogDb.DASHBOARD); } catch (Exception e) { @@ -282,7 +276,7 @@ public List run(List apiCollectionsIds, List runResultsAll = runPerLevel(filtered, modifyHostDetailMap, replaceDetailsMap); + List runResultsAll = runPerLevel(filtered, modifyHostDetailMap, replaceDetailsMap, parentToChildMap, apisReplayedSet); runResults.addAll(runResultsAll); skip += limit; if (all.size() < limit) break; @@ -340,7 +334,7 @@ public static void modifyRequest(OriginalHttpRequest request, ReplaceDetail repl } - public List fillSdList(List sdList) { + public static List fillSdList(List sdList) { if (sdList == null || sdList.isEmpty()) return new ArrayList<>(); List filters = new ArrayList<>(); @@ -359,30 +353,12 @@ public List fillSdList(List sdList) { static ObjectMapper mapper = new ObjectMapper(); static JsonFactory factory = mapper.getFactory(); - public void fillReplaceDetailsMap(ReverseNode reverseNode, OriginalHttpResponse response, Map replaceDetailsMap) { + public static void fillReplaceDetailsMap(ReverseNode reverseNode, OriginalHttpResponse response, Map replaceDetailsMap) { if (reverseNode == null) return; Map deltaReplaceDetailsMap = new HashMap<>(); - String respPayload = response.getBody(); - Map> valuesMap = extractValuesFromPayload(respPayload); - - Map> responseHeaders = response.getHeaders(); - for (String headerKey: responseHeaders.keySet()) { - List values = responseHeaders.get(headerKey); - if (values == null) continue; - - if (headerKey.equalsIgnoreCase("set-cookie")) { - Map cookieMap = AuthPolicy.parseCookie(values); - for (String cookieKey : cookieMap.keySet()) { - String cookieVal = cookieMap.get(cookieKey); - valuesMap.put(cookieKey, new HashSet<>(Collections.singletonList(cookieVal))); - } - } else { - valuesMap.put(headerKey, new HashSet<>(values)); - } - - } + Map> valuesMap = getValuesMap(response); Map connections = reverseNode.getReverseConnections(); for (ReverseConnection reverseConnection: connections.values()) { @@ -421,4 +397,28 @@ public void fillReplaceDetailsMap(ReverseNode reverseNode, OriginalHttpResponse } + public static Map> getValuesMap(OriginalHttpResponse response) { + String respPayload = response.getBody(); + Map> valuesMap = extractValuesFromPayload(respPayload); + + Map> responseHeaders = response.getHeaders(); + for (String headerKey: responseHeaders.keySet()) { + List values = responseHeaders.get(headerKey); + if (values == null) continue; + + if (headerKey.equalsIgnoreCase("set-cookie")) { + Map cookieMap = AuthPolicy.parseCookie(values); + for (String cookieKey : cookieMap.keySet()) { + String cookieVal = cookieMap.get(cookieKey); + valuesMap.put(cookieKey, new HashSet<>(Collections.singletonList(cookieVal))); + } + } else { + valuesMap.put(headerKey, new HashSet<>(values)); + } + + } + + return valuesMap; + } + } diff --git a/apps/dashboard/src/main/java/com/akto/utils/Memory.java b/apps/dashboard/src/main/java/com/akto/utils/Memory.java new file mode 100644 index 0000000000..8d48366c95 --- /dev/null +++ b/apps/dashboard/src/main/java/com/akto/utils/Memory.java @@ -0,0 +1,235 @@ +package com.akto.utils; + +import com.akto.DaoInit; +import com.akto.dao.SampleDataDao; +import com.akto.dao.context.Context; +import com.akto.dto.ApiInfo; +import com.akto.dto.OriginalHttpRequest; +import com.akto.dto.OriginalHttpResponse; +import com.akto.dto.RawApi; +import com.akto.dto.dependency_flow.*; +import com.akto.dto.testing.TestingRunConfig; +import com.akto.dto.traffic.Key; +import com.akto.dto.traffic.SampleData; +import com.akto.dto.type.URLMethods; +import com.akto.testing.ApiExecutor; +import com.akto.util.Constants; +import com.mongodb.ConnectionString; +import com.mongodb.client.model.Filters; +import org.bson.conversions.Bson; + +import java.util.*; + +import static com.akto.utils.Build.*; + +public class Memory { + + Map resultMap = new HashMap<>(); + private final Map parentToChildMap = new HashMap<>(); + private final Map nodesMap = new HashMap<>(); + + Map sampleDataMap = new HashMap<>(); + + private Map replaceDetailsMap = new HashMap<>(); + + public static void main(String[] args) throws Exception { + DaoInit.init(new ConnectionString("mongodb://localhost:27017/admini")); + Context.accountId.set(1_000_000); + +// https://juiceshop.akto.io/rest/user/login + List apiInfoKeys = new ArrayList<>(); + apiInfoKeys.add(new ApiInfo.ApiInfoKey(1709612022, "https://juiceshop.akto.io/rest/products/reviews", URLMethods.Method.PATCH)); + apiInfoKeys.add(new ApiInfo.ApiInfoKey(1709612022, "https://juiceshop.akto.io/rest/user/login", URLMethods.Method.POST)); + + Memory memory = new Memory(); + memory.build(apiInfoKeys, new HashMap<>()); + + OriginalHttpRequest req1 = memory.run(1709612022, "https://juiceshop.akto.io/rest/user/login", URLMethods.Method.POST.name()); + System.out.println("new"); + OriginalHttpResponse resp1 = ApiExecutor.sendRequest(req1, true, null, false, new ArrayList<>()); + memory.fillResponse(req1, resp1, 1709612022, "https://juiceshop.akto.io/rest/user/login", URLMethods.Method.POST.name()); + memory.reset(1709612022, "https://juiceshop.akto.io/rest/user/login", URLMethods.Method.POST.name()); + System.out.println("done"); + + + OriginalHttpRequest req2 = memory.run(1709612022, "https://juiceshop.akto.io/rest/products/reviews", URLMethods.Method.PATCH.name()); + System.out.println("new"); + OriginalHttpResponse resp2 = ApiExecutor.sendRequest(req2, true, null, false, new ArrayList<>()); + memory.fillResponse(req2, resp2, 1709612022, "https://juiceshop.akto.io/rest/products/reviews", URLMethods.Method.PATCH.name()); + memory.reset(1709612022, "https://juiceshop.akto.io/rest/products/reviews", URLMethods.Method.PATCH.name()); + System.out.println("done"); + + + OriginalHttpRequest req3 = memory.run(1709612022, "https://juiceshop.akto.io/rest/user/login", URLMethods.Method.POST.name()); + System.out.println("new"); + OriginalHttpResponse resp3 = ApiExecutor.sendRequest(req3, true, null, false, new ArrayList<>()); + memory.fillResponse(req3, resp3, 1709612022, "https://juiceshop.akto.io/rest/user/login", URLMethods.Method.POST.name()); + memory.reset(1709612022, "https://juiceshop.akto.io/rest/user/login", URLMethods.Method.POST.name()); + System.out.println("done"); + + + OriginalHttpRequest req4 = memory.run(1709612022, "https://juiceshop.akto.io/rest/products/reviews", URLMethods.Method.PATCH.name()); + System.out.println("new"); + OriginalHttpResponse resp4 = ApiExecutor.sendRequest(req4, true, null, false, new ArrayList<>()); + memory.fillResponse(req4, resp4, 1709612022, "https://juiceshop.akto.io/rest/products/reviews", URLMethods.Method.PATCH.name()); + memory.reset(1709612022, "https://juiceshop.akto.io/rest/products/reviews", URLMethods.Method.PATCH.name()); + System.out.println("done"); + + + } + + public void build(List apiInfoKeys, Map replaceDetailsMap) { + this.replaceDetailsMap = replaceDetailsMap; + + // find all parent APIs + TreeHelper treeHelper = new TreeHelper(); + for (ApiInfo.ApiInfoKey apiInfoKey: apiInfoKeys) { + treeHelper.buildTree(apiInfoKey.getApiCollectionId()+"", apiInfoKey.getUrl(), apiInfoKey.getMethod().name()); + } + Collection nodes = treeHelper.result.values(); + List filters = new ArrayList<>(); + for (Node node: nodes) { + nodesMap.put(node.hashCode(), node); + filters.add(Filters.and( + Filters.eq("_id.apiCollectionId", Integer.parseInt(node.getApiCollectionId())), + Filters.eq("_id.url", node.getUrl()), + Filters.eq("_id.method", node.getMethod()) + )); + } + + // fetch sample data + List sdList = SampleDataDao.instance.findAll(Filters.or(filters)); + for (SampleData sampleData: sdList) { + Key id = sampleData.getId(); + sampleDataMap.put(Objects.hash(id.getApiCollectionId(), id.getUrl(), id.getMethod().name()), sampleData); + } + + buildParentToChildMap(nodes, parentToChildMap); + } + + + public OriginalHttpRequest run(int apiCollectionId, String url, String method) { + int hash = Objects.hash(apiCollectionId+"", url, method); + if (resultMap.get(hash) != null) return resultMap.get(hash).getRequest(); + + // todo: optimise this.. no need to make db calls everytime + TreeHelper treeHelper = new TreeHelper(); + treeHelper.buildTree(apiCollectionId+"", url, method); + List nodes = new ArrayList<>(treeHelper.result.values()); + + nodes.sort(Comparator.comparingInt(Node::getMaxDepth)); + + List sampleDataList = new ArrayList<>(); + for (Node node: nodes) { + Integer nodeHash = Objects.hash(Integer.parseInt(node.getApiCollectionId()), node.getUrl(), node.getMethod()); + sampleDataList.add(sampleDataMap.get(nodeHash)); + } + + return execute(sampleDataList); + } + + + public OriginalHttpRequest execute(List sdList) { + int idx = 0; + for (SampleData sampleData: sdList) { + idx++; + boolean isFinal = sdList.size() == idx; // todo: find a better way + Key id = sampleData.getId(); + int hash = Objects.hash(id.getApiCollectionId()+"", id.getUrl(), id.getMethod().name()); + if (resultMap.containsKey(hash)) continue; + try { + List samples = sampleData.getSamples(); + if (samples.isEmpty()) continue;; + + String sample = samples.get(0); + OriginalHttpRequest request = new OriginalHttpRequest(); + request.buildFromSampleMessage(sample); + // todo: String newHost = findNewHost(request, modifyHostDetailMap); + String newHost = null; + + OriginalHttpResponse originalHttpResponse = new OriginalHttpResponse(); + originalHttpResponse.buildFromSampleMessage(sample); + + // do modifications + Node node = nodesMap.get(hash); + ReplaceDetail finalReplaceDetail = getReplaceDetail(node); + modifyRequest(request, finalReplaceDetail); + + if (isFinal) return request; + + TestingRunConfig testingRunConfig = new TestingRunConfig(0, new HashMap<>(), new ArrayList<>(), null,newHost, null); + + OriginalHttpResponse response = null; + try { + response = ApiExecutor.sendRequest(request,false, testingRunConfig, false, new ArrayList<>()); + request.getHeaders().remove(Constants.AKTO_IGNORE_FLAG); + RawApi rawApi = new RawApi(request, response, ""); + resultMap.put(hash, rawApi); + } catch (Exception e) { + e.printStackTrace(); + } + } catch (Exception e) { + e.printStackTrace(); + } + + } + + return null; + } + + + public void fillResponse(OriginalHttpRequest request, OriginalHttpResponse response, int apiCollectionId, String url, String method) { + int hash = Objects.hash(apiCollectionId+"", url, method); + RawApi rawApi = new RawApi(request, response, ""); + resultMap.put(hash, rawApi); + } + + public void reset(int apiCollectionId, String url, String method) { + // find all children + int hash = Objects.hash(apiCollectionId+"", url, method); + ReverseNode reverseNode = parentToChildMap.get(hash); + if (reverseNode == null) return; + + for (ReverseConnection reverseConnection: reverseNode.getReverseConnections().values()) { + for (ReverseEdge reverseEdge: reverseConnection.getReverseEdges()) { + int childApiCollectionId = Integer.parseInt(reverseEdge.getApiCollectionId()); + String childUrl = reverseEdge.getUrl(); + String childMethod = reverseEdge.getMethod(); + + int childHash = Objects.hash(childApiCollectionId+"", childUrl, childMethod); + resultMap.remove(childHash); + reset(childApiCollectionId, childUrl, childMethod); + } + } + } + + + public ReplaceDetail getReplaceDetail(Node node) { + ReplaceDetail replaceDetail = new ReplaceDetail(Integer.parseInt(node.getApiCollectionId()), node.getUrl(), node.getMethod(), new ArrayList<>()); + Map connections = node.getConnections(); + for (Connection connection: connections.values()) { + String requestParam = connection.getParam(); + List edges = connection.getEdges(); + if (edges.isEmpty()) continue; + Edge edge = edges.get(0); + String responseParam = edge.getParam(); + String parentApiCollectionId = edge.getApiCollectionId(); + String parentUrl = edge.getUrl(); + String parentMethod = edge.getMethod(); + int parentHash = Objects.hash(parentApiCollectionId, parentUrl, parentMethod); + RawApi rawApi = resultMap.get(parentHash); + if (rawApi == null) continue; + Map> valuesMap = getValuesMap(rawApi.getResponse()); + Set values = valuesMap.get(responseParam); + Object value = values != null && values.size() > 0 ? values.toArray()[0] : null; // todo: + if (value == null) continue; + + KVPair.KVType type = value instanceof Integer ? KVPair.KVType.INTEGER : KVPair.KVType.STRING; + KVPair kvPair = new KVPair(requestParam, value.toString(), connection.getIsHeader(), connection.getIsUrlParam(), type); + replaceDetail.addIfNotExist(kvPair); + } + + return replaceDetail; + } + +} diff --git a/apps/testing/src/main/java/com/akto/test_editor/execution/Build.java b/apps/testing/src/main/java/com/akto/test_editor/execution/Build.java new file mode 100644 index 0000000000..3ecab13e43 --- /dev/null +++ b/apps/testing/src/main/java/com/akto/test_editor/execution/Build.java @@ -0,0 +1,424 @@ +package com.akto.test_editor.execution; + +import com.akto.dao.DependencyFlowNodesDao; +import com.akto.dao.SampleDataDao; +import com.akto.dao.context.Context; +import com.akto.dto.ApiInfo; +import com.akto.dto.OriginalHttpRequest; +import com.akto.dto.OriginalHttpResponse; +import com.akto.dto.RawApi; +import com.akto.dto.dependency_flow.*; +import com.akto.dto.testing.TestingRunConfig; +import com.akto.dto.traffic.Key; +import com.akto.dto.traffic.SampleData; +import com.akto.dto.type.URLMethods; +import com.akto.log.LoggerMaker; +import com.akto.runtime.policies.AuthPolicy; +import com.akto.testing.ApiExecutor; +import com.akto.util.Constants; +import com.akto.util.HttpRequestResponseUtils; +import com.akto.util.JSONUtils; +import com.akto.util.modifier.SetValueModifier; +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.mongodb.client.model.Filters; +import joptsimple.internal.Strings; +import org.bson.conversions.Bson; + +import java.net.URI; +import java.util.*; + +import static com.akto.util.HttpRequestResponseUtils.FORM_URL_ENCODED_CONTENT_TYPE; +import static com.akto.util.HttpRequestResponseUtils.extractValuesFromPayload; + +public class Build { + + private Map parentToChildMap = new HashMap<>(); + + private static final LoggerMaker loggerMaker = new LoggerMaker(Build.class); + + public static void buildParentToChildMap(Collection nodes, Map parentToChildMap) { + for (Node node: nodes) { + if (node.getConnections() == null) continue; + for (Connection connection: node.getConnections().values()) { + String requestParam = connection.getParam(); + if (connection.getEdges() == null) continue; + for (Edge edge: connection.getEdges()) { + String responseParam = edge.getParam(); + ReverseNode reverseNode = new ReverseNode( + edge.getApiCollectionId(), edge.getUrl(), edge.getMethod(), new HashMap<>() + ); + + int key = reverseNode.hashCode(); + reverseNode = parentToChildMap.getOrDefault(key, reverseNode); + + Map reverseConnections = reverseNode.getReverseConnections(); + ReverseConnection reverseConnection = reverseConnections.getOrDefault(responseParam, new ReverseConnection(responseParam, new ArrayList<>())); + reverseConnection.getReverseEdges().add(new ReverseEdge(node.getApiCollectionId(), node.getUrl(), node.getMethod(), requestParam, edge.getCount(), connection.getIsUrlParam(), connection.getIsHeader())); + + reverseConnections.put(responseParam, reverseConnection); + parentToChildMap.put(key, reverseNode); + } + } + } + + } + + public static Map> buildLevelsToSampleDataMap(List nodes) { + + // divide them into levels + Map> levelsToSampleDataMap = new HashMap<>(); + for (Node node: nodes) { + int maxDepth = node.getMaxDepth(); + List list = levelsToSampleDataMap.getOrDefault(maxDepth, new ArrayList<>()); + int apiCollectionId = Integer.parseInt(node.getApiCollectionId()); + URLMethods.Method method = URLMethods.Method.valueOf(node.getMethod()); + list.add(new SampleData(new Key(apiCollectionId, node.getUrl(), method, 0,0,0), new ArrayList<>())); + levelsToSampleDataMap.put(maxDepth, list); + } + + return levelsToSampleDataMap; + } + + public static class RunResult { + + private ApiInfo.ApiInfoKey apiInfoKey; + private String currentMessage; + private String originalMessage; + private boolean success; + + public RunResult(ApiInfo.ApiInfoKey apiInfoKey, String currentMessage, String originalMessage, boolean success) { + this.apiInfoKey = apiInfoKey; + this.currentMessage = currentMessage; + this.originalMessage = originalMessage; + this.success = success; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + RunResult runResult = (RunResult) o; + return apiInfoKey.equals(runResult.apiInfoKey); + } + + @Override + public int hashCode() { + return Objects.hash(apiInfoKey); + } + + @Override + public String toString() { + return "RunResult{" + + "apiInfoKey=" + apiInfoKey + + ", currentMessage='" + currentMessage + '\'' + + ", originalMessage='" + originalMessage + '\'' + + ", success=" + success + + '}'; + } + + public ApiInfo.ApiInfoKey getApiInfoKey() { + return apiInfoKey; + } + + public void setApiInfoKey(ApiInfo.ApiInfoKey apiInfoKey) { + this.apiInfoKey = apiInfoKey; + } + + public String getCurrentMessage() { + return currentMessage; + } + + public void setCurrentMessage(String currentMessage) { + this.currentMessage = currentMessage; + } + + public String getOriginalMessage() { + return originalMessage; + } + + public void setOriginalMessage(String originalMessage) { + this.originalMessage = originalMessage; + } + + public boolean isSuccess() { + return success; + } + + public boolean getIsSuccess() { + return success; + } + + public void setIsSuccess(boolean success) { + this.success = success; + } + + + } + + Set apisReplayedSet = new HashSet<>(); + public static List runPerLevel(List sdList, Map modifyHostDetailMap, Map replaceDetailsMap, Map parentToChildMap, Set apisReplayedSet) { + List runResults = new ArrayList<>(); + for (SampleData sampleData: sdList) { + Key id = sampleData.getId(); + try { + List samples = sampleData.getSamples(); + if (samples.isEmpty()) continue;; + + String sample = samples.get(0); + OriginalHttpRequest request = new OriginalHttpRequest(); + request.buildFromSampleMessage(sample); + String newHost = findNewHost(request, modifyHostDetailMap); + + OriginalHttpResponse originalHttpResponse = new OriginalHttpResponse(); + originalHttpResponse.buildFromSampleMessage(sample); + + // do modifications + ReplaceDetail replaceDetail = replaceDetailsMap.get(Objects.hash(id.getApiCollectionId(), id.getUrl(), id.getMethod().name())); + modifyRequest(request, replaceDetail); + + TestingRunConfig testingRunConfig = new TestingRunConfig(0, new HashMap<>(), new ArrayList<>(), null,newHost, null); + + OriginalHttpResponse response = null; + try { + response = ApiExecutor.sendRequest(request,false, testingRunConfig, false, new ArrayList<>()); + apisReplayedSet.add(new ApiInfo.ApiInfoKey(id.getApiCollectionId(), id.getUrl(), id.getMethod())); + request.getHeaders().remove(Constants.AKTO_IGNORE_FLAG); + ReverseNode parentToChildNode = parentToChildMap.get(Objects.hash(id.getApiCollectionId()+"", id.getUrl(), id.getMethod().name())); + fillReplaceDetailsMap(parentToChildNode, response, replaceDetailsMap); + RawApi rawApi = new RawApi(request, response, ""); + rawApi.fillOriginalMessage(Context.accountId.get(), Context.now(), "", "HAR"); + RunResult runResult = new RunResult( + new ApiInfo.ApiInfoKey(id.getApiCollectionId(), id.getUrl(), id.getMethod()), + rawApi.getOriginalMessage(), + sample, + rawApi.getResponse().getStatusCode() == originalHttpResponse.getStatusCode() + ); + runResults.add(runResult); + } catch (Exception e) { + loggerMaker.errorAndAddToDb(e, "error while sending request in invoke dependency graph" + id.getUrl(), LoggerMaker.LogDb.DASHBOARD); + } + } catch (Exception e) { + loggerMaker.errorAndAddToDb(e, "error while running runPerLevel for " + id.getUrl(), LoggerMaker.LogDb.DASHBOARD); + } + + } + + return runResults; + } + + public static String findNewHost(OriginalHttpRequest request, Map modifyHostDetailMap) { + try { + String url = request.getFullUrlIncludingDomain(); + URI uri = new URI(url); + String currentHost = uri.getHost(); + ModifyHostDetail modifyHostDetail = modifyHostDetailMap.get(currentHost); + if (modifyHostDetail == null) { + if (uri.getPort() != -1) currentHost += ":" + uri.getPort(); + modifyHostDetail = modifyHostDetailMap.get(currentHost); + } + + if (modifyHostDetail == null) return null; + + String newHost = modifyHostDetail.getNewHost(); + if (newHost == null) return null; + if (newHost.startsWith("http")) { + return newHost; + } else { + return uri.getScheme() + "://" + newHost; + } + } catch (Exception ignored) { + } + return null; + } + + + public List run(List apiCollectionsIds, List modifyHostDetails, Map replaceDetailsMap) { + if (replaceDetailsMap == null) replaceDetailsMap = new HashMap<>(); + if (modifyHostDetails == null) modifyHostDetails = new ArrayList<>(); + + Map modifyHostDetailMap = new HashMap<>(); + for (ModifyHostDetail modifyHostDetail: modifyHostDetails) { + modifyHostDetailMap.put(modifyHostDetail.getCurrentHost(), modifyHostDetail); + } + + + List nodes = DependencyFlowNodesDao.instance.findNodesForCollectionIds(apiCollectionsIds,false,0, 10_000); + buildParentToChildMap(nodes, parentToChildMap); + Map> levelsToSampleDataMap = buildLevelsToSampleDataMap(nodes); + + List runResults = new ArrayList<>(); + // loop over levels and make requests + for (int level: levelsToSampleDataMap.keySet()) { + List sdList =levelsToSampleDataMap.get(level); + sdList = fillSdList(sdList); + if (sdList.isEmpty()) continue; + + loggerMaker.infoAndAddToDb("Running level: " + level, LoggerMaker.LogDb.DASHBOARD); + try { + List runResultsPerLevel = runPerLevel(sdList, modifyHostDetailMap, replaceDetailsMap, parentToChildMap, apisReplayedSet); + runResults.addAll(runResultsPerLevel); + loggerMaker.infoAndAddToDb("Finished running level " + level, LoggerMaker.LogDb.DASHBOARD); + } catch (Exception e) { + e.printStackTrace(); + loggerMaker.errorAndAddToDb(e, "Error while running for level " + level , LoggerMaker.LogDb.DASHBOARD); + } + } + + loggerMaker.infoAndAddToDb("Running independent APIs", LoggerMaker.LogDb.DASHBOARD); + int skip = 0; + int limit = 1000; + while (true) { + List all = SampleDataDao.instance.findAll(Filters.in("_id.apiCollectionId", apiCollectionsIds), skip,limit, null); + if (all.isEmpty()) break; + List filtered = new ArrayList<>(); + for (SampleData sampleData: all) { + Key key = sampleData.getId(); + if (apisReplayedSet.contains(new ApiInfo.ApiInfoKey(key.getApiCollectionId(), key.getUrl(), key.getMethod()))) continue; + filtered.add(sampleData); + } + List runResultsAll = runPerLevel(filtered, modifyHostDetailMap, replaceDetailsMap, parentToChildMap, apisReplayedSet); + runResults.addAll(runResultsAll); + skip += limit; + if (all.size() < limit) break; + } + loggerMaker.infoAndAddToDb("Finished running independent APIs", LoggerMaker.LogDb.DASHBOARD); + + return runResults; + } + + public static void modifyRequest(OriginalHttpRequest request, ReplaceDetail replaceDetail) { + RawApi rawApi = new RawApi(request, null, null); + + if (replaceDetail != null) { + List kvPairs = replaceDetail.getKvPairs(); + if (kvPairs != null && !kvPairs.isEmpty()) { + for (KVPair kvPair: kvPairs) { + if (kvPair.isHeader()) { + String value = kvPair.getValue()+""; + String headerValue = rawApi.getRequest().findHeaderValueIncludingInCookie(kvPair.getKey()); + if (headerValue == null) continue; + String[] headerValueSplit = headerValue.split(" "); + if (headerValueSplit.length == 2) { + String prefix = headerValueSplit[0]; + if (Arrays.asList("bearer", "basic").contains(prefix.toLowerCase())) { + value = prefix + " " + value; + } + } + Operations.modifyHeader(rawApi, kvPair.getKey(), value); + } else if (kvPair.isUrlParam()) { + String url = request.getUrl(); + String[] urlSplit = url.split("/"); + int position = Integer.parseInt(kvPair.getKey()); + urlSplit[position] = kvPair.getValue()+""; + String newUrl = Strings.join(urlSplit, "/"); + request.setUrl(newUrl); + } else { + Map store = new HashMap<>(); + store.put(kvPair.getKey(), kvPair.getValue()); + SetValueModifier setValueModifier = new SetValueModifier(store); + + Set values = new HashSet<>(); + values.add(kvPair.getKey()); + String modifiedBody = JSONUtils.modify(rawApi.getRequest().getJsonRequestBody(), values, setValueModifier); + String contentType = rawApi.getRequest().findContentType(); + if (contentType != null && contentType.equals(FORM_URL_ENCODED_CONTENT_TYPE)) { + modifiedBody = HttpRequestResponseUtils.jsonToFormUrlEncoded(modifiedBody); + } + rawApi.getRequest().setBody(modifiedBody); + + Operations.modifyQueryParam(rawApi, kvPair.getKey(), kvPair.getValue()); + } + } + } + } + + } + + public static List fillSdList(List sdList) { + if (sdList == null || sdList.isEmpty()) return new ArrayList<>(); + + List filters = new ArrayList<>(); + for (SampleData sampleData: sdList) { + // todo: batch for bigger lists + Key id = sampleData.getId(); + filters.add(Filters.and( + Filters.eq("_id.apiCollectionId", id.getApiCollectionId()), + Filters.eq("_id.url", id.getUrl()), + Filters.eq("_id.method", id.getMethod().name()) + )); + } + return SampleDataDao.instance.findAll(Filters.or(filters)); + } + + + static ObjectMapper mapper = new ObjectMapper(); + static JsonFactory factory = mapper.getFactory(); + public static void fillReplaceDetailsMap(ReverseNode reverseNode, OriginalHttpResponse response, Map replaceDetailsMap) { + if (reverseNode == null) return; + + Map deltaReplaceDetailsMap = new HashMap<>(); + + Map> valuesMap = getValuesMap(response); + + Map connections = reverseNode.getReverseConnections(); + for (ReverseConnection reverseConnection: connections.values()) { + String param = reverseConnection.getParam(); + Set values = valuesMap.get(param); + Object value = values != null && values.size() > 0 ? values.toArray()[0] : null; // todo: + if (value == null) continue; + + for (ReverseEdge reverseEdge: reverseConnection.getReverseEdges()) { + int apiCollectionId = Integer.parseInt(reverseEdge.getApiCollectionId()); + Integer id = Objects.hash(apiCollectionId, reverseEdge.getUrl(), reverseEdge.getMethod()); + + ReplaceDetail deltaReplaceDetail = deltaReplaceDetailsMap.get(id); + if (deltaReplaceDetail == null) { + deltaReplaceDetail = new ReplaceDetail(apiCollectionId, reverseEdge.getUrl(), reverseEdge.getMethod(), new ArrayList<>()); + deltaReplaceDetailsMap.put(id, deltaReplaceDetail); + } + + KVPair.KVType type = value instanceof Integer ? KVPair.KVType.INTEGER : KVPair.KVType.STRING; + KVPair kvPair = new KVPair(reverseEdge.getParam(), value.toString(), reverseEdge.getIsHeader(), reverseEdge.isUrlParam(), type); + deltaReplaceDetail.addIfNotExist(kvPair); + } + } + + for (Integer key: deltaReplaceDetailsMap.keySet()) { + ReplaceDetail replaceDetail = replaceDetailsMap.get(key); + ReplaceDetail deltaReplaceDetail = deltaReplaceDetailsMap.get(key); + if (replaceDetail == null) { + replaceDetail = deltaReplaceDetail; + } else { + replaceDetail.addIfNotExist(deltaReplaceDetail.getKvPairs()); + } + + replaceDetailsMap.put(key, replaceDetail); + } + + } + + public static Map> getValuesMap(OriginalHttpResponse response) { + String respPayload = response.getBody(); + Map> valuesMap = extractValuesFromPayload(respPayload); + + Map> responseHeaders = response.getHeaders(); + for (String headerKey: responseHeaders.keySet()) { + List values = responseHeaders.get(headerKey); + if (values == null) continue; + + if (headerKey.equalsIgnoreCase("set-cookie")) { + Map cookieMap = AuthPolicy.parseCookie(values); + for (String cookieKey : cookieMap.keySet()) { + String cookieVal = cookieMap.get(cookieKey); + valuesMap.put(cookieKey, new HashSet<>(Collections.singletonList(cookieVal))); + } + } else { + valuesMap.put(headerKey, new HashSet<>(values)); + } + + } + + return valuesMap; + } +} diff --git a/apps/testing/src/main/java/com/akto/test_editor/execution/Memory.java b/apps/testing/src/main/java/com/akto/test_editor/execution/Memory.java new file mode 100644 index 0000000000..0f01814e6b --- /dev/null +++ b/apps/testing/src/main/java/com/akto/test_editor/execution/Memory.java @@ -0,0 +1,242 @@ +package com.akto.test_editor.execution; + +import com.akto.DaoInit; +import com.akto.dao.SampleDataDao; +import com.akto.dao.context.Context; +import com.akto.dto.ApiInfo; +import com.akto.dto.OriginalHttpRequest; +import com.akto.dto.OriginalHttpResponse; +import com.akto.dto.RawApi; +import com.akto.dto.dependency_flow.*; +import com.akto.dto.testing.TestingRunConfig; +import com.akto.dto.traffic.Key; +import com.akto.dto.traffic.SampleData; +import com.akto.dto.type.URLMethods; +import com.akto.testing.ApiExecutor; +import com.akto.util.Constants; +import com.mongodb.ConnectionString; +import com.mongodb.client.model.Filters; +import org.bson.conversions.Bson; + +import java.util.*; + +import static com.akto.test_editor.execution.Build.*; + + +public class Memory { + + Map resultMap = new HashMap<>(); + private final Map parentToChildMap = new HashMap<>(); + private final Map nodesMap = new HashMap<>(); + + Map sampleDataMap = new HashMap<>(); + + public static Memory memory = new Memory( + Arrays.asList( + new ApiInfo.ApiInfoKey(1709612022, "https://juiceshop.akto.io/rest/products/reviews", URLMethods.Method.PATCH), + new ApiInfo.ApiInfoKey(1709612022, "https://juiceshop.akto.io/rest/user/login", URLMethods.Method.POST) + ), new HashMap<>() + ); + + private Map replaceDetailsMap = new HashMap<>(); + + public static void main(String[] args) throws Exception { + DaoInit.init(new ConnectionString("mongodb://localhost:27017/admini")); + Context.accountId.set(1_000_000); + +// https://juiceshop.akto.io/rest/user/login + List apiInfoKeys = new ArrayList<>(); + apiInfoKeys.add(new ApiInfo.ApiInfoKey(1709612022, "https://juiceshop.akto.io/rest/products/reviews", URLMethods.Method.PATCH)); + apiInfoKeys.add(new ApiInfo.ApiInfoKey(1709612022, "https://juiceshop.akto.io/rest/user/login", URLMethods.Method.POST)); + + Memory memory = new Memory(apiInfoKeys, new HashMap<>()); + + OriginalHttpRequest req1 = memory.run(1709612022, "https://juiceshop.akto.io/rest/user/login", URLMethods.Method.POST.name()); + System.out.println("new"); + OriginalHttpResponse resp1 = ApiExecutor.sendRequest(req1, true, null, false, new ArrayList<>()); + memory.fillResponse(req1, resp1, 1709612022, "https://juiceshop.akto.io/rest/user/login", URLMethods.Method.POST.name()); + memory.reset(1709612022, "https://juiceshop.akto.io/rest/user/login", URLMethods.Method.POST.name()); + System.out.println("done"); + + + OriginalHttpRequest req2 = memory.run(1709612022, "https://juiceshop.akto.io/rest/products/reviews", URLMethods.Method.PATCH.name()); + System.out.println("new"); + OriginalHttpResponse resp2 = ApiExecutor.sendRequest(req2, true, null, false, new ArrayList<>()); + memory.fillResponse(req2, resp2, 1709612022, "https://juiceshop.akto.io/rest/products/reviews", URLMethods.Method.PATCH.name()); + memory.reset(1709612022, "https://juiceshop.akto.io/rest/products/reviews", URLMethods.Method.PATCH.name()); + System.out.println("done"); + + + OriginalHttpRequest req3 = memory.run(1709612022, "https://juiceshop.akto.io/rest/user/login", URLMethods.Method.POST.name()); + System.out.println("new"); + OriginalHttpResponse resp3 = ApiExecutor.sendRequest(req3, true, null, false, new ArrayList<>()); + memory.fillResponse(req3, resp3, 1709612022, "https://juiceshop.akto.io/rest/user/login", URLMethods.Method.POST.name()); + memory.reset(1709612022, "https://juiceshop.akto.io/rest/user/login", URLMethods.Method.POST.name()); + System.out.println("done"); + + + OriginalHttpRequest req4 = memory.run(1709612022, "https://juiceshop.akto.io/rest/products/reviews", URLMethods.Method.PATCH.name()); + System.out.println("new"); + OriginalHttpResponse resp4 = ApiExecutor.sendRequest(req4, true, null, false, new ArrayList<>()); + memory.fillResponse(req4, resp4, 1709612022, "https://juiceshop.akto.io/rest/products/reviews", URLMethods.Method.PATCH.name()); + memory.reset(1709612022, "https://juiceshop.akto.io/rest/products/reviews", URLMethods.Method.PATCH.name()); + System.out.println("done"); + + + } + + public Memory(List apiInfoKeys, Map replaceDetailsMap) { + this.replaceDetailsMap = replaceDetailsMap; + + // find all parent APIs + TreeHelper treeHelper = new TreeHelper(); + for (ApiInfo.ApiInfoKey apiInfoKey: apiInfoKeys) { + treeHelper.buildTree(apiInfoKey.getApiCollectionId()+"", apiInfoKey.getUrl(), apiInfoKey.getMethod().name()); + } + Collection nodes = treeHelper.result.values(); + List filters = new ArrayList<>(); + for (Node node: nodes) { + nodesMap.put(node.hashCode(), node); + filters.add(Filters.and( + Filters.eq("_id.apiCollectionId", Integer.parseInt(node.getApiCollectionId())), + Filters.eq("_id.url", node.getUrl()), + Filters.eq("_id.method", node.getMethod()) + )); + } + + // fetch sample data + List sdList = SampleDataDao.instance.findAll(Filters.or(filters)); + for (SampleData sampleData: sdList) { + Key id = sampleData.getId(); + sampleDataMap.put(Objects.hash(id.getApiCollectionId(), id.getUrl(), id.getMethod().name()), sampleData); + } + + buildParentToChildMap(nodes, parentToChildMap); + } + + + public OriginalHttpRequest run(int apiCollectionId, String url, String method) { + int hash = Objects.hash(apiCollectionId+"", url, method); + if (resultMap.get(hash) != null) return resultMap.get(hash).getRequest(); + + // todo: optimise this.. no need to make db calls everytime + TreeHelper treeHelper = new TreeHelper(); + treeHelper.buildTree(apiCollectionId+"", url, method); + List nodes = new ArrayList<>(treeHelper.result.values()); + + nodes.sort(Comparator.comparingInt(Node::getMaxDepth)); + + List sampleDataList = new ArrayList<>(); + for (Node node: nodes) { + Integer nodeHash = Objects.hash(Integer.parseInt(node.getApiCollectionId()), node.getUrl(), node.getMethod()); + sampleDataList.add(sampleDataMap.get(nodeHash)); + } + + return execute(sampleDataList); + } + + + private OriginalHttpRequest execute(List sdList) { + int idx = 0; + for (SampleData sampleData: sdList) { + idx++; + boolean isFinal = sdList.size() == idx; // todo: find a better way + Key id = sampleData.getId(); + int hash = Objects.hash(id.getApiCollectionId()+"", id.getUrl(), id.getMethod().name()); + if (resultMap.containsKey(hash)) continue; + try { + List samples = sampleData.getSamples(); + if (samples.isEmpty()) continue;; + + String sample = samples.get(0); + OriginalHttpRequest request = new OriginalHttpRequest(); + request.buildFromSampleMessage(sample); + // todo: String newHost = findNewHost(request, modifyHostDetailMap); + String newHost = null; + + OriginalHttpResponse originalHttpResponse = new OriginalHttpResponse(); + originalHttpResponse.buildFromSampleMessage(sample); + + // do modifications + Node node = nodesMap.get(hash); + ReplaceDetail finalReplaceDetail = getReplaceDetail(node); + modifyRequest(request, finalReplaceDetail); + + if (isFinal) return request; + + TestingRunConfig testingRunConfig = new TestingRunConfig(0, new HashMap<>(), new ArrayList<>(), null,newHost, null); + + OriginalHttpResponse response = null; + try { + response = ApiExecutor.sendRequest(request,false, testingRunConfig, false, new ArrayList<>()); + request.getHeaders().remove(Constants.AKTO_IGNORE_FLAG); + RawApi rawApi = new RawApi(request, response, ""); + resultMap.put(hash, rawApi); + } catch (Exception e) { + e.printStackTrace(); + } + } catch (Exception e) { + e.printStackTrace(); + } + + } + + return null; + } + + + public void fillResponse(OriginalHttpRequest request, OriginalHttpResponse response, int apiCollectionId, String url, String method) { + int hash = Objects.hash(apiCollectionId+"", url, method); + RawApi rawApi = new RawApi(request, response, ""); + resultMap.put(hash, rawApi); + } + + public void reset(int apiCollectionId, String url, String method) { + // find all children + int hash = Objects.hash(apiCollectionId+"", url, method); + ReverseNode reverseNode = parentToChildMap.get(hash); + if (reverseNode == null) return; + + for (ReverseConnection reverseConnection: reverseNode.getReverseConnections().values()) { + for (ReverseEdge reverseEdge: reverseConnection.getReverseEdges()) { + int childApiCollectionId = Integer.parseInt(reverseEdge.getApiCollectionId()); + String childUrl = reverseEdge.getUrl(); + String childMethod = reverseEdge.getMethod(); + + int childHash = Objects.hash(childApiCollectionId+"", childUrl, childMethod); + resultMap.remove(childHash); + reset(childApiCollectionId, childUrl, childMethod); + } + } + } + + + public ReplaceDetail getReplaceDetail(Node node) { + ReplaceDetail replaceDetail = new ReplaceDetail(Integer.parseInt(node.getApiCollectionId()), node.getUrl(), node.getMethod(), new ArrayList<>()); + Map connections = node.getConnections(); + for (Connection connection: connections.values()) { + String requestParam = connection.getParam(); + List edges = connection.getEdges(); + if (edges.isEmpty()) continue; + Edge edge = edges.get(0); + String responseParam = edge.getParam(); + String parentApiCollectionId = edge.getApiCollectionId(); + String parentUrl = edge.getUrl(); + String parentMethod = edge.getMethod(); + int parentHash = Objects.hash(parentApiCollectionId, parentUrl, parentMethod); + RawApi rawApi = resultMap.get(parentHash); + if (rawApi == null) continue; + Map> valuesMap = getValuesMap(rawApi.getResponse()); + Set values = valuesMap.get(responseParam); + Object value = values != null && values.size() > 0 ? values.toArray()[0] : null; // todo: + if (value == null) continue; + + KVPair.KVType type = value instanceof Integer ? KVPair.KVType.INTEGER : KVPair.KVType.STRING; + KVPair kvPair = new KVPair(requestParam, value.toString(), connection.getIsHeader(), connection.getIsUrlParam(), type); + replaceDetail.addIfNotExist(kvPair); + } + + return replaceDetail; + } + +} diff --git a/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/YamlNodeExecutor.java b/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/YamlNodeExecutor.java index ec8f747cbf..dafb84245e 100644 --- a/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/YamlNodeExecutor.java +++ b/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/YamlNodeExecutor.java @@ -8,15 +8,14 @@ import java.util.Map; import com.akto.testing.Main; +import com.akto.dto.*; +import com.akto.dto.type.URLMethods; +import com.akto.test_editor.execution.Memory; import org.json.JSONObject; import com.akto.dao.context.Context; import com.akto.dao.test_editor.TestEditorEnums; import com.akto.dao.test_editor.YamlTemplateDao; -import com.akto.dto.ApiInfo; -import com.akto.dto.CustomAuthType; -import com.akto.dto.OriginalHttpResponse; -import com.akto.dto.RawApi; import com.akto.dto.api_workflow.Node; import com.akto.dto.test_editor.ConfigParserResult; import com.akto.dto.test_editor.ExecuteAlgoObj; @@ -48,7 +47,10 @@ public class YamlNodeExecutor extends NodeExecutor { private static final Gson gson = new Gson(); + static int counter = 0; + public NodeResult processNode(Node node, Map varMap, Boolean allowAllStatusCodes, boolean debug, List testLogs) { + counter++; List testErrors = new ArrayList<>(); YamlNodeDetails yamlNodeDetails = (YamlNodeDetails) node.getWorkflowNodeDetails(); @@ -56,9 +58,16 @@ public NodeResult processNode(Node node, Map varMap, Boolean all if (yamlNodeDetails.getTestId() != null && yamlNodeDetails.getTestId().length() > 0) { return processYamlNode(node, varMap, allowAllStatusCodes, yamlNodeDetails, debug, testLogs); } - + RawApi rawApi = yamlNodeDetails.getRawApi(); RawApi sampleRawApi = rawApi.copy(); + ApiInfo.ApiInfoKey apiInfoKey = counter % 2 != 0 ? new ApiInfo.ApiInfoKey(1709612022, "https://juiceshop.akto.io/rest/user/login", URLMethods.Method.POST) : new ApiInfo.ApiInfoKey(1709612022, "https://juiceshop.akto.io/rest/products/reviews", URLMethods.Method.PATCH); + System.out.println("*******"); + System.out.println(apiInfoKey); + + OriginalHttpRequest newRequest = Memory.memory.run(apiInfoKey.getApiCollectionId(), apiInfoKey.getUrl(), apiInfoKey.getMethod().name()); + rawApi.setRequest(newRequest); + List rawApis = new ArrayList<>(); rawApis.add(rawApi.copy()); @@ -114,6 +123,8 @@ public NodeResult processNode(Node node, Map varMap, Boolean all try { tsBeforeReq = Context.nowInMillis(); testResponse = ApiExecutor.sendRequest(testReq.getRequest(), followRedirect, testingRunConfig, debug, testLogs, Main.SKIP_SSRF_CHECK); + Memory.memory.fillResponse(testReq.getRequest(), testResponse, apiInfoKey.getApiCollectionId(), apiInfoKey.getUrl(), apiInfoKey.getMethod().name()); + Memory.memory.reset(apiInfoKey.getApiCollectionId(), apiInfoKey.getUrl(), apiInfoKey.getMethod().name()); tsAfterReq = Context.nowInMillis(); responseTimeArr.add(tsAfterReq - tsBeforeReq); ExecutionResult attempt = new ExecutionResult(singleReq.getSuccess(), singleReq.getErrMsg(), testReq.getRequest(), testResponse); diff --git a/libs/dao/src/main/java/com/akto/dto/dependency_flow/TreeHelper.java b/libs/dao/src/main/java/com/akto/dto/dependency_flow/TreeHelper.java index 1c094b02a4..1917d2108f 100644 --- a/libs/dao/src/main/java/com/akto/dto/dependency_flow/TreeHelper.java +++ b/libs/dao/src/main/java/com/akto/dto/dependency_flow/TreeHelper.java @@ -34,10 +34,13 @@ public void buildTree(String apiCollectionId, String url, String method) { ) ); - if (node == null || node.getMaxDepth() == 0) return; + if (node == null) return; result.put(node.hashCode(), node); + if (node.getMaxDepth() == 0) return; // todo: check implication on front end graph + + Map connections = node.getConnections(); for (Connection connection: connections.values()) { List edges = connection.getEdges(); From 35ff291c1ecccf9e4dc1f3663923424bd0b947b6 Mon Sep 17 00:00:00 2001 From: Avneesh Hota Date: Mon, 11 Mar 2024 18:34:40 +0530 Subject: [PATCH 02/16] removed duplicate files and added API to TestEditorEnums.java --- .../java/com/akto/parsers/HttpCallParser.java | 6 +- .../com/akto/action/DependencyAction.java | 2 +- .../main/java/com/akto/action/HarAction.java | 10 +- .../src/main/java/com/akto/utils/Build.java | 424 ------------------ .../src/main/java/com/akto/utils/Memory.java | 235 ---------- .../src/main/java/com/akto/utils/Utils.java | 14 +- .../akto/test_editor/execution/Memory.java | 32 +- .../YamlNodeExecutor.java | 27 +- .../akto/dao/test_editor/TestEditorEnums.java | 1 + 9 files changed, 55 insertions(+), 696 deletions(-) delete mode 100644 apps/dashboard/src/main/java/com/akto/utils/Build.java delete mode 100644 apps/dashboard/src/main/java/com/akto/utils/Memory.java diff --git a/apps/api-runtime/src/main/java/com/akto/parsers/HttpCallParser.java b/apps/api-runtime/src/main/java/com/akto/parsers/HttpCallParser.java index 42f0603b51..e7d5ed0de5 100644 --- a/apps/api-runtime/src/main/java/com/akto/parsers/HttpCallParser.java +++ b/apps/api-runtime/src/main/java/com/akto/parsers/HttpCallParser.java @@ -172,9 +172,9 @@ public void syncFunction(List responseParams, boolean syncIm apiCatalogSync.computeDelta(aggregator, false, apiCollectionId); } - // for (HttpResponseParams responseParam: filteredResponseParams) { - // dependencyAnalyser.analyse(responseParam.getOrig(), responseParam.requestParams.getApiCollectionId()); - // } + for (HttpResponseParams responseParam: filteredResponseParams) { + dependencyAnalyser.analyse(responseParam.getOrig(), responseParam.requestParams.getApiCollectionId()); + } this.sync_count += filteredResponseParams.size(); int syncThresh = numberOfSyncs < 10 ? 10000 : sync_threshold_count; diff --git a/apps/dashboard/src/main/java/com/akto/action/DependencyAction.java b/apps/dashboard/src/main/java/com/akto/action/DependencyAction.java index f522c1dd77..0575280189 100644 --- a/apps/dashboard/src/main/java/com/akto/action/DependencyAction.java +++ b/apps/dashboard/src/main/java/com/akto/action/DependencyAction.java @@ -14,7 +14,7 @@ import com.akto.dto.type.URLMethods.Method; import com.akto.log.LoggerMaker; import com.akto.runtime.RelationshipSync; -import com.akto.utils.Build; +import com.akto.test_editor.execution.Build; import com.akto.utils.Utils; import com.mongodb.BasicDBObject; import com.mongodb.ConnectionString; diff --git a/apps/dashboard/src/main/java/com/akto/action/HarAction.java b/apps/dashboard/src/main/java/com/akto/action/HarAction.java index 7956b6abc6..802a602ff9 100644 --- a/apps/dashboard/src/main/java/com/akto/action/HarAction.java +++ b/apps/dashboard/src/main/java/com/akto/action/HarAction.java @@ -48,7 +48,7 @@ public static void main(String[] args) { DaoInit.init(new ConnectionString("mongodb://localhost:27017/admini")); Context.accountId.set(1_000_000); - ApiInfo.ApiInfoKey apiInfoKey = new ApiInfo.ApiInfoKey(1709612022, "https://juiceshop.akto.io/rest/products/reviews", URLMethods.Method.PATCH); + ApiInfo.ApiInfoKey apiInfoKey = new ApiInfo.ApiInfoKey(1710156663, "https://juiceshop.akto.io/rest/products/reviews", URLMethods.Method.PATCH); SampleData sampleData = SampleDataDao.instance.findOne( Filters.and( @@ -77,6 +77,10 @@ public static void main(String[] args) { } } + public static void main1(String[] args) { + System.out.println(testContent); + } + public static String testContent = "id: REPORT_GENERATION_DOS\n" + "info:\n" + @@ -140,6 +144,7 @@ public static void main(String[] args) { " type: multiple\n" + " requests:\n" + " - req:\n" + + " - api: AVNEESH\n" + " - add_header:\n" + " dummy_Header_Key: \"dummyValue\"\n" + " - validate:\n" + @@ -149,6 +154,7 @@ public static void main(String[] args) { " - failure: exit\n" + "\n" + " - req:\n" + + " - api: AVNEESH\n" + " - add_header:\n" + " dummy_Header_Key: \"dummyValue\"\n" + " - validate:\n" + @@ -158,6 +164,7 @@ public static void main(String[] args) { " - failure: exit\n" + "\n" + " - req:\n" + + " - api: AVNEESH\n" + " - add_header:\n" + " dummy_Header_Key: \"dummyValue\"\n" + " - validate:\n" + @@ -167,6 +174,7 @@ public static void main(String[] args) { " - failure: exit\n" + "\n" + " - req:\n" + + " - api: AVNEESH\n" + " - add_header:\n" + " dummy_Header_Key: \"dummyValue\"\n" + " - validate:\n" + diff --git a/apps/dashboard/src/main/java/com/akto/utils/Build.java b/apps/dashboard/src/main/java/com/akto/utils/Build.java deleted file mode 100644 index b94387504c..0000000000 --- a/apps/dashboard/src/main/java/com/akto/utils/Build.java +++ /dev/null @@ -1,424 +0,0 @@ -package com.akto.utils; - -import com.akto.dao.DependencyFlowNodesDao; -import com.akto.dao.SampleDataDao; -import com.akto.dao.context.Context; -import com.akto.dto.*; -import com.akto.dto.dependency_flow.*; -import com.akto.dto.testing.TestingRunConfig; -import com.akto.dto.traffic.Key; -import com.akto.dto.traffic.SampleData; -import com.akto.dto.type.URLMethods; -import com.akto.log.LoggerMaker; -import com.akto.runtime.policies.AuthPolicy; -import com.akto.test_editor.execution.Operations; -import com.akto.testing.ApiExecutor; -import com.akto.util.Constants; -import com.akto.util.HttpRequestResponseUtils; -import com.akto.util.JSONUtils; - -import com.akto.util.modifier.SetValueModifier; -import com.fasterxml.jackson.core.JsonFactory; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.mongodb.client.model.Filters; -import joptsimple.internal.Strings; -import org.bson.conversions.Bson; - -import java.net.URI; -import java.util.*; - -import static com.akto.util.HttpRequestResponseUtils.FORM_URL_ENCODED_CONTENT_TYPE; -import static com.akto.util.HttpRequestResponseUtils.extractValuesFromPayload; - -public class Build { - - private Map parentToChildMap = new HashMap<>(); - - private static final LoggerMaker loggerMaker = new LoggerMaker(Build.class); - - public static void buildParentToChildMap(Collection nodes, Map parentToChildMap) { - for (Node node: nodes) { - if (node.getConnections() == null) continue; - for (Connection connection: node.getConnections().values()) { - String requestParam = connection.getParam(); - if (connection.getEdges() == null) continue; - for (Edge edge: connection.getEdges()) { - String responseParam = edge.getParam(); - ReverseNode reverseNode = new ReverseNode( - edge.getApiCollectionId(), edge.getUrl(), edge.getMethod(), new HashMap<>() - ); - - int key = reverseNode.hashCode(); - reverseNode = parentToChildMap.getOrDefault(key, reverseNode); - - Map reverseConnections = reverseNode.getReverseConnections(); - ReverseConnection reverseConnection = reverseConnections.getOrDefault(responseParam, new ReverseConnection(responseParam, new ArrayList<>())); - reverseConnection.getReverseEdges().add(new ReverseEdge(node.getApiCollectionId(), node.getUrl(), node.getMethod(), requestParam, edge.getCount(), connection.getIsUrlParam(), connection.getIsHeader())); - - reverseConnections.put(responseParam, reverseConnection); - parentToChildMap.put(key, reverseNode); - } - } - } - - } - - public static Map> buildLevelsToSampleDataMap(List nodes) { - - // divide them into levels - Map> levelsToSampleDataMap = new HashMap<>(); - for (Node node: nodes) { - int maxDepth = node.getMaxDepth(); - List list = levelsToSampleDataMap.getOrDefault(maxDepth, new ArrayList<>()); - int apiCollectionId = Integer.parseInt(node.getApiCollectionId()); - URLMethods.Method method = URLMethods.Method.valueOf(node.getMethod()); - list.add(new SampleData(new Key(apiCollectionId, node.getUrl(), method, 0,0,0), new ArrayList<>())); - levelsToSampleDataMap.put(maxDepth, list); - } - - return levelsToSampleDataMap; - } - - public static class RunResult { - - private ApiInfo.ApiInfoKey apiInfoKey; - private String currentMessage; - private String originalMessage; - private boolean success; - - public RunResult(ApiInfo.ApiInfoKey apiInfoKey, String currentMessage, String originalMessage, boolean success) { - this.apiInfoKey = apiInfoKey; - this.currentMessage = currentMessage; - this.originalMessage = originalMessage; - this.success = success; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - RunResult runResult = (RunResult) o; - return apiInfoKey.equals(runResult.apiInfoKey); - } - - @Override - public int hashCode() { - return Objects.hash(apiInfoKey); - } - - @Override - public String toString() { - return "RunResult{" + - "apiInfoKey=" + apiInfoKey + - ", currentMessage='" + currentMessage + '\'' + - ", originalMessage='" + originalMessage + '\'' + - ", success=" + success + - '}'; - } - - public ApiInfo.ApiInfoKey getApiInfoKey() { - return apiInfoKey; - } - - public void setApiInfoKey(ApiInfo.ApiInfoKey apiInfoKey) { - this.apiInfoKey = apiInfoKey; - } - - public String getCurrentMessage() { - return currentMessage; - } - - public void setCurrentMessage(String currentMessage) { - this.currentMessage = currentMessage; - } - - public String getOriginalMessage() { - return originalMessage; - } - - public void setOriginalMessage(String originalMessage) { - this.originalMessage = originalMessage; - } - - public boolean isSuccess() { - return success; - } - - public boolean getIsSuccess() { - return success; - } - - public void setIsSuccess(boolean success) { - this.success = success; - } - - - } - - Set apisReplayedSet = new HashSet<>(); - public static List runPerLevel(List sdList, Map modifyHostDetailMap, Map replaceDetailsMap, Map parentToChildMap, Set apisReplayedSet) { - List runResults = new ArrayList<>(); - for (SampleData sampleData: sdList) { - Key id = sampleData.getId(); - try { - List samples = sampleData.getSamples(); - if (samples.isEmpty()) continue;; - - String sample = samples.get(0); - OriginalHttpRequest request = new OriginalHttpRequest(); - request.buildFromSampleMessage(sample); - String newHost = findNewHost(request, modifyHostDetailMap); - - OriginalHttpResponse originalHttpResponse = new OriginalHttpResponse(); - originalHttpResponse.buildFromSampleMessage(sample); - - // do modifications - ReplaceDetail replaceDetail = replaceDetailsMap.get(Objects.hash(id.getApiCollectionId(), id.getUrl(), id.getMethod().name())); - modifyRequest(request, replaceDetail); - - TestingRunConfig testingRunConfig = new TestingRunConfig(0, new HashMap<>(), new ArrayList<>(), null,newHost, null); - - OriginalHttpResponse response = null; - try { - response = ApiExecutor.sendRequest(request,false, testingRunConfig, false, new ArrayList<>()); - apisReplayedSet.add(new ApiInfo.ApiInfoKey(id.getApiCollectionId(), id.getUrl(), id.getMethod())); - request.getHeaders().remove(Constants.AKTO_IGNORE_FLAG); - ReverseNode parentToChildNode = parentToChildMap.get(Objects.hash(id.getApiCollectionId()+"", id.getUrl(), id.getMethod().name())); - fillReplaceDetailsMap(parentToChildNode, response, replaceDetailsMap); - RawApi rawApi = new RawApi(request, response, ""); - rawApi.fillOriginalMessage(Context.accountId.get(), Context.now(), "", "HAR"); - RunResult runResult = new RunResult( - new ApiInfo.ApiInfoKey(id.getApiCollectionId(), id.getUrl(), id.getMethod()), - rawApi.getOriginalMessage(), - sample, - rawApi.getResponse().getStatusCode() == originalHttpResponse.getStatusCode() - ); - runResults.add(runResult); - } catch (Exception e) { - loggerMaker.errorAndAddToDb(e, "error while sending request in invoke dependency graph" + id.getUrl(), LoggerMaker.LogDb.DASHBOARD); - } - } catch (Exception e) { - loggerMaker.errorAndAddToDb(e, "error while running runPerLevel for " + id.getUrl(), LoggerMaker.LogDb.DASHBOARD); - } - - } - - return runResults; - } - - public static String findNewHost(OriginalHttpRequest request, Map modifyHostDetailMap) { - try { - String url = request.getFullUrlIncludingDomain(); - URI uri = new URI(url); - String currentHost = uri.getHost(); - ModifyHostDetail modifyHostDetail = modifyHostDetailMap.get(currentHost); - if (modifyHostDetail == null) { - if (uri.getPort() != -1) currentHost += ":" + uri.getPort(); - modifyHostDetail = modifyHostDetailMap.get(currentHost); - } - - if (modifyHostDetail == null) return null; - - String newHost = modifyHostDetail.getNewHost(); - if (newHost == null) return null; - if (newHost.startsWith("http")) { - return newHost; - } else { - return uri.getScheme() + "://" + newHost; - } - } catch (Exception ignored) { - } - return null; - } - - - public List run(List apiCollectionsIds, List modifyHostDetails, Map replaceDetailsMap) { - if (replaceDetailsMap == null) replaceDetailsMap = new HashMap<>(); - if (modifyHostDetails == null) modifyHostDetails = new ArrayList<>(); - - Map modifyHostDetailMap = new HashMap<>(); - for (ModifyHostDetail modifyHostDetail: modifyHostDetails) { - modifyHostDetailMap.put(modifyHostDetail.getCurrentHost(), modifyHostDetail); - } - - - List nodes = DependencyFlowNodesDao.instance.findNodesForCollectionIds(apiCollectionsIds,false,0, 10_000); - buildParentToChildMap(nodes, parentToChildMap); - Map> levelsToSampleDataMap = buildLevelsToSampleDataMap(nodes); - - List runResults = new ArrayList<>(); - // loop over levels and make requests - for (int level: levelsToSampleDataMap.keySet()) { - List sdList =levelsToSampleDataMap.get(level); - sdList = fillSdList(sdList); - if (sdList.isEmpty()) continue; - - loggerMaker.infoAndAddToDb("Running level: " + level, LoggerMaker.LogDb.DASHBOARD); - try { - List runResultsPerLevel = runPerLevel(sdList, modifyHostDetailMap, replaceDetailsMap, parentToChildMap, apisReplayedSet); - runResults.addAll(runResultsPerLevel); - loggerMaker.infoAndAddToDb("Finished running level " + level, LoggerMaker.LogDb.DASHBOARD); - } catch (Exception e) { - e.printStackTrace(); - loggerMaker.errorAndAddToDb(e, "Error while running for level " + level , LoggerMaker.LogDb.DASHBOARD); - } - } - - loggerMaker.infoAndAddToDb("Running independent APIs", LoggerMaker.LogDb.DASHBOARD); - int skip = 0; - int limit = 1000; - while (true) { - List all = SampleDataDao.instance.findAll(Filters.in("_id.apiCollectionId", apiCollectionsIds), skip,limit, null); - if (all.isEmpty()) break; - List filtered = new ArrayList<>(); - for (SampleData sampleData: all) { - Key key = sampleData.getId(); - if (apisReplayedSet.contains(new ApiInfo.ApiInfoKey(key.getApiCollectionId(), key.getUrl(), key.getMethod()))) continue; - filtered.add(sampleData); - } - List runResultsAll = runPerLevel(filtered, modifyHostDetailMap, replaceDetailsMap, parentToChildMap, apisReplayedSet); - runResults.addAll(runResultsAll); - skip += limit; - if (all.size() < limit) break; - } - loggerMaker.infoAndAddToDb("Finished running independent APIs", LoggerMaker.LogDb.DASHBOARD); - - return runResults; - } - - public static void modifyRequest(OriginalHttpRequest request, ReplaceDetail replaceDetail) { - RawApi rawApi = new RawApi(request, null, null); - - if (replaceDetail != null) { - List kvPairs = replaceDetail.getKvPairs(); - if (kvPairs != null && !kvPairs.isEmpty()) { - for (KVPair kvPair: kvPairs) { - if (kvPair.isHeader()) { - String value = kvPair.getValue()+""; - String headerValue = rawApi.getRequest().findHeaderValueIncludingInCookie(kvPair.getKey()); - if (headerValue == null) continue; - String[] headerValueSplit = headerValue.split(" "); - if (headerValueSplit.length == 2) { - String prefix = headerValueSplit[0]; - if (Arrays.asList("bearer", "basic").contains(prefix.toLowerCase())) { - value = prefix + " " + value; - } - } - Operations.modifyHeader(rawApi, kvPair.getKey(), value); - } else if (kvPair.isUrlParam()) { - String url = request.getUrl(); - String[] urlSplit = url.split("/"); - int position = Integer.parseInt(kvPair.getKey()); - urlSplit[position] = kvPair.getValue()+""; - String newUrl = Strings.join(urlSplit, "/"); - request.setUrl(newUrl); - } else { - Map store = new HashMap<>(); - store.put(kvPair.getKey(), kvPair.getValue()); - SetValueModifier setValueModifier = new SetValueModifier(store); - - Set values = new HashSet<>(); - values.add(kvPair.getKey()); - String modifiedBody = JSONUtils.modify(rawApi.getRequest().getJsonRequestBody(), values, setValueModifier); - String contentType = rawApi.getRequest().findContentType(); - if (contentType != null && contentType.equals(FORM_URL_ENCODED_CONTENT_TYPE)) { - modifiedBody = HttpRequestResponseUtils.jsonToFormUrlEncoded(modifiedBody); - } - rawApi.getRequest().setBody(modifiedBody); - - Operations.modifyQueryParam(rawApi, kvPair.getKey(), kvPair.getValue()); - } - } - } - } - - } - - public static List fillSdList(List sdList) { - if (sdList == null || sdList.isEmpty()) return new ArrayList<>(); - - List filters = new ArrayList<>(); - for (SampleData sampleData: sdList) { - // todo: batch for bigger lists - Key id = sampleData.getId(); - filters.add(Filters.and( - Filters.eq("_id.apiCollectionId", id.getApiCollectionId()), - Filters.eq("_id.url", id.getUrl()), - Filters.eq("_id.method", id.getMethod().name()) - )); - } - return SampleDataDao.instance.findAll(Filters.or(filters)); - } - - - static ObjectMapper mapper = new ObjectMapper(); - static JsonFactory factory = mapper.getFactory(); - public static void fillReplaceDetailsMap(ReverseNode reverseNode, OriginalHttpResponse response, Map replaceDetailsMap) { - if (reverseNode == null) return; - - Map deltaReplaceDetailsMap = new HashMap<>(); - - Map> valuesMap = getValuesMap(response); - - Map connections = reverseNode.getReverseConnections(); - for (ReverseConnection reverseConnection: connections.values()) { - String param = reverseConnection.getParam(); - Set values = valuesMap.get(param); - Object value = values != null && values.size() > 0 ? values.toArray()[0] : null; // todo: - if (value == null) continue; - - for (ReverseEdge reverseEdge: reverseConnection.getReverseEdges()) { - int apiCollectionId = Integer.parseInt(reverseEdge.getApiCollectionId()); - Integer id = Objects.hash(apiCollectionId, reverseEdge.getUrl(), reverseEdge.getMethod()); - - ReplaceDetail deltaReplaceDetail = deltaReplaceDetailsMap.get(id); - if (deltaReplaceDetail == null) { - deltaReplaceDetail = new ReplaceDetail(apiCollectionId, reverseEdge.getUrl(), reverseEdge.getMethod(), new ArrayList<>()); - deltaReplaceDetailsMap.put(id, deltaReplaceDetail); - } - - KVPair.KVType type = value instanceof Integer ? KVPair.KVType.INTEGER : KVPair.KVType.STRING; - KVPair kvPair = new KVPair(reverseEdge.getParam(), value.toString(), reverseEdge.getIsHeader(), reverseEdge.isUrlParam(), type); - deltaReplaceDetail.addIfNotExist(kvPair); - } - } - - for (Integer key: deltaReplaceDetailsMap.keySet()) { - ReplaceDetail replaceDetail = replaceDetailsMap.get(key); - ReplaceDetail deltaReplaceDetail = deltaReplaceDetailsMap.get(key); - if (replaceDetail == null) { - replaceDetail = deltaReplaceDetail; - } else { - replaceDetail.addIfNotExist(deltaReplaceDetail.getKvPairs()); - } - - replaceDetailsMap.put(key, replaceDetail); - } - - } - - public static Map> getValuesMap(OriginalHttpResponse response) { - String respPayload = response.getBody(); - Map> valuesMap = extractValuesFromPayload(respPayload); - - Map> responseHeaders = response.getHeaders(); - for (String headerKey: responseHeaders.keySet()) { - List values = responseHeaders.get(headerKey); - if (values == null) continue; - - if (headerKey.equalsIgnoreCase("set-cookie")) { - Map cookieMap = AuthPolicy.parseCookie(values); - for (String cookieKey : cookieMap.keySet()) { - String cookieVal = cookieMap.get(cookieKey); - valuesMap.put(cookieKey, new HashSet<>(Collections.singletonList(cookieVal))); - } - } else { - valuesMap.put(headerKey, new HashSet<>(values)); - } - - } - - return valuesMap; - } - -} diff --git a/apps/dashboard/src/main/java/com/akto/utils/Memory.java b/apps/dashboard/src/main/java/com/akto/utils/Memory.java deleted file mode 100644 index 8d48366c95..0000000000 --- a/apps/dashboard/src/main/java/com/akto/utils/Memory.java +++ /dev/null @@ -1,235 +0,0 @@ -package com.akto.utils; - -import com.akto.DaoInit; -import com.akto.dao.SampleDataDao; -import com.akto.dao.context.Context; -import com.akto.dto.ApiInfo; -import com.akto.dto.OriginalHttpRequest; -import com.akto.dto.OriginalHttpResponse; -import com.akto.dto.RawApi; -import com.akto.dto.dependency_flow.*; -import com.akto.dto.testing.TestingRunConfig; -import com.akto.dto.traffic.Key; -import com.akto.dto.traffic.SampleData; -import com.akto.dto.type.URLMethods; -import com.akto.testing.ApiExecutor; -import com.akto.util.Constants; -import com.mongodb.ConnectionString; -import com.mongodb.client.model.Filters; -import org.bson.conversions.Bson; - -import java.util.*; - -import static com.akto.utils.Build.*; - -public class Memory { - - Map resultMap = new HashMap<>(); - private final Map parentToChildMap = new HashMap<>(); - private final Map nodesMap = new HashMap<>(); - - Map sampleDataMap = new HashMap<>(); - - private Map replaceDetailsMap = new HashMap<>(); - - public static void main(String[] args) throws Exception { - DaoInit.init(new ConnectionString("mongodb://localhost:27017/admini")); - Context.accountId.set(1_000_000); - -// https://juiceshop.akto.io/rest/user/login - List apiInfoKeys = new ArrayList<>(); - apiInfoKeys.add(new ApiInfo.ApiInfoKey(1709612022, "https://juiceshop.akto.io/rest/products/reviews", URLMethods.Method.PATCH)); - apiInfoKeys.add(new ApiInfo.ApiInfoKey(1709612022, "https://juiceshop.akto.io/rest/user/login", URLMethods.Method.POST)); - - Memory memory = new Memory(); - memory.build(apiInfoKeys, new HashMap<>()); - - OriginalHttpRequest req1 = memory.run(1709612022, "https://juiceshop.akto.io/rest/user/login", URLMethods.Method.POST.name()); - System.out.println("new"); - OriginalHttpResponse resp1 = ApiExecutor.sendRequest(req1, true, null, false, new ArrayList<>()); - memory.fillResponse(req1, resp1, 1709612022, "https://juiceshop.akto.io/rest/user/login", URLMethods.Method.POST.name()); - memory.reset(1709612022, "https://juiceshop.akto.io/rest/user/login", URLMethods.Method.POST.name()); - System.out.println("done"); - - - OriginalHttpRequest req2 = memory.run(1709612022, "https://juiceshop.akto.io/rest/products/reviews", URLMethods.Method.PATCH.name()); - System.out.println("new"); - OriginalHttpResponse resp2 = ApiExecutor.sendRequest(req2, true, null, false, new ArrayList<>()); - memory.fillResponse(req2, resp2, 1709612022, "https://juiceshop.akto.io/rest/products/reviews", URLMethods.Method.PATCH.name()); - memory.reset(1709612022, "https://juiceshop.akto.io/rest/products/reviews", URLMethods.Method.PATCH.name()); - System.out.println("done"); - - - OriginalHttpRequest req3 = memory.run(1709612022, "https://juiceshop.akto.io/rest/user/login", URLMethods.Method.POST.name()); - System.out.println("new"); - OriginalHttpResponse resp3 = ApiExecutor.sendRequest(req3, true, null, false, new ArrayList<>()); - memory.fillResponse(req3, resp3, 1709612022, "https://juiceshop.akto.io/rest/user/login", URLMethods.Method.POST.name()); - memory.reset(1709612022, "https://juiceshop.akto.io/rest/user/login", URLMethods.Method.POST.name()); - System.out.println("done"); - - - OriginalHttpRequest req4 = memory.run(1709612022, "https://juiceshop.akto.io/rest/products/reviews", URLMethods.Method.PATCH.name()); - System.out.println("new"); - OriginalHttpResponse resp4 = ApiExecutor.sendRequest(req4, true, null, false, new ArrayList<>()); - memory.fillResponse(req4, resp4, 1709612022, "https://juiceshop.akto.io/rest/products/reviews", URLMethods.Method.PATCH.name()); - memory.reset(1709612022, "https://juiceshop.akto.io/rest/products/reviews", URLMethods.Method.PATCH.name()); - System.out.println("done"); - - - } - - public void build(List apiInfoKeys, Map replaceDetailsMap) { - this.replaceDetailsMap = replaceDetailsMap; - - // find all parent APIs - TreeHelper treeHelper = new TreeHelper(); - for (ApiInfo.ApiInfoKey apiInfoKey: apiInfoKeys) { - treeHelper.buildTree(apiInfoKey.getApiCollectionId()+"", apiInfoKey.getUrl(), apiInfoKey.getMethod().name()); - } - Collection nodes = treeHelper.result.values(); - List filters = new ArrayList<>(); - for (Node node: nodes) { - nodesMap.put(node.hashCode(), node); - filters.add(Filters.and( - Filters.eq("_id.apiCollectionId", Integer.parseInt(node.getApiCollectionId())), - Filters.eq("_id.url", node.getUrl()), - Filters.eq("_id.method", node.getMethod()) - )); - } - - // fetch sample data - List sdList = SampleDataDao.instance.findAll(Filters.or(filters)); - for (SampleData sampleData: sdList) { - Key id = sampleData.getId(); - sampleDataMap.put(Objects.hash(id.getApiCollectionId(), id.getUrl(), id.getMethod().name()), sampleData); - } - - buildParentToChildMap(nodes, parentToChildMap); - } - - - public OriginalHttpRequest run(int apiCollectionId, String url, String method) { - int hash = Objects.hash(apiCollectionId+"", url, method); - if (resultMap.get(hash) != null) return resultMap.get(hash).getRequest(); - - // todo: optimise this.. no need to make db calls everytime - TreeHelper treeHelper = new TreeHelper(); - treeHelper.buildTree(apiCollectionId+"", url, method); - List nodes = new ArrayList<>(treeHelper.result.values()); - - nodes.sort(Comparator.comparingInt(Node::getMaxDepth)); - - List sampleDataList = new ArrayList<>(); - for (Node node: nodes) { - Integer nodeHash = Objects.hash(Integer.parseInt(node.getApiCollectionId()), node.getUrl(), node.getMethod()); - sampleDataList.add(sampleDataMap.get(nodeHash)); - } - - return execute(sampleDataList); - } - - - public OriginalHttpRequest execute(List sdList) { - int idx = 0; - for (SampleData sampleData: sdList) { - idx++; - boolean isFinal = sdList.size() == idx; // todo: find a better way - Key id = sampleData.getId(); - int hash = Objects.hash(id.getApiCollectionId()+"", id.getUrl(), id.getMethod().name()); - if (resultMap.containsKey(hash)) continue; - try { - List samples = sampleData.getSamples(); - if (samples.isEmpty()) continue;; - - String sample = samples.get(0); - OriginalHttpRequest request = new OriginalHttpRequest(); - request.buildFromSampleMessage(sample); - // todo: String newHost = findNewHost(request, modifyHostDetailMap); - String newHost = null; - - OriginalHttpResponse originalHttpResponse = new OriginalHttpResponse(); - originalHttpResponse.buildFromSampleMessage(sample); - - // do modifications - Node node = nodesMap.get(hash); - ReplaceDetail finalReplaceDetail = getReplaceDetail(node); - modifyRequest(request, finalReplaceDetail); - - if (isFinal) return request; - - TestingRunConfig testingRunConfig = new TestingRunConfig(0, new HashMap<>(), new ArrayList<>(), null,newHost, null); - - OriginalHttpResponse response = null; - try { - response = ApiExecutor.sendRequest(request,false, testingRunConfig, false, new ArrayList<>()); - request.getHeaders().remove(Constants.AKTO_IGNORE_FLAG); - RawApi rawApi = new RawApi(request, response, ""); - resultMap.put(hash, rawApi); - } catch (Exception e) { - e.printStackTrace(); - } - } catch (Exception e) { - e.printStackTrace(); - } - - } - - return null; - } - - - public void fillResponse(OriginalHttpRequest request, OriginalHttpResponse response, int apiCollectionId, String url, String method) { - int hash = Objects.hash(apiCollectionId+"", url, method); - RawApi rawApi = new RawApi(request, response, ""); - resultMap.put(hash, rawApi); - } - - public void reset(int apiCollectionId, String url, String method) { - // find all children - int hash = Objects.hash(apiCollectionId+"", url, method); - ReverseNode reverseNode = parentToChildMap.get(hash); - if (reverseNode == null) return; - - for (ReverseConnection reverseConnection: reverseNode.getReverseConnections().values()) { - for (ReverseEdge reverseEdge: reverseConnection.getReverseEdges()) { - int childApiCollectionId = Integer.parseInt(reverseEdge.getApiCollectionId()); - String childUrl = reverseEdge.getUrl(); - String childMethod = reverseEdge.getMethod(); - - int childHash = Objects.hash(childApiCollectionId+"", childUrl, childMethod); - resultMap.remove(childHash); - reset(childApiCollectionId, childUrl, childMethod); - } - } - } - - - public ReplaceDetail getReplaceDetail(Node node) { - ReplaceDetail replaceDetail = new ReplaceDetail(Integer.parseInt(node.getApiCollectionId()), node.getUrl(), node.getMethod(), new ArrayList<>()); - Map connections = node.getConnections(); - for (Connection connection: connections.values()) { - String requestParam = connection.getParam(); - List edges = connection.getEdges(); - if (edges.isEmpty()) continue; - Edge edge = edges.get(0); - String responseParam = edge.getParam(); - String parentApiCollectionId = edge.getApiCollectionId(); - String parentUrl = edge.getUrl(); - String parentMethod = edge.getMethod(); - int parentHash = Objects.hash(parentApiCollectionId, parentUrl, parentMethod); - RawApi rawApi = resultMap.get(parentHash); - if (rawApi == null) continue; - Map> valuesMap = getValuesMap(rawApi.getResponse()); - Set values = valuesMap.get(responseParam); - Object value = values != null && values.size() > 0 ? values.toArray()[0] : null; // todo: - if (value == null) continue; - - KVPair.KVType type = value instanceof Integer ? KVPair.KVType.INTEGER : KVPair.KVType.STRING; - KVPair kvPair = new KVPair(requestParam, value.toString(), connection.getIsHeader(), connection.getIsUrlParam(), type); - replaceDetail.addIfNotExist(kvPair); - } - - return replaceDetail; - } - -} diff --git a/apps/dashboard/src/main/java/com/akto/utils/Utils.java b/apps/dashboard/src/main/java/com/akto/utils/Utils.java index caa76cdf88..42c1d33e34 100644 --- a/apps/dashboard/src/main/java/com/akto/utils/Utils.java +++ b/apps/dashboard/src/main/java/com/akto/utils/Utils.java @@ -475,13 +475,13 @@ public static void pushDataToKafka(int apiCollectionId, String topic, List() ); @@ -46,40 +46,40 @@ public static void main(String[] args) throws Exception { // https://juiceshop.akto.io/rest/user/login List apiInfoKeys = new ArrayList<>(); - apiInfoKeys.add(new ApiInfo.ApiInfoKey(1709612022, "https://juiceshop.akto.io/rest/products/reviews", URLMethods.Method.PATCH)); - apiInfoKeys.add(new ApiInfo.ApiInfoKey(1709612022, "https://juiceshop.akto.io/rest/user/login", URLMethods.Method.POST)); + apiInfoKeys.add(new ApiInfo.ApiInfoKey(1710156663, "https://juiceshop.akto.io/rest/products/reviews", URLMethods.Method.PATCH)); + apiInfoKeys.add(new ApiInfo.ApiInfoKey(1710156663, "https://juiceshop.akto.io/rest/user/login", URLMethods.Method.POST)); Memory memory = new Memory(apiInfoKeys, new HashMap<>()); - OriginalHttpRequest req1 = memory.run(1709612022, "https://juiceshop.akto.io/rest/user/login", URLMethods.Method.POST.name()); + OriginalHttpRequest req1 = memory.run(1710156663, "https://juiceshop.akto.io/rest/user/login", URLMethods.Method.POST.name()); System.out.println("new"); OriginalHttpResponse resp1 = ApiExecutor.sendRequest(req1, true, null, false, new ArrayList<>()); - memory.fillResponse(req1, resp1, 1709612022, "https://juiceshop.akto.io/rest/user/login", URLMethods.Method.POST.name()); - memory.reset(1709612022, "https://juiceshop.akto.io/rest/user/login", URLMethods.Method.POST.name()); + memory.fillResponse(req1, resp1, 1710156663, "https://juiceshop.akto.io/rest/user/login", URLMethods.Method.POST.name()); + memory.reset(1710156663, "https://juiceshop.akto.io/rest/user/login", URLMethods.Method.POST.name()); System.out.println("done"); - OriginalHttpRequest req2 = memory.run(1709612022, "https://juiceshop.akto.io/rest/products/reviews", URLMethods.Method.PATCH.name()); + OriginalHttpRequest req2 = memory.run(1710156663, "https://juiceshop.akto.io/rest/products/reviews", URLMethods.Method.PATCH.name()); System.out.println("new"); OriginalHttpResponse resp2 = ApiExecutor.sendRequest(req2, true, null, false, new ArrayList<>()); - memory.fillResponse(req2, resp2, 1709612022, "https://juiceshop.akto.io/rest/products/reviews", URLMethods.Method.PATCH.name()); - memory.reset(1709612022, "https://juiceshop.akto.io/rest/products/reviews", URLMethods.Method.PATCH.name()); + memory.fillResponse(req2, resp2, 1710156663, "https://juiceshop.akto.io/rest/products/reviews", URLMethods.Method.PATCH.name()); + memory.reset(1710156663, "https://juiceshop.akto.io/rest/products/reviews", URLMethods.Method.PATCH.name()); System.out.println("done"); - OriginalHttpRequest req3 = memory.run(1709612022, "https://juiceshop.akto.io/rest/user/login", URLMethods.Method.POST.name()); + OriginalHttpRequest req3 = memory.run(1710156663, "https://juiceshop.akto.io/rest/user/login", URLMethods.Method.POST.name()); System.out.println("new"); OriginalHttpResponse resp3 = ApiExecutor.sendRequest(req3, true, null, false, new ArrayList<>()); - memory.fillResponse(req3, resp3, 1709612022, "https://juiceshop.akto.io/rest/user/login", URLMethods.Method.POST.name()); - memory.reset(1709612022, "https://juiceshop.akto.io/rest/user/login", URLMethods.Method.POST.name()); + memory.fillResponse(req3, resp3, 1710156663, "https://juiceshop.akto.io/rest/user/login", URLMethods.Method.POST.name()); + memory.reset(1710156663, "https://juiceshop.akto.io/rest/user/login", URLMethods.Method.POST.name()); System.out.println("done"); - OriginalHttpRequest req4 = memory.run(1709612022, "https://juiceshop.akto.io/rest/products/reviews", URLMethods.Method.PATCH.name()); + OriginalHttpRequest req4 = memory.run(1710156663, "https://juiceshop.akto.io/rest/products/reviews", URLMethods.Method.PATCH.name()); System.out.println("new"); OriginalHttpResponse resp4 = ApiExecutor.sendRequest(req4, true, null, false, new ArrayList<>()); - memory.fillResponse(req4, resp4, 1709612022, "https://juiceshop.akto.io/rest/products/reviews", URLMethods.Method.PATCH.name()); - memory.reset(1709612022, "https://juiceshop.akto.io/rest/products/reviews", URLMethods.Method.PATCH.name()); + memory.fillResponse(req4, resp4, 1710156663, "https://juiceshop.akto.io/rest/products/reviews", URLMethods.Method.PATCH.name()); + memory.reset(1710156663, "https://juiceshop.akto.io/rest/products/reviews", URLMethods.Method.PATCH.name()); System.out.println("done"); diff --git a/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/YamlNodeExecutor.java b/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/YamlNodeExecutor.java index dafb84245e..f015231e39 100644 --- a/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/YamlNodeExecutor.java +++ b/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/YamlNodeExecutor.java @@ -47,10 +47,8 @@ public class YamlNodeExecutor extends NodeExecutor { private static final Gson gson = new Gson(); - static int counter = 0; public NodeResult processNode(Node node, Map varMap, Boolean allowAllStatusCodes, boolean debug, List testLogs) { - counter++; List testErrors = new ArrayList<>(); YamlNodeDetails yamlNodeDetails = (YamlNodeDetails) node.getWorkflowNodeDetails(); @@ -61,12 +59,6 @@ public NodeResult processNode(Node node, Map varMap, Boolean all RawApi rawApi = yamlNodeDetails.getRawApi(); RawApi sampleRawApi = rawApi.copy(); - ApiInfo.ApiInfoKey apiInfoKey = counter % 2 != 0 ? new ApiInfo.ApiInfoKey(1709612022, "https://juiceshop.akto.io/rest/user/login", URLMethods.Method.POST) : new ApiInfo.ApiInfoKey(1709612022, "https://juiceshop.akto.io/rest/products/reviews", URLMethods.Method.PATCH); - System.out.println("*******"); - System.out.println(apiInfoKey); - - OriginalHttpRequest newRequest = Memory.memory.run(apiInfoKey.getApiCollectionId(), apiInfoKey.getUrl(), apiInfoKey.getMethod().name()); - rawApi.setRequest(newRequest); List rawApis = new ArrayList<>(); rawApis.add(rawApi.copy()); @@ -74,8 +66,25 @@ public NodeResult processNode(Node node, Map varMap, Boolean all Executor executor = new Executor(); ExecutorNode executorNode = yamlNodeDetails.getExecutorNode(); FilterNode validatorNode = yamlNodeDetails.getValidatorNode(); + List childNodes = executorNode.getChildNodes(); - for (ExecutorNode execNode: executorNode.getChildNodes()) { + String api = null; + ExecutorNode firstChildNode = childNodes.get(0); // todo check for length + if (firstChildNode.getOperationType().equals("API")) { + api = firstChildNode.getValues().toString(); + childNodes.remove(0); + } + + ApiInfo.ApiInfoKey apiInfoKey = new ApiInfo.ApiInfoKey(1710156663, "https://juiceshop.akto.io/rest/user/login", URLMethods.Method.POST); + + OriginalHttpRequest newRequest = Memory.memory.run(apiInfoKey.getApiCollectionId(), apiInfoKey.getUrl(), apiInfoKey.getMethod().name()); + rawApi.setRequest(newRequest); + + if (api != null) { +// ApiInfo.ApiInfoKey apiInfoKey = counter % 2 != 0 ? new ApiInfo.ApiInfoKey(1710156663, "https://juiceshop.akto.io/rest/user/login", URLMethods.Method.POST) : new ApiInfo.ApiInfoKey(1710156663, "https://juiceshop.akto.io/rest/products/reviews", URLMethods.Method.PATCH); + } + + for (ExecutorNode execNode: childNodes) { if (execNode.getNodeType().equalsIgnoreCase(TestEditorEnums.ValidateExecutorDataOperands.Validate.toString())) { validatorNode = (FilterNode) execNode.getChildNodes().get(0).getValues(); } diff --git a/libs/dao/src/main/java/com/akto/dao/test_editor/TestEditorEnums.java b/libs/dao/src/main/java/com/akto/dao/test_editor/TestEditorEnums.java index 098aa53af1..c8e1ab2407 100644 --- a/libs/dao/src/main/java/com/akto/dao/test_editor/TestEditorEnums.java +++ b/libs/dao/src/main/java/com/akto/dao/test_editor/TestEditorEnums.java @@ -194,6 +194,7 @@ public enum RequestParentOperand { } public enum TerminalExecutorDataOperands { + API, DELETE_HEADER, DELETE_BODY_PARAM, DELETE_QUERY_PARAM, From 21af36942b6b7e79b4e54e4d09fcfa1c4f13cae9 Mon Sep 17 00:00:00 2001 From: Avneesh Hota Date: Fri, 15 Mar 2024 18:45:13 +0530 Subject: [PATCH 03/16] added workflow_filter in testing --- .../main/java/com/akto/action/HarAction.java | 77 +++++++++---------- .../akto/test_editor/execution/Executor.java | 10 ++- .../akto/test_editor/execution/Memory.java | 56 +------------- .../com/akto/testing/ApiWorkflowExecutor.java | 7 +- .../java/com/akto/testing/TestExecutor.java | 10 +-- .../ApiNodeExecutor.java | 4 +- .../ConditionalGraphExecutor.java | 8 +- .../workflow_node_executor/GraphExecutor.java | 5 +- .../LinearGraphExecutor.java | 6 +- .../workflow_node_executor/NodeExecutor.java | 4 +- .../testing/workflow_node_executor/Utils.java | 16 ++-- .../YamlNodeExecutor.java | 29 ++++--- .../yaml_tests/SecurityTestTemplate.java | 47 ++++++++--- .../testing/yaml_tests/YamlTestTemplate.java | 53 +++++++++++-- .../dao/test_editor/TestConfigYamlParser.java | 35 ++++++--- .../com/akto/dto/test_editor/TestConfig.java | 15 +++- 16 files changed, 212 insertions(+), 170 deletions(-) diff --git a/apps/dashboard/src/main/java/com/akto/action/HarAction.java b/apps/dashboard/src/main/java/com/akto/action/HarAction.java index 802a602ff9..2b8105e5b3 100644 --- a/apps/dashboard/src/main/java/com/akto/action/HarAction.java +++ b/apps/dashboard/src/main/java/com/akto/action/HarAction.java @@ -83,35 +83,12 @@ public static void main1(String[] args) { public static String testContent = "id: REPORT_GENERATION_DOS\n" + + "\n" + "info:\n" + " name: \"Denial of Service Test on Report Generation Endpoint\"\n" + - " description: \"A Denial of Service (DoS) test on a report generation endpoint involves \n" + - " overwhelming the system with a high volume of requests to assess its \n" + - " resilience under heavy load. By bombarding the endpoint with \n" + - " numerous simultaneous requests for report generation, testers \n" + - " evaluate how well the system handles the load and whether it \n" + - " remains responsive. This testing helps identify potential bottlenecks \n" + - " or vulnerabilities in the report generation process, enabling \n" + - " proactive measures to fortify the system's defenses against \n" + - " DoS attacks targeting this endpoint.\"\n" + - " details: \"In this test, the report generation endpoint is bombarded with an \n" + - " excessive number of requests, aiming to simulate real-world peak \n" + - " loads and stress the system. Testers assess how the endpoint \n" + - " responds to this influx of requests, evaluating its ability \n" + - " to maintain responsiveness and generate reports efficiently. \n" + - " Through this process, potential weaknesses in scalability \n" + - " and performance are identified, enabling organizations to \n" + - " fortify their systems against Denial of Service (DoS) attacks \n" + - " on report generation functionalities.\"\n" + - " impact: \"A successful Denial of Service (DoS) test on a report generation endpoint \n" + - " can have significant consequences. It may lead to system slowdowns, \n" + - " unavailability, or crashes, hindering users' access to vital reports \n" + - " and disrupting business operations. Additionally, prolonged \n" + - " service disruptions can tarnish the organization's reputation, \n" + - " eroding user trust and potentially resulting in financial \n" + - " losses. Identifying and addressing vulnerabilities in the \n" + - " report generation process is crucial for maintaining system \n" + - " reliability and resilience against DoS attacks.\"\n" + + " description: \"A Denial of Service (DoS) test\"\n" + + " details: \"In this test.\"\n" + + " impact: \"A\"\n" + " category:\n" + " name: RL\n" + " shortName: Lack of Resources & Rate Limiting\n" + @@ -129,13 +106,28 @@ public static void main1(String[] args) { " cve:\n" + " - CVE-2023-4647\n" + " - CVE-2023-38254\n" + + "\n" + + "workflow_selection_filters:\n" + + " API_1:\n" + + " response_code:\n" + + " gte: 200\n" + + " lt: 300\n" + + " url:\n" + + " contains_either:\n" + + " - rest/user/login\n" + + " API_2:\n" + + " response_code:\n" + + " gte: 200\n" + + " lt: 300\n" + + " url:\n" + + " contains_either:\n" + + " - rest/products/reviews\n" + + "\n" + "api_selection_filters:\n" + " response_code:\n" + " gte: 200\n" + " lt: 300\n" + - " url:\n" + - " contains_either:\n" + - " - a\n" + + "\n" + "wordLists:\n" + " dummyHeaders:\n" + " - a\n" + @@ -144,44 +136,45 @@ public static void main1(String[] args) { " type: multiple\n" + " requests:\n" + " - req:\n" + - " - api: AVNEESH\n" + + " - api: API_1\n" + " - add_header:\n" + - " dummy_Header_Key: \"dummyValue\"\n" + + " dummy_Header_Key1: \"dummyValue\"\n" + " - validate:\n" + " percentage_match:\n" + " gte: 90\n" + " - success: x2\n" + - " - failure: exit\n" + + " - failure: x2\n" + "\n" + " - req:\n" + - " - api: AVNEESH\n" + + " - api: API_2\n" + " - add_header:\n" + - " dummy_Header_Key: \"dummyValue\"\n" + + " dummy_Header_Key2: \"dummyValue\"\n" + " - validate:\n" + " percentage_match:\n" + " gte: 90\n" + " - success: x3\n" + - " - failure: exit\n" + + " - failure: x3\n" + "\n" + " - req:\n" + - " - api: AVNEESH\n" + + " - api: API_1\n" + " - add_header:\n" + - " dummy_Header_Key: \"dummyValue\"\n" + + " dummy_Header_Key3: \"dummyValue\"\n" + " - validate:\n" + " percentage_match:\n" + " gte: 90\n" + " - success: x4\n" + - " - failure: exit\n" + + " - failure: x4\n" + "\n" + " - req:\n" + - " - api: AVNEESH\n" + + " - api: API_2\n" + " - add_header:\n" + - " dummy_Header_Key: \"dummyValue\"\n" + + " dummy_Header_Key4: \"dummyValue\"\n" + " - validate:\n" + " percentage_match:\n" + " gte: 90\n" + " - success: x5\n" + - " - failure: exit\n" + + " - failure: x5\n" + + "\n" + "\n" + "validate:\n" + " and:\n" + diff --git a/apps/testing/src/main/java/com/akto/test_editor/execution/Executor.java b/apps/testing/src/main/java/com/akto/test_editor/execution/Executor.java index fc18afa3a0..a0b3851e81 100644 --- a/apps/testing/src/main/java/com/akto/test_editor/execution/Executor.java +++ b/apps/testing/src/main/java/com/akto/test_editor/execution/Executor.java @@ -60,7 +60,9 @@ public class Executor { public final String _HOST = "host"; public YamlTestResult execute(ExecutorNode node, RawApi rawApi, Map varMap, String logId, - AuthMechanism authMechanism, FilterNode validatorNode, ApiInfo.ApiInfoKey apiInfoKey, TestingRunConfig testingRunConfig, List customAuthTypes, boolean debug, List testLogs) { + AuthMechanism authMechanism, FilterNode validatorNode, ApiInfo.ApiInfoKey apiInfoKey, TestingRunConfig testingRunConfig, + List customAuthTypes, boolean debug, List testLogs, + Memory memory, Map apiNameToApiInfoKey) { List result = new ArrayList<>(); ExecutionListBuilder executionListBuilder = new ExecutionListBuilder(); @@ -131,7 +133,7 @@ public YamlTestResult execute(ExecutorNode node, RawApi rawApi, Map customAuthTypes, ApiInfo.ApiInfoKey apiInfoKey, Map varMap, FilterNode validatorNode, boolean debug, List testLogs) { + List customAuthTypes, ApiInfo.ApiInfoKey apiInfoKey, Map varMap, FilterNode validatorNode, boolean debug, List testLogs, Memory memory, Map apiNameToApiInfoKey) { ApiWorkflowExecutor apiWorkflowExecutor = new ApiWorkflowExecutor(); Graph graph = new Graph(); @@ -252,7 +254,7 @@ public MultiExecTestResult triggerMultiExecution(WorkflowTest workflowTest, Exec List executionOrder = new ArrayList<>(); WorkflowTestResult workflowTestResult = new WorkflowTestResult(id, workflowTest.getId(), new HashMap<>(), null, null); GraphExecutorRequest graphExecutorRequest = new GraphExecutorRequest(graph, graph.getNode("x1"), workflowTest, null, null, varMap, "conditional", workflowTestResult, new HashMap<>(), executionOrder); - GraphExecutorResult graphExecutorResult = apiWorkflowExecutor.init(graphExecutorRequest, debug, testLogs); + GraphExecutorResult graphExecutorResult = apiWorkflowExecutor.init(graphExecutorRequest, debug, testLogs, memory, apiNameToApiInfoKey); return new MultiExecTestResult(graphExecutorResult.getWorkflowTestResult().getNodeResultMap(), graphExecutorResult.getVulnerable(), Confidence.HIGH, graphExecutorRequest.getExecutionOrder()); } diff --git a/apps/testing/src/main/java/com/akto/test_editor/execution/Memory.java b/apps/testing/src/main/java/com/akto/test_editor/execution/Memory.java index d9a8bea526..6f08b49db1 100644 --- a/apps/testing/src/main/java/com/akto/test_editor/execution/Memory.java +++ b/apps/testing/src/main/java/com/akto/test_editor/execution/Memory.java @@ -1,8 +1,6 @@ package com.akto.test_editor.execution; -import com.akto.DaoInit; import com.akto.dao.SampleDataDao; -import com.akto.dao.context.Context; import com.akto.dto.ApiInfo; import com.akto.dto.OriginalHttpRequest; import com.akto.dto.OriginalHttpResponse; @@ -11,10 +9,8 @@ import com.akto.dto.testing.TestingRunConfig; import com.akto.dto.traffic.Key; import com.akto.dto.traffic.SampleData; -import com.akto.dto.type.URLMethods; import com.akto.testing.ApiExecutor; import com.akto.util.Constants; -import com.mongodb.ConnectionString; import com.mongodb.client.model.Filters; import org.bson.conversions.Bson; @@ -31,61 +27,11 @@ public class Memory { Map sampleDataMap = new HashMap<>(); - public static Memory memory = new Memory( - Arrays.asList( - new ApiInfo.ApiInfoKey(1710156663, "https://juiceshop.akto.io/rest/products/reviews", URLMethods.Method.PATCH), - new ApiInfo.ApiInfoKey(1710156663, "https://juiceshop.akto.io/rest/user/login", URLMethods.Method.POST) - ), new HashMap<>() - ); private Map replaceDetailsMap = new HashMap<>(); - public static void main(String[] args) throws Exception { - DaoInit.init(new ConnectionString("mongodb://localhost:27017/admini")); - Context.accountId.set(1_000_000); - -// https://juiceshop.akto.io/rest/user/login - List apiInfoKeys = new ArrayList<>(); - apiInfoKeys.add(new ApiInfo.ApiInfoKey(1710156663, "https://juiceshop.akto.io/rest/products/reviews", URLMethods.Method.PATCH)); - apiInfoKeys.add(new ApiInfo.ApiInfoKey(1710156663, "https://juiceshop.akto.io/rest/user/login", URLMethods.Method.POST)); - - Memory memory = new Memory(apiInfoKeys, new HashMap<>()); - - OriginalHttpRequest req1 = memory.run(1710156663, "https://juiceshop.akto.io/rest/user/login", URLMethods.Method.POST.name()); - System.out.println("new"); - OriginalHttpResponse resp1 = ApiExecutor.sendRequest(req1, true, null, false, new ArrayList<>()); - memory.fillResponse(req1, resp1, 1710156663, "https://juiceshop.akto.io/rest/user/login", URLMethods.Method.POST.name()); - memory.reset(1710156663, "https://juiceshop.akto.io/rest/user/login", URLMethods.Method.POST.name()); - System.out.println("done"); - - - OriginalHttpRequest req2 = memory.run(1710156663, "https://juiceshop.akto.io/rest/products/reviews", URLMethods.Method.PATCH.name()); - System.out.println("new"); - OriginalHttpResponse resp2 = ApiExecutor.sendRequest(req2, true, null, false, new ArrayList<>()); - memory.fillResponse(req2, resp2, 1710156663, "https://juiceshop.akto.io/rest/products/reviews", URLMethods.Method.PATCH.name()); - memory.reset(1710156663, "https://juiceshop.akto.io/rest/products/reviews", URLMethods.Method.PATCH.name()); - System.out.println("done"); - - - OriginalHttpRequest req3 = memory.run(1710156663, "https://juiceshop.akto.io/rest/user/login", URLMethods.Method.POST.name()); - System.out.println("new"); - OriginalHttpResponse resp3 = ApiExecutor.sendRequest(req3, true, null, false, new ArrayList<>()); - memory.fillResponse(req3, resp3, 1710156663, "https://juiceshop.akto.io/rest/user/login", URLMethods.Method.POST.name()); - memory.reset(1710156663, "https://juiceshop.akto.io/rest/user/login", URLMethods.Method.POST.name()); - System.out.println("done"); - - - OriginalHttpRequest req4 = memory.run(1710156663, "https://juiceshop.akto.io/rest/products/reviews", URLMethods.Method.PATCH.name()); - System.out.println("new"); - OriginalHttpResponse resp4 = ApiExecutor.sendRequest(req4, true, null, false, new ArrayList<>()); - memory.fillResponse(req4, resp4, 1710156663, "https://juiceshop.akto.io/rest/products/reviews", URLMethods.Method.PATCH.name()); - memory.reset(1710156663, "https://juiceshop.akto.io/rest/products/reviews", URLMethods.Method.PATCH.name()); - System.out.println("done"); - - - } - public Memory(List apiInfoKeys, Map replaceDetailsMap) { + if (apiInfoKeys == null || apiInfoKeys.isEmpty()) return; this.replaceDetailsMap = replaceDetailsMap; // find all parent APIs diff --git a/apps/testing/src/main/java/com/akto/testing/ApiWorkflowExecutor.java b/apps/testing/src/main/java/com/akto/testing/ApiWorkflowExecutor.java index eaf243170f..194103dba4 100644 --- a/apps/testing/src/main/java/com/akto/testing/ApiWorkflowExecutor.java +++ b/apps/testing/src/main/java/com/akto/testing/ApiWorkflowExecutor.java @@ -1,16 +1,19 @@ package com.akto.testing; +import com.akto.dto.ApiInfo; import com.akto.dto.testing.*; +import com.akto.test_editor.execution.Memory; import com.akto.testing.workflow_node_executor.GraphExecutor; import com.akto.testing.workflow_node_executor.GraphExecutorFactory; import java.util.List; +import java.util.Map; public class ApiWorkflowExecutor { - public GraphExecutorResult init(GraphExecutorRequest graphExecutorRequest, boolean debug, List testLogs) { + public GraphExecutorResult init(GraphExecutorRequest graphExecutorRequest, boolean debug, List testLogs, Memory memory, Map apiNameToApiInfoKey) { GraphExecutor graphExecutor = GraphExecutorFactory.fetchExecutor(graphExecutorRequest); - GraphExecutorResult graphExecutorResult = graphExecutor.executeGraph(graphExecutorRequest,debug,testLogs); + GraphExecutorResult graphExecutorResult = graphExecutor.executeGraph(graphExecutorRequest,debug,testLogs, memory, apiNameToApiInfoKey); return graphExecutorResult; } diff --git a/apps/testing/src/main/java/com/akto/testing/TestExecutor.java b/apps/testing/src/main/java/com/akto/testing/TestExecutor.java index 91cdfc20f3..7d487dd881 100644 --- a/apps/testing/src/main/java/com/akto/testing/TestExecutor.java +++ b/apps/testing/src/main/java/com/akto/testing/TestExecutor.java @@ -16,10 +16,7 @@ import com.akto.dto.OriginalHttpRequest; import com.akto.dto.RawApi; import com.akto.dto.api_workflow.Graph; -import com.akto.dto.test_editor.Auth; -import com.akto.dto.test_editor.ExecutorNode; -import com.akto.dto.test_editor.FilterNode; -import com.akto.dto.test_editor.TestConfig; +import com.akto.dto.test_editor.*; import com.akto.dto.testing.*; import com.akto.dto.testing.TestResult.Confidence; import com.akto.dto.testing.TestResult.TestError; @@ -99,7 +96,7 @@ public void workflowInit (TestingRun testingRun, ObjectId summaryId, boolean deb Graph graph = new Graph(); graph.buildGraph(workflowTest); GraphExecutorRequest graphExecutorRequest = new GraphExecutorRequest(graph, workflowTest, testingRun.getId(), summaryId, valuesMap, false, "linear"); - GraphExecutorResult graphExecutorResult = apiWorkflowExecutor.init(graphExecutorRequest, debug, testLogs); + GraphExecutorResult graphExecutorResult = apiWorkflowExecutor.init(graphExecutorRequest, debug, testLogs, null, null); WorkflowTestResultsDao.instance.insertOne(graphExecutorResult.getWorkflowTestResult()); } catch (Exception e) { loggerMaker.errorAndAddToDb("Error while executing workflow test " + e, LogDb.TESTING); @@ -638,6 +635,7 @@ public TestingRunResult runTestNew(ApiInfo.ApiInfoKey apiInfoKey, ObjectId testR } FilterNode filterNode = testConfig.getApiSelectionFilters().getNode(); + Map workFlowSelectionFilters = testConfig.getWorkFlowSelectionFilters(); FilterNode validatorNode = null; if (testConfig.getValidation() != null) { validatorNode = testConfig.getValidation().getNode(); @@ -661,7 +659,7 @@ public TestingRunResult runTestNew(ApiInfo.ApiInfoKey apiInfoKey, ObjectId testR List customAuthTypes = testingUtil.getCustomAuthTypes(); YamlTestTemplate yamlTestTemplate = new YamlTestTemplate(apiInfoKey,filterNode, validatorNode, executorNode, - rawApi, varMap, auth, testingUtil.getAuthMechanism(), testExecutionLogId, testingRunConfig, customAuthTypes, testConfig.getStrategy()); + rawApi, varMap, auth, testingUtil.getAuthMechanism(), testExecutionLogId, testingRunConfig, customAuthTypes, testConfig.getStrategy(), workFlowSelectionFilters); YamlTestResult testResults = yamlTestTemplate.run(debug, testLogs); if (testResults == null || testResults.getTestResults().isEmpty()) { List res = new ArrayList<>(); diff --git a/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/ApiNodeExecutor.java b/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/ApiNodeExecutor.java index c79885d67c..8b67217871 100644 --- a/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/ApiNodeExecutor.java +++ b/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/ApiNodeExecutor.java @@ -5,6 +5,7 @@ import java.util.Map; import java.util.Collections; +import com.akto.dto.ApiInfo; import com.akto.dto.OriginalHttpRequest; import com.akto.dto.OriginalHttpResponse; import com.akto.dto.api_workflow.Node; @@ -15,6 +16,7 @@ import com.akto.dto.testing.WorkflowTestResult.NodeResult; import com.akto.log.LoggerMaker; import com.akto.log.LoggerMaker.LogDb; +import com.akto.test_editor.execution.Memory; import com.akto.testing.ApiExecutor; import com.akto.testing.Main; import com.akto.testing.Utils; @@ -24,7 +26,7 @@ public class ApiNodeExecutor extends NodeExecutor { private static final LoggerMaker loggerMaker = new LoggerMaker(ApiNodeExecutor.class); - public NodeResult processNode(Node node, Map valuesMap, Boolean allowAllStatusCodes, boolean debug, List testLogs) { + public NodeResult processNode(Node node, Map valuesMap, Boolean allowAllStatusCodes, boolean debug, List testLogs, Memory memory, Map apiNameToApiInfoKey) { loggerMaker.infoAndAddToDb("\n", LogDb.TESTING); loggerMaker.infoAndAddToDb("NODE: " + node.getId(), LogDb.TESTING); List testErrors = new ArrayList<>(); diff --git a/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/ConditionalGraphExecutor.java b/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/ConditionalGraphExecutor.java index 903921508b..afc598d5a9 100644 --- a/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/ConditionalGraphExecutor.java +++ b/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/ConditionalGraphExecutor.java @@ -5,16 +5,18 @@ import java.util.Map; import com.akto.dao.test_editor.TestEditorEnums; +import com.akto.dto.ApiInfo; import com.akto.dto.api_workflow.Node; import com.akto.dto.test_editor.DataOperandsFilterResponse; import com.akto.dto.test_editor.ExecutorNode; import com.akto.dto.test_editor.FilterNode; import com.akto.dto.testing.*; +import com.akto.test_editor.execution.Memory; import com.akto.test_editor.filter.Filter; public class ConditionalGraphExecutor extends GraphExecutor { - public GraphExecutorResult executeGraph(GraphExecutorRequest graphExecutorRequest,boolean debug, List testLogs) { + public GraphExecutorResult executeGraph(GraphExecutorRequest graphExecutorRequest, boolean debug, List testLogs, Memory memory, Map apiNameToApiInfoKey) { Map visitedMap = graphExecutorRequest.getVisitedMap(); List errors = new ArrayList<>(); @@ -30,7 +32,7 @@ public GraphExecutorResult executeGraph(GraphExecutorRequest graphExecutorReques boolean success = false; WorkflowTestResult.NodeResult nodeResult; - nodeResult = Utils.executeNode(node, graphExecutorRequest.getValuesMap(), debug, testLogs); + nodeResult = Utils.executeNode(node, graphExecutorRequest.getValuesMap(), debug, testLogs, memory, apiNameToApiInfoKey); graphExecutorRequest.getWorkflowTestResult().getNodeResultMap().put(node.getId(), nodeResult); graphExecutorRequest.getExecutionOrder().add(node.getId()); @@ -77,7 +79,7 @@ public GraphExecutorResult executeGraph(GraphExecutorRequest graphExecutorReques boolean vulnerable = success; if (childNode != null) { GraphExecutorRequest childExecReq = new GraphExecutorRequest(graphExecutorRequest, childNode, graphExecutorRequest.getWorkflowTestResult(), visitedMap, graphExecutorRequest.getExecutionOrder()); - GraphExecutorResult childExecResult = executeGraph(childExecReq, debug, testLogs); + GraphExecutorResult childExecResult = executeGraph(childExecReq, debug, testLogs, memory, apiNameToApiInfoKey); vulnerable = childExecResult.getVulnerable(); return new GraphExecutorResult(graphExecutorRequest.getWorkflowTestResult(), vulnerable, errors); } else { diff --git a/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/GraphExecutor.java b/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/GraphExecutor.java index 42faeac0a7..5e6ab7f15a 100644 --- a/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/GraphExecutor.java +++ b/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/GraphExecutor.java @@ -1,13 +1,16 @@ package com.akto.testing.workflow_node_executor; +import com.akto.dto.ApiInfo; import com.akto.dto.testing.GraphExecutorRequest; import com.akto.dto.testing.GraphExecutorResult; import com.akto.dto.testing.TestingRunResult; +import com.akto.test_editor.execution.Memory; import java.util.List; +import java.util.Map; public abstract class GraphExecutor { - public abstract GraphExecutorResult executeGraph(GraphExecutorRequest graphExecutorRequest, boolean debug, List testLogs); + public abstract GraphExecutorResult executeGraph(GraphExecutorRequest graphExecutorRequest, boolean debug, List testLogs, Memory memory, Map apiNameToApiInfoKey); } diff --git a/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/LinearGraphExecutor.java b/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/LinearGraphExecutor.java index 6f269502bf..95ad59a8aa 100644 --- a/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/LinearGraphExecutor.java +++ b/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/LinearGraphExecutor.java @@ -6,15 +6,17 @@ import java.util.Map; import com.akto.dao.context.Context; +import com.akto.dto.ApiInfo; import com.akto.dto.api_workflow.Node; import com.akto.dto.testing.GraphExecutorRequest; import com.akto.dto.testing.GraphExecutorResult; import com.akto.dto.testing.TestingRunResult; import com.akto.dto.testing.WorkflowTestResult; +import com.akto.test_editor.execution.Memory; public class LinearGraphExecutor extends GraphExecutor { - public GraphExecutorResult executeGraph(GraphExecutorRequest graphExecutorRequest, boolean debug, List testLogs) { + public GraphExecutorResult executeGraph(GraphExecutorRequest graphExecutorRequest, boolean debug, List testLogs, Memory memory, Map apiNameToApiInfoKey) { List nodes = graphExecutorRequest.getGraph().sort(); int id = Context.now(); @@ -22,7 +24,7 @@ public GraphExecutorResult executeGraph(GraphExecutorRequest graphExecutorReques Map testResultMap = workflowTestResult.getNodeResultMap(); for (Node node: nodes) { WorkflowTestResult.NodeResult nodeResult; - nodeResult = Utils.executeNode(node, graphExecutorRequest.getValuesMap(), debug, testLogs); + nodeResult = Utils.executeNode(node, graphExecutorRequest.getValuesMap(), debug, testLogs, memory, apiNameToApiInfoKey); testResultMap.put(node.getId(), nodeResult); if (nodeResult.getErrors().size() > 0) break; if (graphExecutorRequest.getSkipIfNotVulnerable() && !nodeResult.isVulnerable()) { diff --git a/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/NodeExecutor.java b/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/NodeExecutor.java index 5fd8c29f34..63a1415d5e 100644 --- a/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/NodeExecutor.java +++ b/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/NodeExecutor.java @@ -3,11 +3,13 @@ import java.util.List; import java.util.Map; +import com.akto.dto.ApiInfo; import com.akto.dto.api_workflow.Node; import com.akto.dto.testing.TestingRunResult; import com.akto.dto.testing.WorkflowTestResult; +import com.akto.test_editor.execution.Memory; public abstract class NodeExecutor { - public abstract WorkflowTestResult.NodeResult processNode(Node node, Map valuesMap, Boolean allowAllStatusCodes, boolean debug, List testLogs); + public abstract WorkflowTestResult.NodeResult processNode(Node node, Map valuesMap, Boolean allowAllStatusCodes, boolean debug, List testLogs, Memory memory, Map apiNameToApiInfoKey); } diff --git a/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/Utils.java b/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/Utils.java index afb7fea057..6cdaa14efc 100644 --- a/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/Utils.java +++ b/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/Utils.java @@ -14,7 +14,9 @@ import javax.script.ScriptEngineManager; import javax.script.ScriptException; +import com.akto.dto.ApiInfo; import com.akto.dto.testing.*; +import com.akto.test_editor.execution.Memory; import org.apache.commons.lang3.StringUtils; import org.bson.conversions.Bson; import org.json.JSONObject; @@ -187,7 +189,7 @@ private static String fetchToken(RecordedLoginFlowInput recordedLoginFlowInput, return token; } - public static WorkflowTestResult.NodeResult processNode(Node node, Map valuesMap, Boolean allowAllStatusCodes, boolean debug, List testLogs) { + public static WorkflowTestResult.NodeResult processNode(Node node, Map valuesMap, Boolean allowAllStatusCodes, boolean debug, List testLogs, Memory memory, Map apiNameToApiInfoKey) { if (node.getWorkflowNodeDetails().getType() == WorkflowNodeDetails.Type.RECORDED) { return processRecorderNode(node, valuesMap); } @@ -195,22 +197,22 @@ else if (node.getWorkflowNodeDetails().getType() == WorkflowNodeDetails.Type.OTP return processOtpNode(node, valuesMap); } else { - return processApiNode(node, valuesMap, allowAllStatusCodes, debug, testLogs); + return processApiNode(node, valuesMap, allowAllStatusCodes, debug, testLogs, memory, apiNameToApiInfoKey); } } - public static WorkflowTestResult.NodeResult processApiNode(Node node, Map valuesMap, Boolean allowAllStatusCodes, boolean debug, List testLogs) { + public static WorkflowTestResult.NodeResult processApiNode(Node node, Map valuesMap, Boolean allowAllStatusCodes, boolean debug, List testLogs, Memory memory, Map apiNameToApiInfoKey) { NodeExecutorFactory nodeExecutorFactory = new NodeExecutorFactory(); NodeExecutor nodeExecutor = nodeExecutorFactory.getExecutor(node); - return nodeExecutor.processNode(node, valuesMap, allowAllStatusCodes, debug, testLogs); + return nodeExecutor.processNode(node, valuesMap, allowAllStatusCodes, debug, testLogs, memory, apiNameToApiInfoKey); } - public static WorkflowTestResult.NodeResult executeNode(Node node, Map valuesMap,boolean debug, List testLogs) { + public static WorkflowTestResult.NodeResult executeNode(Node node, Map valuesMap,boolean debug, List testLogs, Memory memory, Map apiNameToApiInfoKey) { WorkflowTestResult.NodeResult nodeResult; try { - nodeResult = Utils.processNode(node, valuesMap, true, debug, testLogs); + nodeResult = Utils.processNode(node, valuesMap, true, debug, testLogs, memory, apiNameToApiInfoKey); } catch (Exception e) { ; List testErrors = new ArrayList<>(); @@ -239,7 +241,7 @@ public static LoginFlowResponse runLoginFlow(WorkflowTest workflowTest, AuthMech for (Node node: nodes) { WorkflowTestResult.NodeResult nodeResult; try { - nodeResult = processNode(node, valuesMap, false, false, new ArrayList<>()); + nodeResult = processNode(node, valuesMap, false, false, new ArrayList<>(), null, null); } catch (Exception e) { ; List testErrors = new ArrayList<>(); diff --git a/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/YamlNodeExecutor.java b/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/YamlNodeExecutor.java index f015231e39..e3d172782d 100644 --- a/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/YamlNodeExecutor.java +++ b/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/YamlNodeExecutor.java @@ -48,7 +48,7 @@ public class YamlNodeExecutor extends NodeExecutor { private static final Gson gson = new Gson(); - public NodeResult processNode(Node node, Map varMap, Boolean allowAllStatusCodes, boolean debug, List testLogs) { + public NodeResult processNode(Node node, Map varMap, Boolean allowAllStatusCodes, boolean debug, List testLogs, Memory memory, Map apiNameToApiInfoKey) { List testErrors = new ArrayList<>(); YamlNodeDetails yamlNodeDetails = (YamlNodeDetails) node.getWorkflowNodeDetails(); @@ -58,31 +58,26 @@ public NodeResult processNode(Node node, Map varMap, Boolean all } RawApi rawApi = yamlNodeDetails.getRawApi(); - RawApi sampleRawApi = rawApi.copy(); - - List rawApis = new ArrayList<>(); - rawApis.add(rawApi.copy()); Executor executor = new Executor(); ExecutorNode executorNode = yamlNodeDetails.getExecutorNode(); FilterNode validatorNode = yamlNodeDetails.getValidatorNode(); List childNodes = executorNode.getChildNodes(); - String api = null; + ApiInfo.ApiInfoKey apiInfoKey = null; ExecutorNode firstChildNode = childNodes.get(0); // todo check for length if (firstChildNode.getOperationType().equals("API")) { - api = firstChildNode.getValues().toString(); + String api = firstChildNode.getValues().toString(); + apiInfoKey = apiNameToApiInfoKey.get(api); + OriginalHttpRequest newRequest = memory.run(apiInfoKey.getApiCollectionId(), apiInfoKey.getUrl(), apiInfoKey.getMethod().name()); + rawApi.setRequest(newRequest); childNodes.remove(0); } - ApiInfo.ApiInfoKey apiInfoKey = new ApiInfo.ApiInfoKey(1710156663, "https://juiceshop.akto.io/rest/user/login", URLMethods.Method.POST); - - OriginalHttpRequest newRequest = Memory.memory.run(apiInfoKey.getApiCollectionId(), apiInfoKey.getUrl(), apiInfoKey.getMethod().name()); - rawApi.setRequest(newRequest); + RawApi sampleRawApi = rawApi.copy(); - if (api != null) { -// ApiInfo.ApiInfoKey apiInfoKey = counter % 2 != 0 ? new ApiInfo.ApiInfoKey(1710156663, "https://juiceshop.akto.io/rest/user/login", URLMethods.Method.POST) : new ApiInfo.ApiInfoKey(1710156663, "https://juiceshop.akto.io/rest/products/reviews", URLMethods.Method.PATCH); - } + List rawApis = new ArrayList<>(); + rawApis.add(rawApi.copy()); for (ExecutorNode execNode: childNodes) { if (execNode.getNodeType().equalsIgnoreCase(TestEditorEnums.ValidateExecutorDataOperands.Validate.toString())) { @@ -132,8 +127,10 @@ public NodeResult processNode(Node node, Map varMap, Boolean all try { tsBeforeReq = Context.nowInMillis(); testResponse = ApiExecutor.sendRequest(testReq.getRequest(), followRedirect, testingRunConfig, debug, testLogs, Main.SKIP_SSRF_CHECK); - Memory.memory.fillResponse(testReq.getRequest(), testResponse, apiInfoKey.getApiCollectionId(), apiInfoKey.getUrl(), apiInfoKey.getMethod().name()); - Memory.memory.reset(apiInfoKey.getApiCollectionId(), apiInfoKey.getUrl(), apiInfoKey.getMethod().name()); + if (apiInfoKey != null) { + memory.fillResponse(testReq.getRequest(), testResponse, apiInfoKey.getApiCollectionId(), apiInfoKey.getUrl(), apiInfoKey.getMethod().name()); + memory.reset(apiInfoKey.getApiCollectionId(), apiInfoKey.getUrl(), apiInfoKey.getMethod().name()); + } tsAfterReq = Context.nowInMillis(); responseTimeArr.add(tsAfterReq - tsBeforeReq); ExecutionResult attempt = new ExecutionResult(singleReq.getSuccess(), singleReq.getErrMsg(), testReq.getRequest(), testResponse); diff --git a/apps/testing/src/main/java/com/akto/testing/yaml_tests/SecurityTestTemplate.java b/apps/testing/src/main/java/com/akto/testing/yaml_tests/SecurityTestTemplate.java index 1cc9a6607f..8859ea3e6a 100644 --- a/apps/testing/src/main/java/com/akto/testing/yaml_tests/SecurityTestTemplate.java +++ b/apps/testing/src/main/java/com/akto/testing/yaml_tests/SecurityTestTemplate.java @@ -2,17 +2,12 @@ import com.akto.dto.ApiInfo; import com.akto.dto.RawApi; -import com.akto.dto.test_editor.Auth; -import com.akto.dto.test_editor.ExecutorNode; -import com.akto.dto.test_editor.FilterNode; -import com.akto.dto.test_editor.Strategy; +import com.akto.dto.test_editor.*; import com.akto.dto.testing.*; import com.akto.dto.testing.TestResult.TestError; +import com.akto.test_editor.execution.Memory; -import java.util.Collections; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; +import java.util.*; public abstract class SecurityTestTemplate { @@ -29,7 +24,13 @@ public abstract class SecurityTestTemplate { TestingRunConfig testingRunConfig; Strategy strategy; - public SecurityTestTemplate(ApiInfo.ApiInfoKey apiInfoKey, FilterNode filterNode, FilterNode validatorNode, ExecutorNode executorNode ,RawApi rawApi, Map varMap, Auth auth, AuthMechanism authMechanism, String logId, TestingRunConfig testingRunConfig, Strategy strategy) { + Map apiNameToApiInfoKey = new HashMap<>(); + + Map workFlowSelectionFilters; + + Memory memory; + + public SecurityTestTemplate(ApiInfo.ApiInfoKey apiInfoKey, FilterNode filterNode, FilterNode validatorNode, ExecutorNode executorNode ,RawApi rawApi, Map varMap, Auth auth, AuthMechanism authMechanism, String logId, TestingRunConfig testingRunConfig, Strategy strategy, Map workFlowSelectionFilters) { this.apiInfoKey = apiInfoKey; this.filterNode = filterNode; this.validatorNode = validatorNode; @@ -41,10 +42,13 @@ public SecurityTestTemplate(ApiInfo.ApiInfoKey apiInfoKey, FilterNode filterNode this.logId = logId; this.testingRunConfig = testingRunConfig; this.strategy = strategy; + this.workFlowSelectionFilters = workFlowSelectionFilters; } public abstract boolean filter(); + public abstract boolean workflowFilter(); + public abstract boolean checkAuthBeforeExecution(boolean debug, List testLogs); public abstract YamlTestResult executor(boolean debug, List testLogs); @@ -65,6 +69,14 @@ public YamlTestResult run(boolean debug, List testLogs testResults.add(new TestResult(null, rawApi.getOriginalMessage(), Collections.singletonList("Request API failed authentication check, skipping execution"), 0, false, TestResult.Confidence.HIGH, null)); return new YamlTestResult(testResults, null); } + + boolean workflowFound = workflowFilter(); + if (!workflowFound) { + List testResults = new ArrayList<>(); + testResults.add(new TestResult(null, rawApi.getOriginalMessage(), Collections.singletonList("Request API failed to satisfy workflow_selection_filters block, skipping execution"), 0, false, TestResult.Confidence.HIGH, null)); + return new YamlTestResult(testResults, null); + } + YamlTestResult attempts = executor(debug, testLogs); if(attempts == null || attempts.getTestResults().isEmpty()){ List res = new ArrayList<>(); @@ -146,5 +158,20 @@ public String getLogId() { public void setLogId(String logId) { this.logId = logId; } - + + public Map getApiNameToApiInfoKey() { + return apiNameToApiInfoKey; + } + + public void setApiNameToApiInfoKey(Map apiNameToApiInfoKey) { + this.apiNameToApiInfoKey = apiNameToApiInfoKey; + } + + public Map getWorkFlowSelectionFilters() { + return workFlowSelectionFilters; + } + + public void setWorkFlowSelectionFilters(Map workFlowSelectionFilters) { + this.workFlowSelectionFilters = workFlowSelectionFilters; + } } diff --git a/apps/testing/src/main/java/com/akto/testing/yaml_tests/YamlTestTemplate.java b/apps/testing/src/main/java/com/akto/testing/yaml_tests/YamlTestTemplate.java index e49fd0983c..8c58a12f71 100644 --- a/apps/testing/src/main/java/com/akto/testing/yaml_tests/YamlTestTemplate.java +++ b/apps/testing/src/main/java/com/akto/testing/yaml_tests/YamlTestTemplate.java @@ -1,21 +1,25 @@ package com.akto.testing.yaml_tests; +import com.akto.dao.SampleDataDao; import com.akto.dto.ApiInfo; import com.akto.dto.CustomAuthType; import com.akto.dto.OriginalHttpResponse; import com.akto.dto.RawApi; -import com.akto.dto.test_editor.Auth; -import com.akto.dto.test_editor.ExecutionResult; -import com.akto.dto.test_editor.ExecutorNode; -import com.akto.dto.test_editor.FilterNode; -import com.akto.dto.test_editor.Strategy; +import com.akto.dto.test_editor.*; import com.akto.dto.testing.*; +import com.akto.dto.traffic.SampleData; import com.akto.log.LoggerMaker; import com.akto.rules.TestPlugin; import com.akto.test_editor.auth.AuthValidator; import com.akto.test_editor.execution.Executor; +import com.akto.test_editor.execution.Memory; import com.akto.testing.StatusCodeAnalyser; +import com.mongodb.BasicDBObject; +import com.mongodb.client.model.Projections; +import com.mongodb.client.model.Sorts; +import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; @@ -25,8 +29,9 @@ public class YamlTestTemplate extends SecurityTestTemplate { private final List customAuthTypes; public YamlTestTemplate(ApiInfo.ApiInfoKey apiInfoKey, FilterNode filterNode, FilterNode validatorNode, ExecutorNode executorNode, RawApi rawApi, Map varMap, Auth auth, - AuthMechanism authMechanism, String logId, TestingRunConfig testingRunConfig, List customAuthTypes, Strategy strategy) { - super(apiInfoKey, filterNode, validatorNode, executorNode ,rawApi, varMap, auth, authMechanism, logId, testingRunConfig, strategy); + AuthMechanism authMechanism, String logId, TestingRunConfig testingRunConfig, + List customAuthTypes, Strategy strategy, Map workFlowSelectionFilters) { + super(apiInfoKey, filterNode, validatorNode, executorNode ,rawApi, varMap, auth, authMechanism, logId, testingRunConfig, strategy, workFlowSelectionFilters); this.customAuthTypes = customAuthTypes; } @@ -51,6 +56,37 @@ public boolean filter() { return isValid; } + @Override + public boolean workflowFilter() { + Map result = new HashMap<>(); + int skip = 0; + int limit = 1000; + while (true) { + List sampleDataList = SampleDataDao.instance.findAll(new BasicDBObject(), skip, limit, Sorts.ascending("_id.url"));// todo: can be improved + for (SampleData sampleData: sampleDataList) { + RawApi localRawApi = RawApi.buildFromMessage(sampleData.getSamples().get(0)); + ApiInfo.ApiInfoKey localApiInfoKey = new ApiInfo.ApiInfoKey(sampleData.getId().getApiCollectionId(), sampleData.getId().getUrl(), sampleData.getId().getMethod()); + for (String key: this.workFlowSelectionFilters.keySet()) { + if (result.containsKey(key)) continue; + ConfigParserResult configParserResult = this.workFlowSelectionFilters.get(key); + FilterNode localFilterNode = configParserResult.getNode(); + boolean isValid = TestPlugin.validateFilter(localFilterNode, localRawApi, localApiInfoKey, this.varMap, this.logId); + if (isValid) { + result.put(key, localApiInfoKey); + break; + } + } + if (result.size() == this.workFlowSelectionFilters.size()) break; + } + + if (sampleDataList.size() < limit || result.size() == this.workFlowSelectionFilters.size()) break; + } + + memory = new Memory(new ArrayList<>(result.values()), new HashMap<>()); + this.apiNameToApiInfoKey = result; + return true; + } + @Override public boolean checkAuthBeforeExecution(boolean debug, List testLogs) { if (this.auth != null && this.auth.getAuthenticated() != null && this.auth.getAuthenticated() == true) { @@ -72,7 +108,8 @@ public boolean checkAuthBeforeExecution(boolean debug, List testLogs) { // loggerMaker.infoAndAddToDb("executor started" + logId, LogDb.TESTING); YamlTestResult results = new Executor().execute(this.executorNode, this.rawApi, this.varMap, this.logId, - this.authMechanism, this.validatorNode, this.apiInfoKey, this.testingRunConfig, this.customAuthTypes, debug, testLogs); + this.authMechanism, this.validatorNode, this.apiInfoKey, this.testingRunConfig, this.customAuthTypes, + debug, testLogs, memory, apiNameToApiInfoKey); // loggerMaker.infoAndAddToDb("execution result size " + results.size() + " " + logId, LogDb.TESTING); return results; } diff --git a/libs/dao/src/main/java/com/akto/dao/test_editor/TestConfigYamlParser.java b/libs/dao/src/main/java/com/akto/dao/test_editor/TestConfigYamlParser.java index e719d7e419..835670f192 100644 --- a/libs/dao/src/main/java/com/akto/dao/test_editor/TestConfigYamlParser.java +++ b/libs/dao/src/main/java/com/akto/dao/test_editor/TestConfigYamlParser.java @@ -1,5 +1,6 @@ package com.akto.dao.test_editor; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -56,21 +57,33 @@ public static TestConfig parseConfig(Map config) throws Exceptio Parser authParser = new Parser(); auth = authParser.parse(authMap); if (auth == null) { - return new TestConfig(id, info, null, null, null, null, null, null); + return new TestConfig(id, info, null, null, null, null, null, null, null); } } Object filterMap = config.get("api_selection_filters"); if (filterMap == null) { // todo: should not be null, throw error - return new TestConfig(id, info, auth, null, null, null, null, null); + return new TestConfig(id, info, auth, null, null, null, null, null, null); } - + + Object workflowSelectionFilterObj = config.get("workflow_selection_filters"); + Map workFlowSelectionFilters = new HashMap<>(); + if (workflowSelectionFilterObj != null) { + Map workflowSelectionFilterMap = (Map) workflowSelectionFilterObj; + for (String apiName: workflowSelectionFilterMap.keySet()) { + Object o = workflowSelectionFilterMap.get(apiName); + ConfigParserResult childFilter = new ConfigParser().parse(o); + workFlowSelectionFilters.put(apiName, childFilter); + } + // todo: should not be null, throw error + } + ConfigParser configParser = new ConfigParser(); ConfigParserResult filters = configParser.parse(filterMap); if (filters == null) { // todo: throw error - new TestConfig(id, info, auth, null, null, null, null, null); + new TestConfig(id, info, auth, null, null, null, null, null, workFlowSelectionFilters); } Map> wordListMap = new HashMap<>(); @@ -79,44 +92,44 @@ public static TestConfig parseConfig(Map config) throws Exceptio wordListMap = (Map) config.get("wordLists"); } } catch (Exception e) { - return new TestConfig(id, info, null, null, null, null, null, null); + return new TestConfig(id, info, null, null, null, null, null, null, workFlowSelectionFilters); } Object executionMap = config.get("execute"); if (executionMap == null) { // todo: should not be null, throw error - return new TestConfig(id, info, auth, filters, wordListMap, null, null, null); + return new TestConfig(id, info, auth, filters, wordListMap, null, null, null, workFlowSelectionFilters); } com.akto.dao.test_editor.executor.ConfigParser executorConfigParser = new com.akto.dao.test_editor.executor.ConfigParser(); ExecutorConfigParserResult executeOperations = executorConfigParser.parseConfigMap(executionMap); if (executeOperations == null) { // todo: throw error - new TestConfig(id, info, auth, filters, wordListMap, null, null, null); + new TestConfig(id, info, auth, filters, wordListMap, null, null, null, workFlowSelectionFilters); } Object validationMap = config.get("validate"); if (validationMap == null) { // todo: should not be null, throw error - return new TestConfig(id, info, auth, filters, wordListMap, executeOperations, null, null); + return new TestConfig(id, info, auth, filters, wordListMap, executeOperations, null, null, workFlowSelectionFilters); } ConfigParserResult validations = configParser.parse(validationMap); if (validations == null) { // todo: throw error - new TestConfig(id, info, auth, filters, wordListMap, executeOperations, null, null); + new TestConfig(id, info, auth, filters, wordListMap, executeOperations, null, null, workFlowSelectionFilters); } Object strategyObject = config.get("strategy"); if (strategyObject == null) { - return new TestConfig(id, info, auth, filters, wordListMap, executeOperations, validations, null); + return new TestConfig(id, info, auth, filters, wordListMap, executeOperations, validations, null, workFlowSelectionFilters); } StrategyParser strategyParser = new StrategyParser(); Strategy strategy = strategyParser.parse(strategyObject); - testConfig = new TestConfig(id, info, auth, filters, wordListMap, executeOperations, validations, strategy); + testConfig = new TestConfig(id, info, auth, filters, wordListMap, executeOperations, validations, strategy, workFlowSelectionFilters); return testConfig; } diff --git a/libs/dao/src/main/java/com/akto/dto/test_editor/TestConfig.java b/libs/dao/src/main/java/com/akto/dto/test_editor/TestConfig.java index 347b742561..2f4d77fb1b 100644 --- a/libs/dao/src/main/java/com/akto/dto/test_editor/TestConfig.java +++ b/libs/dao/src/main/java/com/akto/dto/test_editor/TestConfig.java @@ -14,6 +14,7 @@ public class TestConfig { private Auth auth; private ConfigParserResult apiSelectionFilters; + private Map workFlowSelectionFilters; private Map> wordlists; @@ -29,7 +30,7 @@ public class TestConfig { private String author; public TestConfig(String id, Info info, Auth auth, ConfigParserResult apiSelectionFilters, Map> wordlists, ExecutorConfigParserResult execute, - ConfigParserResult validation, Strategy strategy) { + ConfigParserResult validation, Strategy strategy, Map workFlowSelectionFilters) { this.id = id; info.setSubCategory(id); @@ -40,6 +41,7 @@ public TestConfig(String id, Info info, Auth auth, ConfigParserResult apiSelecti this.execute = execute; this.validation = validation; this.strategy = strategy; + this.workFlowSelectionFilters = workFlowSelectionFilters; } public TestConfig() { } @@ -148,4 +150,15 @@ public void setAuthor(String author) { this.author = author; } + public Map getWorkFlowSelectionFilters() { + return workFlowSelectionFilters; + } + + public void setWorkFlowSelectionFilters(Map workFlowSelectionFilters) { + this.workFlowSelectionFilters = workFlowSelectionFilters; + } + + public boolean isInactive() { + return inactive; + } } From 5e03999386c7cc57a0d8e90de724a3d1e4bf40bb Mon Sep 17 00:00:00 2001 From: Avneesh Hota Date: Sat, 13 Apr 2024 14:56:04 +0530 Subject: [PATCH 04/16] allow get_assets call in templates --- .../akto/dependency/DependencyAnalyser.java | 7 +- .../main/java/com/akto/action/HarAction.java | 111 ++++++++---------- .../akto/test_editor/execution/Executor.java | 7 +- .../akto/test_editor/execution/Memory.java | 83 +++++++++++++ .../YamlNodeExecutor.java | 18 +-- .../testing/yaml_tests/YamlTestTemplate.java | 26 ---- 6 files changed, 156 insertions(+), 96 deletions(-) diff --git a/apps/api-runtime/src/main/java/com/akto/dependency/DependencyAnalyser.java b/apps/api-runtime/src/main/java/com/akto/dependency/DependencyAnalyser.java index c4b83424f4..9e1f57b5c9 100644 --- a/apps/api-runtime/src/main/java/com/akto/dependency/DependencyAnalyser.java +++ b/apps/api-runtime/src/main/java/com/akto/dependency/DependencyAnalyser.java @@ -152,7 +152,10 @@ public void analyse(String message, int finalApiCollectionId) { SingleTypeInfo.SuperType superType = urlTemplate.getTypes()[i]; if (superType == null) continue; int idx = ogUrl.startsWith("http") ? i:i+1; - String s = ogUrlSplit[idx]; // because ogUrl=/api/books/123 while template url=api/books/INTEGER + Object s = ogUrlSplit[idx]; // because ogUrl=/api/books/123 while template url=api/books/INTEGER + if (superType.equals(SingleTypeInfo.SuperType.INTEGER)) { + s = Integer.parseInt(ogUrlSplit[idx]); + } Set val = new HashSet<>(); val.add(s); processRequestParam(i+"", val, combinedUrl, true, false); @@ -235,7 +238,7 @@ public boolean filterValues(Object val) { if (val == null) return false; if (val instanceof Boolean) return false; if (val instanceof String) return val.toString().length() > 4 && val.toString().length() <= 4096; - if (val instanceof Integer) return ((int) val) > 50; + if (val instanceof Integer) return ((int) val) > 0; return true; } diff --git a/apps/dashboard/src/main/java/com/akto/action/HarAction.java b/apps/dashboard/src/main/java/com/akto/action/HarAction.java index ab746d2f01..296409bf75 100644 --- a/apps/dashboard/src/main/java/com/akto/action/HarAction.java +++ b/apps/dashboard/src/main/java/com/akto/action/HarAction.java @@ -10,6 +10,7 @@ import com.akto.dto.ApiCollection; import com.akto.dto.ApiInfo; import com.akto.dto.HttpResponseParams; +import com.akto.dto.dependency_flow.DependencyFlow; import com.akto.dto.testing.TestingRunResult; import com.akto.dto.testing.YamlTestResult; import com.akto.dto.traffic.SampleData; @@ -41,6 +42,8 @@ import java.io.File; import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; import java.util.*; public class HarAction extends UserAction { @@ -54,11 +57,42 @@ public class HarAction extends UserAction { private byte[] tcpContent; private static final LoggerMaker loggerMaker = new LoggerMaker(HarAction.class); - public static void main(String[] args) { + public static void main(String[] args) throws IOException { + DaoInit.init(new ConnectionString("mongodb://localhost:27017")); + Context.accountId.set(1_000_000); + + DependencyFlow dependencyFlow = new DependencyFlow(); + dependencyFlow.run(); + dependencyFlow.syncWithDb(); + } + + public static void main3(String[] args) throws IOException { + DaoInit.init(new ConnectionString("mongodb://localhost:27017")); + Context.accountId.set(1_000_000); + + String filePath = "/Users/avneesh/Downloads/juiceshop_address_1.har"; + String content; + try { + content = new String(Files.readAllBytes(Paths.get(filePath))); + } catch (IOException e) { + e.printStackTrace(); + return; + } + + System.out.println(content.length()); + + HarAction harAction = new HarAction(); + harAction.setHarString(content); + harAction.setApiCollectionName("a3"); + harAction.skipKafka = true; + harAction.execute(); + } + + public static void main2(String[] args) { DaoInit.init(new ConnectionString("mongodb://localhost:27017/admini")); Context.accountId.set(1_000_000); - ApiInfo.ApiInfoKey apiInfoKey = new ApiInfo.ApiInfoKey(1710156663, "https://juiceshop.akto.io/rest/products/reviews", URLMethods.Method.PATCH); + ApiInfo.ApiInfoKey apiInfoKey = new ApiInfo.ApiInfoKey(1712980524, "https://juiceshop.akto.io/api/Cards/INTEGER", URLMethods.Method.DELETE); SampleData sampleData = SampleDataDao.instance.findOne( Filters.and( @@ -117,73 +151,29 @@ public static void main1(String[] args) { " - CVE-2023-4647\n" + " - CVE-2023-38254\n" + "\n" + - "workflow_selection_filters:\n" + - " API_1:\n" + - " response_code:\n" + - " gte: 200\n" + - " lt: 300\n" + - " url:\n" + - " contains_either:\n" + - " - rest/user/login\n" + - " API_2:\n" + - " response_code:\n" + - " gte: 200\n" + - " lt: 300\n" + - " url:\n" + - " contains_either:\n" + - " - rest/products/reviews\n" + "\n" + "api_selection_filters:\n" + " response_code:\n" + " gte: 200\n" + " lt: 300\n" + "\n" + - "wordLists:\n" + - " dummyHeaders:\n" + - " - a\n" + - "\n" + "execute:\n" + - " type: multiple\n" + + " type: graph\n" + " requests:\n" + " - req:\n" + - " - api: API_1\n" + - " - add_header:\n" + - " dummy_Header_Key1: \"dummyValue\"\n" + + " - replace_auth_header: true\n" + " - validate:\n" + " percentage_match:\n" + " gte: 90\n" + " - success: x2\n" + " - failure: x2\n" + - "\n" + " - req:\n" + - " - api: API_2\n" + - " - add_header:\n" + - " dummy_Header_Key2: \"dummyValue\"\n" + - " - validate:\n" + - " percentage_match:\n" + - " gte: 90\n" + - " - success: x3\n" + - " - failure: x3\n" + + " - api: get_asset_api\n" + + " - validate: \n" + + " response_code: 4xx\n" + + " success: vulnerability\n" + + " failure: exit\n" + "\n" + - " - req:\n" + - " - api: API_1\n" + - " - add_header:\n" + - " dummy_Header_Key3: \"dummyValue\"\n" + - " - validate:\n" + - " percentage_match:\n" + - " gte: 90\n" + - " - success: x4\n" + - " - failure: x4\n" + - "\n" + - " - req:\n" + - " - api: API_2\n" + - " - add_header:\n" + - " dummy_Header_Key4: \"dummyValue\"\n" + - " - validate:\n" + - " percentage_match:\n" + - " gte: 90\n" + - " - success: x5\n" + - " - failure: x5\n" + "\n" + "\n" + "validate:\n" + @@ -194,6 +184,7 @@ public static void main1(String[] args) { " - compare_greater:\n" + " - ${x2.response.stats.median_response_time}\n" + " - ${x1.response.stats.median_response_time} * 3"; + public String executeWithSkipKafka(boolean skipKafka) throws IOException { this.skipKafka = skipKafka; execute(); @@ -212,7 +203,7 @@ public String execute() throws IOException { apiCollection = ApiCollectionsDao.instance.findByName(apiCollectionName); if (apiCollection == null) { ApiCollectionsAction apiCollectionsAction = new ApiCollectionsAction(); - apiCollectionsAction.setSession(this.getSession()); +// apiCollectionsAction.setSession(this.getSession()); apiCollectionsAction.setCollectionName(apiCollectionName); String result = apiCollectionsAction.createCollection(); if (result.equalsIgnoreCase(Action.SUCCESS)) { @@ -269,16 +260,16 @@ public String execute() throws IOException { return ERROR.toUpperCase(); } - if (getSession().getOrDefault("utility","").equals(Utility.BURP.toString())) { - BurpPluginInfoDao.instance.updateLastDataSentTimestamp(getSUser().getLogin()); - } +// if (getSession().getOrDefault("utility","").equals(Utility.BURP.toString())) { +// BurpPluginInfoDao.instance.updateLastDataSentTimestamp(getSUser().getLogin()); +// } try { HAR har = new HAR(); loggerMaker.infoAndAddToDb("Har file upload processing for collectionId:" + apiCollectionId, LoggerMaker.LogDb.DASHBOARD); - String zippedString = GzipUtils.zipString(harString); - com.akto.dto.files.File file = new com.akto.dto.files.File(HttpResponseParams.Source.HAR.toString(),zippedString); - FilesDao.instance.insertOne(file); +// String zippedString = GzipUtils.zipString(harString); +// com.akto.dto.files.File file = new com.akto.dto.files.File(HttpResponseParams.Source.HAR.toString(),zippedString); +// FilesDao.instance.insertOne(file); List messages = har.getMessages(harString, apiCollectionId, Context.accountId.get()); harErrors = har.getErrors(); Utils.pushDataToKafka(apiCollectionId, topic, messages, harErrors, skipKafka); diff --git a/apps/testing/src/main/java/com/akto/test_editor/execution/Executor.java b/apps/testing/src/main/java/com/akto/test_editor/execution/Executor.java index c805125ff5..bc3e05c96b 100644 --- a/apps/testing/src/main/java/com/akto/test_editor/execution/Executor.java +++ b/apps/testing/src/main/java/com/akto/test_editor/execution/Executor.java @@ -145,7 +145,12 @@ public YamlTestResult execute(ExecutorNode node, RawApi rawApi, Map apiInfoKeys = new ArrayList<>(); + apiInfoKeys.add(apiInfoKey); + memory = new Memory(apiInfoKeys, new HashMap<>()); + } workflowTest = buildWorkflowGraph(reqNodes, rawApi, authMechanism, customAuthTypes, apiInfoKey, varMap, validatorNode); result.add(triggerMultiExecution(workflowTest, reqNodes, rawApi, authMechanism, customAuthTypes, apiInfoKey, varMap, validatorNode, debug, testLogs, memory, apiNameToApiInfoKey)); yamlTestResult = new YamlTestResult(result, workflowTest); diff --git a/apps/testing/src/main/java/com/akto/test_editor/execution/Memory.java b/apps/testing/src/main/java/com/akto/test_editor/execution/Memory.java index 6f08b49db1..a66cb2bcc2 100644 --- a/apps/testing/src/main/java/com/akto/test_editor/execution/Memory.java +++ b/apps/testing/src/main/java/com/akto/test_editor/execution/Memory.java @@ -1,5 +1,6 @@ package com.akto.test_editor.execution; +import com.akto.dao.DependencyFlowNodesDao; import com.akto.dao.SampleDataDao; import com.akto.dto.ApiInfo; import com.akto.dto.OriginalHttpRequest; @@ -9,7 +10,9 @@ import com.akto.dto.testing.TestingRunConfig; import com.akto.dto.traffic.Key; import com.akto.dto.traffic.SampleData; +import com.akto.dto.type.SingleTypeInfo; import com.akto.testing.ApiExecutor; +import com.akto.types.CappedSet; import com.akto.util.Constants; import com.mongodb.client.model.Filters; import org.bson.conversions.Bson; @@ -26,10 +29,41 @@ public class Memory { private final Map nodesMap = new HashMap<>(); Map sampleDataMap = new HashMap<>(); + private Map assetsMap = new HashMap<>(); private Map replaceDetailsMap = new HashMap<>(); + public void findAssets(ApiInfo.ApiInfoKey apiInfoKey) { + List results = new ArrayList<>(); + Node node = DependencyFlowNodesDao.instance.findOne( + Filters.and( + Filters.eq("apiCollectionId", apiInfoKey.getApiCollectionId()+""), + Filters.eq("url", apiInfoKey.getUrl()), + Filters.eq("method", apiInfoKey.getMethod().name()) + ) + ); + + if (node == null || node.getConnections() == null) return; + + Map connections = node.getConnections(); + + for (String key: connections.keySet()) { + Connection connection = connections.get(key); + if (connection == null) continue; + if (connection.getIsHeader()) continue; + + SingleTypeInfo.ParamId paramId = new SingleTypeInfo.ParamId(apiInfoKey.getUrl(), apiInfoKey.getMethod().name(), -1, connection.getIsHeader(), connection.getParam(), SingleTypeInfo.GENERIC, apiInfoKey.getApiCollectionId(), connection.getIsUrlParam()); + results.add(paramId); + } + + if (!results.isEmpty()) { + SingleTypeInfo.ParamId paramId = results.get(0); + assetsMap.put(apiInfoKey, paramId); + } + + } + public Memory(List apiInfoKeys, Map replaceDetailsMap) { if (apiInfoKeys == null || apiInfoKeys.isEmpty()) return; this.replaceDetailsMap = replaceDetailsMap; @@ -58,6 +92,8 @@ public Memory(List apiInfoKeys, Map } buildParentToChildMap(nodes, parentToChildMap); + + for (ApiInfo.ApiInfoKey apiInfoKey: apiInfoKeys) findAssets(apiInfoKey); } @@ -81,6 +117,53 @@ public OriginalHttpRequest run(int apiCollectionId, String url, String method) { return execute(sampleDataList); } + public RawApi findAssetGetterRequest(ApiInfo.ApiInfoKey apiInfoKey) { + SingleTypeInfo.ParamId paramId = assetsMap.get(apiInfoKey); + // find getter API + Node node = DependencyFlowNodesDao.instance.findOne( + Filters.and( + Filters.eq("apiCollectionId", apiInfoKey.getApiCollectionId()+""), + Filters.eq("url", apiInfoKey.getUrl()), + Filters.eq("method", apiInfoKey.getMethod().name()) + ) + ); + if (node == null || node.getConnections() == null) return null; + + Map connections = node.getConnections(); + + int apiCollectionId = 0; + String url = null; + String method = null; + + for (String key: connections.keySet()) { + Connection connection = connections.get(key); + if (connection == null) continue; + if (connection.getIsHeader()) continue; + + String connectionParam = connection.getParam(); + String param = paramId.getParam(); + if (!param.equals(connectionParam)) continue; + + if ((paramId.getIsUrlParam() && connection.getIsUrlParam()) || (!paramId.getIsUrlParam() || !connection.getIsUrlParam())) { + List edges = connection.getEdges(); + if (edges.isEmpty()) continue; + + Edge edge = edges.get(0); + apiCollectionId = Integer.parseInt(edge.getApiCollectionId()); + url = edge.getUrl(); + method = edge.getMethod(); + } + } + + if (url == null) return null; + + // find sample message from result map + int hash = Objects.hash(apiCollectionId+"", url, method); + + // return the request + return resultMap.get(hash); + } + private OriginalHttpRequest execute(List sdList) { int idx = 0; diff --git a/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/YamlNodeExecutor.java b/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/YamlNodeExecutor.java index e3d172782d..92f275b84d 100644 --- a/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/YamlNodeExecutor.java +++ b/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/YamlNodeExecutor.java @@ -7,6 +7,7 @@ import java.util.List; import java.util.Map; +import com.akto.dto.type.SingleTypeInfo; import com.akto.testing.Main; import com.akto.dto.*; import com.akto.dto.type.URLMethods; @@ -58,23 +59,26 @@ public NodeResult processNode(Node node, Map varMap, Boolean all } RawApi rawApi = yamlNodeDetails.getRawApi(); + RawApi sampleRawApi = rawApi.copy(); Executor executor = new Executor(); ExecutorNode executorNode = yamlNodeDetails.getExecutorNode(); FilterNode validatorNode = yamlNodeDetails.getValidatorNode(); List childNodes = executorNode.getChildNodes(); - ApiInfo.ApiInfoKey apiInfoKey = null; + ApiInfo.ApiInfoKey apiInfoKey = ((YamlNodeDetails) node.getWorkflowNodeDetails()).getApiInfoKey(); ExecutorNode firstChildNode = childNodes.get(0); // todo check for length - if (firstChildNode.getOperationType().equals("API")) { - String api = firstChildNode.getValues().toString(); - apiInfoKey = apiNameToApiInfoKey.get(api); - OriginalHttpRequest newRequest = memory.run(apiInfoKey.getApiCollectionId(), apiInfoKey.getUrl(), apiInfoKey.getMethod().name()); - rawApi.setRequest(newRequest); + if (firstChildNode.getOperationType().equalsIgnoreCase("api")) { + String apiType = firstChildNode.getValues().toString(); + if (apiType.equalsIgnoreCase("get_asset_api")) { + rawApi = memory.findAssetGetterRequest(apiInfoKey); + } childNodes.remove(0); + } else { + OriginalHttpRequest request = memory.run(apiInfoKey.getApiCollectionId(), apiInfoKey.getUrl(), apiInfoKey.getMethod().name()); + rawApi.setRequest(request); } - RawApi sampleRawApi = rawApi.copy(); List rawApis = new ArrayList<>(); rawApis.add(rawApi.copy()); diff --git a/apps/testing/src/main/java/com/akto/testing/yaml_tests/YamlTestTemplate.java b/apps/testing/src/main/java/com/akto/testing/yaml_tests/YamlTestTemplate.java index 8c58a12f71..fbc6c5c0b4 100644 --- a/apps/testing/src/main/java/com/akto/testing/yaml_tests/YamlTestTemplate.java +++ b/apps/testing/src/main/java/com/akto/testing/yaml_tests/YamlTestTemplate.java @@ -58,32 +58,6 @@ public boolean filter() { @Override public boolean workflowFilter() { - Map result = new HashMap<>(); - int skip = 0; - int limit = 1000; - while (true) { - List sampleDataList = SampleDataDao.instance.findAll(new BasicDBObject(), skip, limit, Sorts.ascending("_id.url"));// todo: can be improved - for (SampleData sampleData: sampleDataList) { - RawApi localRawApi = RawApi.buildFromMessage(sampleData.getSamples().get(0)); - ApiInfo.ApiInfoKey localApiInfoKey = new ApiInfo.ApiInfoKey(sampleData.getId().getApiCollectionId(), sampleData.getId().getUrl(), sampleData.getId().getMethod()); - for (String key: this.workFlowSelectionFilters.keySet()) { - if (result.containsKey(key)) continue; - ConfigParserResult configParserResult = this.workFlowSelectionFilters.get(key); - FilterNode localFilterNode = configParserResult.getNode(); - boolean isValid = TestPlugin.validateFilter(localFilterNode, localRawApi, localApiInfoKey, this.varMap, this.logId); - if (isValid) { - result.put(key, localApiInfoKey); - break; - } - } - if (result.size() == this.workFlowSelectionFilters.size()) break; - } - - if (sampleDataList.size() < limit || result.size() == this.workFlowSelectionFilters.size()) break; - } - - memory = new Memory(new ArrayList<>(result.values()), new HashMap<>()); - this.apiNameToApiInfoKey = result; return true; } From 68e9ceb0fd82ce1c11236fad0ec5f8b9734e7ab2 Mon Sep 17 00:00:00 2001 From: Ankush Jain <91221068+ankush-jain-akto@users.noreply.github.com> Date: Tue, 16 Apr 2024 14:21:02 +0530 Subject: [PATCH 05/16] remove mac-os --- .github/workflows/codeql.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 809055f635..42a88e99cf 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -14,16 +14,13 @@ name: "CodeQL" on: push: branches: [ "master" ] - pull_request: - # The branches below must be a subset of the branches above - branches: [ "master" ] schedule: - cron: '22 0 * * 1' jobs: analyze: name: Analyze - runs-on: ${{ (matrix.language == 'java' && 'macos-latest') || 'ubuntu-latest' }} + runs-on: 'ubuntu-latest' timeout-minutes: ${{ (matrix.language == 'java' && 120) || 360 }} permissions: actions: read From 674d2cdc00eb4fc38f24e73ccf8b4e7c6104a383 Mon Sep 17 00:00:00 2001 From: Avneesh Hota Date: Tue, 16 Apr 2024 14:46:16 +0530 Subject: [PATCH 06/16] code cleanup --- .../akto/dependency/DependencyAnalyser.java | 2 +- .../src/main/java/com/akto/utils/Utils.java | 1 + .../akto/test_editor/execution/Executor.java | 8 ++-- .../com/akto/testing/ApiWorkflowExecutor.java | 4 +- .../java/com/akto/testing/TestExecutor.java | 5 +-- .../ApiNodeExecutor.java | 2 +- .../ConditionalGraphExecutor.java | 6 +-- .../workflow_node_executor/GraphExecutor.java | 2 +- .../LinearGraphExecutor.java | 4 +- .../workflow_node_executor/NodeExecutor.java | 2 +- .../testing/workflow_node_executor/Utils.java | 14 +++---- .../YamlNodeExecutor.java | 20 +++++---- .../yaml_tests/SecurityTestTemplate.java | 42 ++++--------------- .../testing/yaml_tests/YamlTestTemplate.java | 18 ++------ .../dao/test_editor/TestConfigYamlParser.java | 33 +++++---------- .../com/akto/dto/test_editor/TestConfig.java | 12 +----- 16 files changed, 59 insertions(+), 116 deletions(-) diff --git a/apps/api-runtime/src/main/java/com/akto/dependency/DependencyAnalyser.java b/apps/api-runtime/src/main/java/com/akto/dependency/DependencyAnalyser.java index 9e1f57b5c9..71a90833cd 100644 --- a/apps/api-runtime/src/main/java/com/akto/dependency/DependencyAnalyser.java +++ b/apps/api-runtime/src/main/java/com/akto/dependency/DependencyAnalyser.java @@ -238,7 +238,7 @@ public boolean filterValues(Object val) { if (val == null) return false; if (val instanceof Boolean) return false; if (val instanceof String) return val.toString().length() > 4 && val.toString().length() <= 4096; - if (val instanceof Integer) return ((int) val) > 0; + if (val instanceof Integer) return ((int) val) > 50; return true; } diff --git a/apps/dashboard/src/main/java/com/akto/utils/Utils.java b/apps/dashboard/src/main/java/com/akto/utils/Utils.java index 0223ddb394..23ed12f4b8 100644 --- a/apps/dashboard/src/main/java/com/akto/utils/Utils.java +++ b/apps/dashboard/src/main/java/com/akto/utils/Utils.java @@ -470,6 +470,7 @@ public static void pushDataToKafka(int apiCollectionId, String topic, List varMap, String logId, AuthMechanism authMechanism, FilterNode validatorNode, ApiInfo.ApiInfoKey apiInfoKey, TestingRunConfig testingRunConfig, List customAuthTypes, boolean debug, List testLogs, - Memory memory, Map apiNameToApiInfoKey) { + Memory memory) { List result = new ArrayList<>(); ExecutionListBuilder executionListBuilder = new ExecutionListBuilder(); @@ -152,7 +152,7 @@ public YamlTestResult execute(ExecutorNode node, RawApi rawApi, Map()); } workflowTest = buildWorkflowGraph(reqNodes, rawApi, authMechanism, customAuthTypes, apiInfoKey, varMap, validatorNode); - result.add(triggerMultiExecution(workflowTest, reqNodes, rawApi, authMechanism, customAuthTypes, apiInfoKey, varMap, validatorNode, debug, testLogs, memory, apiNameToApiInfoKey)); + result.add(triggerMultiExecution(workflowTest, reqNodes, rawApi, authMechanism, customAuthTypes, apiInfoKey, varMap, validatorNode, debug, testLogs, memory)); yamlTestResult = new YamlTestResult(result, workflowTest); return yamlTestResult; @@ -264,7 +264,7 @@ public WorkflowTest buildWorkflowGraph(ExecutorNode reqNodes, RawApi rawApi, Aut } public MultiExecTestResult triggerMultiExecution(WorkflowTest workflowTest, ExecutorNode reqNodes, RawApi rawApi, AuthMechanism authMechanism, - List customAuthTypes, ApiInfo.ApiInfoKey apiInfoKey, Map varMap, FilterNode validatorNode, boolean debug, List testLogs, Memory memory, Map apiNameToApiInfoKey) { + List customAuthTypes, ApiInfo.ApiInfoKey apiInfoKey, Map varMap, FilterNode validatorNode, boolean debug, List testLogs, Memory memory) { ApiWorkflowExecutor apiWorkflowExecutor = new ApiWorkflowExecutor(); Graph graph = new Graph(); @@ -273,7 +273,7 @@ public MultiExecTestResult triggerMultiExecution(WorkflowTest workflowTest, Exec List executionOrder = new ArrayList<>(); WorkflowTestResult workflowTestResult = new WorkflowTestResult(id, workflowTest.getId(), new HashMap<>(), null, null); GraphExecutorRequest graphExecutorRequest = new GraphExecutorRequest(graph, graph.getNode("x1"), workflowTest, null, null, varMap, "conditional", workflowTestResult, new HashMap<>(), executionOrder); - GraphExecutorResult graphExecutorResult = apiWorkflowExecutor.init(graphExecutorRequest, debug, testLogs, memory, apiNameToApiInfoKey); + GraphExecutorResult graphExecutorResult = apiWorkflowExecutor.init(graphExecutorRequest, debug, testLogs, memory); return new MultiExecTestResult(graphExecutorResult.getWorkflowTestResult().getNodeResultMap(), graphExecutorResult.getVulnerable(), Confidence.HIGH, graphExecutorRequest.getExecutionOrder()); } diff --git a/apps/testing/src/main/java/com/akto/testing/ApiWorkflowExecutor.java b/apps/testing/src/main/java/com/akto/testing/ApiWorkflowExecutor.java index 194103dba4..caaa6fa05c 100644 --- a/apps/testing/src/main/java/com/akto/testing/ApiWorkflowExecutor.java +++ b/apps/testing/src/main/java/com/akto/testing/ApiWorkflowExecutor.java @@ -11,9 +11,9 @@ public class ApiWorkflowExecutor { - public GraphExecutorResult init(GraphExecutorRequest graphExecutorRequest, boolean debug, List testLogs, Memory memory, Map apiNameToApiInfoKey) { + public GraphExecutorResult init(GraphExecutorRequest graphExecutorRequest, boolean debug, List testLogs, Memory memory) { GraphExecutor graphExecutor = GraphExecutorFactory.fetchExecutor(graphExecutorRequest); - GraphExecutorResult graphExecutorResult = graphExecutor.executeGraph(graphExecutorRequest,debug,testLogs, memory, apiNameToApiInfoKey); + GraphExecutorResult graphExecutorResult = graphExecutor.executeGraph(graphExecutorRequest,debug,testLogs, memory); return graphExecutorResult; } diff --git a/apps/testing/src/main/java/com/akto/testing/TestExecutor.java b/apps/testing/src/main/java/com/akto/testing/TestExecutor.java index b019b03ecd..d81fe068cb 100644 --- a/apps/testing/src/main/java/com/akto/testing/TestExecutor.java +++ b/apps/testing/src/main/java/com/akto/testing/TestExecutor.java @@ -107,7 +107,7 @@ public void workflowInit (TestingRun testingRun, ObjectId summaryId, boolean deb Graph graph = new Graph(); graph.buildGraph(workflowTest); GraphExecutorRequest graphExecutorRequest = new GraphExecutorRequest(graph, workflowTest, testingRun.getId(), summaryId, valuesMap, false, "linear"); - GraphExecutorResult graphExecutorResult = apiWorkflowExecutor.init(graphExecutorRequest, debug, testLogs, null, null); + GraphExecutorResult graphExecutorResult = apiWorkflowExecutor.init(graphExecutorRequest, debug, testLogs, null); WorkflowTestResultsDao.instance.insertOne(graphExecutorResult.getWorkflowTestResult()); } catch (Exception e) { loggerMaker.errorAndAddToDb("Error while executing workflow test " + e, LogDb.TESTING); @@ -651,7 +651,6 @@ public TestingRunResult runTestNew(ApiInfo.ApiInfoKey apiInfoKey, ObjectId testR } FilterNode filterNode = testConfig.getApiSelectionFilters().getNode(); - Map workFlowSelectionFilters = testConfig.getWorkFlowSelectionFilters(); FilterNode validatorNode = null; if (testConfig.getValidation() != null) { validatorNode = testConfig.getValidation().getNode(); @@ -677,7 +676,7 @@ public TestingRunResult runTestNew(ApiInfo.ApiInfoKey apiInfoKey, ObjectId testR // TestingUtil -> authMechanism // TestingConfig -> auth YamlTestTemplate yamlTestTemplate = new YamlTestTemplate(apiInfoKey,filterNode, validatorNode, executorNode, - rawApi, varMap, auth, testingUtil.getAuthMechanism(), testExecutionLogId, testingRunConfig, customAuthTypes, testConfig.getStrategy(), workFlowSelectionFilters); + rawApi, varMap, auth, testingUtil.getAuthMechanism(), testExecutionLogId, testingRunConfig, customAuthTypes, testConfig.getStrategy()); YamlTestResult testResults = yamlTestTemplate.run(debug, testLogs); if (testResults == null || testResults.getTestResults().isEmpty()) { List res = new ArrayList<>(); diff --git a/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/ApiNodeExecutor.java b/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/ApiNodeExecutor.java index 8b67217871..92645aa6c6 100644 --- a/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/ApiNodeExecutor.java +++ b/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/ApiNodeExecutor.java @@ -26,7 +26,7 @@ public class ApiNodeExecutor extends NodeExecutor { private static final LoggerMaker loggerMaker = new LoggerMaker(ApiNodeExecutor.class); - public NodeResult processNode(Node node, Map valuesMap, Boolean allowAllStatusCodes, boolean debug, List testLogs, Memory memory, Map apiNameToApiInfoKey) { + public NodeResult processNode(Node node, Map valuesMap, Boolean allowAllStatusCodes, boolean debug, List testLogs, Memory memory) { loggerMaker.infoAndAddToDb("\n", LogDb.TESTING); loggerMaker.infoAndAddToDb("NODE: " + node.getId(), LogDb.TESTING); List testErrors = new ArrayList<>(); diff --git a/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/ConditionalGraphExecutor.java b/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/ConditionalGraphExecutor.java index afc598d5a9..80d68cc439 100644 --- a/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/ConditionalGraphExecutor.java +++ b/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/ConditionalGraphExecutor.java @@ -16,7 +16,7 @@ public class ConditionalGraphExecutor extends GraphExecutor { - public GraphExecutorResult executeGraph(GraphExecutorRequest graphExecutorRequest, boolean debug, List testLogs, Memory memory, Map apiNameToApiInfoKey) { + public GraphExecutorResult executeGraph(GraphExecutorRequest graphExecutorRequest, boolean debug, List testLogs, Memory memory) { Map visitedMap = graphExecutorRequest.getVisitedMap(); List errors = new ArrayList<>(); @@ -32,7 +32,7 @@ public GraphExecutorResult executeGraph(GraphExecutorRequest graphExecutorReques boolean success = false; WorkflowTestResult.NodeResult nodeResult; - nodeResult = Utils.executeNode(node, graphExecutorRequest.getValuesMap(), debug, testLogs, memory, apiNameToApiInfoKey); + nodeResult = Utils.executeNode(node, graphExecutorRequest.getValuesMap(), debug, testLogs, memory); graphExecutorRequest.getWorkflowTestResult().getNodeResultMap().put(node.getId(), nodeResult); graphExecutorRequest.getExecutionOrder().add(node.getId()); @@ -79,7 +79,7 @@ public GraphExecutorResult executeGraph(GraphExecutorRequest graphExecutorReques boolean vulnerable = success; if (childNode != null) { GraphExecutorRequest childExecReq = new GraphExecutorRequest(graphExecutorRequest, childNode, graphExecutorRequest.getWorkflowTestResult(), visitedMap, graphExecutorRequest.getExecutionOrder()); - GraphExecutorResult childExecResult = executeGraph(childExecReq, debug, testLogs, memory, apiNameToApiInfoKey); + GraphExecutorResult childExecResult = executeGraph(childExecReq, debug, testLogs, memory); vulnerable = childExecResult.getVulnerable(); return new GraphExecutorResult(graphExecutorRequest.getWorkflowTestResult(), vulnerable, errors); } else { diff --git a/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/GraphExecutor.java b/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/GraphExecutor.java index 5e6ab7f15a..cc876d2a3c 100644 --- a/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/GraphExecutor.java +++ b/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/GraphExecutor.java @@ -11,6 +11,6 @@ public abstract class GraphExecutor { - public abstract GraphExecutorResult executeGraph(GraphExecutorRequest graphExecutorRequest, boolean debug, List testLogs, Memory memory, Map apiNameToApiInfoKey); + public abstract GraphExecutorResult executeGraph(GraphExecutorRequest graphExecutorRequest, boolean debug, List testLogs, Memory memory); } diff --git a/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/LinearGraphExecutor.java b/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/LinearGraphExecutor.java index 95ad59a8aa..5cc7ace500 100644 --- a/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/LinearGraphExecutor.java +++ b/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/LinearGraphExecutor.java @@ -16,7 +16,7 @@ public class LinearGraphExecutor extends GraphExecutor { - public GraphExecutorResult executeGraph(GraphExecutorRequest graphExecutorRequest, boolean debug, List testLogs, Memory memory, Map apiNameToApiInfoKey) { + public GraphExecutorResult executeGraph(GraphExecutorRequest graphExecutorRequest, boolean debug, List testLogs, Memory memory) { List nodes = graphExecutorRequest.getGraph().sort(); int id = Context.now(); @@ -24,7 +24,7 @@ public GraphExecutorResult executeGraph(GraphExecutorRequest graphExecutorReques Map testResultMap = workflowTestResult.getNodeResultMap(); for (Node node: nodes) { WorkflowTestResult.NodeResult nodeResult; - nodeResult = Utils.executeNode(node, graphExecutorRequest.getValuesMap(), debug, testLogs, memory, apiNameToApiInfoKey); + nodeResult = Utils.executeNode(node, graphExecutorRequest.getValuesMap(), debug, testLogs, memory); testResultMap.put(node.getId(), nodeResult); if (nodeResult.getErrors().size() > 0) break; if (graphExecutorRequest.getSkipIfNotVulnerable() && !nodeResult.isVulnerable()) { diff --git a/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/NodeExecutor.java b/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/NodeExecutor.java index 63a1415d5e..68e5b00c00 100644 --- a/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/NodeExecutor.java +++ b/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/NodeExecutor.java @@ -11,5 +11,5 @@ public abstract class NodeExecutor { - public abstract WorkflowTestResult.NodeResult processNode(Node node, Map valuesMap, Boolean allowAllStatusCodes, boolean debug, List testLogs, Memory memory, Map apiNameToApiInfoKey); + public abstract WorkflowTestResult.NodeResult processNode(Node node, Map valuesMap, Boolean allowAllStatusCodes, boolean debug, List testLogs, Memory memory); } diff --git a/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/Utils.java b/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/Utils.java index 8889e0c8bc..216d4afec7 100644 --- a/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/Utils.java +++ b/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/Utils.java @@ -189,7 +189,7 @@ public static String fetchToken(RecordedLoginFlowInput recordedLoginFlowInput, i return token; } - public static WorkflowTestResult.NodeResult processNode(Node node, Map valuesMap, Boolean allowAllStatusCodes, boolean debug, List testLogs, Memory memory, Map apiNameToApiInfoKey) { + public static WorkflowTestResult.NodeResult processNode(Node node, Map valuesMap, Boolean allowAllStatusCodes, boolean debug, List testLogs, Memory memory) { if (node.getWorkflowNodeDetails().getType() == WorkflowNodeDetails.Type.RECORDED) { return processRecorderNode(node, valuesMap); } @@ -197,22 +197,22 @@ else if (node.getWorkflowNodeDetails().getType() == WorkflowNodeDetails.Type.OTP return processOtpNode(node, valuesMap); } else { - return processApiNode(node, valuesMap, allowAllStatusCodes, debug, testLogs, memory, apiNameToApiInfoKey); + return processApiNode(node, valuesMap, allowAllStatusCodes, debug, testLogs, memory); } } - public static WorkflowTestResult.NodeResult processApiNode(Node node, Map valuesMap, Boolean allowAllStatusCodes, boolean debug, List testLogs, Memory memory, Map apiNameToApiInfoKey) { + public static WorkflowTestResult.NodeResult processApiNode(Node node, Map valuesMap, Boolean allowAllStatusCodes, boolean debug, List testLogs, Memory memory) { NodeExecutorFactory nodeExecutorFactory = new NodeExecutorFactory(); NodeExecutor nodeExecutor = nodeExecutorFactory.getExecutor(node); - return nodeExecutor.processNode(node, valuesMap, allowAllStatusCodes, debug, testLogs, memory, apiNameToApiInfoKey); + return nodeExecutor.processNode(node, valuesMap, allowAllStatusCodes, debug, testLogs, memory); } - public static WorkflowTestResult.NodeResult executeNode(Node node, Map valuesMap,boolean debug, List testLogs, Memory memory, Map apiNameToApiInfoKey) { + public static WorkflowTestResult.NodeResult executeNode(Node node, Map valuesMap,boolean debug, List testLogs, Memory memory) { WorkflowTestResult.NodeResult nodeResult; try { - nodeResult = Utils.processNode(node, valuesMap, true, debug, testLogs, memory, apiNameToApiInfoKey); + nodeResult = Utils.processNode(node, valuesMap, true, debug, testLogs, memory); } catch (Exception e) { ; List testErrors = new ArrayList<>(); @@ -246,7 +246,7 @@ public static LoginFlowResponse runLoginFlow(WorkflowTest workflowTest, AuthMech if (authMechanism.getRequestData() != null && authMechanism.getRequestData().size() > 0 && authMechanism.getRequestData().get(index).getAllowAllStatusCodes()) { allowAllStatusCodes = authMechanism.getRequestData().get(0).getAllowAllStatusCodes(); } - nodeResult = processNode(node, valuesMap, allowAllStatusCodes, false, new ArrayList<>(), null, null); + nodeResult = processNode(node, valuesMap, allowAllStatusCodes, false, new ArrayList<>(), null); } catch (Exception e) { ; List testErrors = new ArrayList<>(); diff --git a/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/YamlNodeExecutor.java b/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/YamlNodeExecutor.java index 934737a3d5..0f16a8fb53 100644 --- a/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/YamlNodeExecutor.java +++ b/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/YamlNodeExecutor.java @@ -50,7 +50,7 @@ public class YamlNodeExecutor extends NodeExecutor { private static final Gson gson = new Gson(); - public NodeResult processNode(Node node, Map varMap, Boolean allowAllStatusCodes, boolean debug, List testLogs, Memory memory, Map apiNameToApiInfoKey) { + public NodeResult processNode(Node node, Map varMap, Boolean allowAllStatusCodes, boolean debug, List testLogs, Memory memory) { List testErrors = new ArrayList<>(); YamlNodeDetails yamlNodeDetails = (YamlNodeDetails) node.getWorkflowNodeDetails(); @@ -69,15 +69,17 @@ public NodeResult processNode(Node node, Map varMap, Boolean all ApiInfo.ApiInfoKey apiInfoKey = ((YamlNodeDetails) node.getWorkflowNodeDetails()).getApiInfoKey(); ExecutorNode firstChildNode = childNodes.get(0); // todo check for length - if (firstChildNode.getOperationType().equalsIgnoreCase("api")) { - String apiType = firstChildNode.getValues().toString(); - if (apiType.equalsIgnoreCase("get_asset_api")) { - rawApi = memory.findAssetGetterRequest(apiInfoKey); + if (memory != null) { + if (firstChildNode.getOperationType().equalsIgnoreCase("api")) { + String apiType = firstChildNode.getValues().toString(); + if (apiType.equalsIgnoreCase("get_asset_api")) { + rawApi = memory.findAssetGetterRequest(apiInfoKey); + } + childNodes.remove(0); + } else { + OriginalHttpRequest request = memory.run(apiInfoKey.getApiCollectionId(), apiInfoKey.getUrl(), apiInfoKey.getMethod().name()); + rawApi.setRequest(request); } - childNodes.remove(0); - } else { - OriginalHttpRequest request = memory.run(apiInfoKey.getApiCollectionId(), apiInfoKey.getUrl(), apiInfoKey.getMethod().name()); - rawApi.setRequest(request); } diff --git a/apps/testing/src/main/java/com/akto/testing/yaml_tests/SecurityTestTemplate.java b/apps/testing/src/main/java/com/akto/testing/yaml_tests/SecurityTestTemplate.java index 47db26f346..9027116dbf 100644 --- a/apps/testing/src/main/java/com/akto/testing/yaml_tests/SecurityTestTemplate.java +++ b/apps/testing/src/main/java/com/akto/testing/yaml_tests/SecurityTestTemplate.java @@ -2,12 +2,18 @@ import com.akto.dto.ApiInfo; import com.akto.dto.RawApi; -import com.akto.dto.test_editor.*; +import com.akto.dto.test_editor.Auth; +import com.akto.dto.test_editor.ExecutorNode; +import com.akto.dto.test_editor.FilterNode; +import com.akto.dto.test_editor.Strategy; import com.akto.dto.testing.*; import com.akto.dto.testing.TestResult.TestError; import com.akto.test_editor.execution.Memory; -import java.util.*; +import java.util.Collections; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; import static com.akto.dto.testing.TestResult.TestError.*; @@ -26,13 +32,9 @@ public abstract class SecurityTestTemplate { TestingRunConfig testingRunConfig; Strategy strategy; - Map apiNameToApiInfoKey = new HashMap<>(); - - Map workFlowSelectionFilters; - Memory memory; - public SecurityTestTemplate(ApiInfo.ApiInfoKey apiInfoKey, FilterNode filterNode, FilterNode validatorNode, ExecutorNode executorNode ,RawApi rawApi, Map varMap, Auth auth, AuthMechanism authMechanism, String logId, TestingRunConfig testingRunConfig, Strategy strategy, Map workFlowSelectionFilters) { + public SecurityTestTemplate(ApiInfo.ApiInfoKey apiInfoKey, FilterNode filterNode, FilterNode validatorNode, ExecutorNode executorNode ,RawApi rawApi, Map varMap, Auth auth, AuthMechanism authMechanism, String logId, TestingRunConfig testingRunConfig, Strategy strategy) { this.apiInfoKey = apiInfoKey; this.filterNode = filterNode; this.validatorNode = validatorNode; @@ -44,13 +46,10 @@ public SecurityTestTemplate(ApiInfo.ApiInfoKey apiInfoKey, FilterNode filterNode this.logId = logId; this.testingRunConfig = testingRunConfig; this.strategy = strategy; - this.workFlowSelectionFilters = workFlowSelectionFilters; } public abstract boolean filter(); - public abstract boolean workflowFilter(); - public abstract boolean checkAuthBeforeExecution(boolean debug, List testLogs); public abstract YamlTestResult executor(boolean debug, List testLogs); @@ -71,14 +70,6 @@ public YamlTestResult run(boolean debug, List testLogs testResults.add(new TestResult(null, rawApi.getOriginalMessage(), Collections.singletonList(SKIPPING_EXECUTION_BECAUSE_AUTH.getMessage()), 0, false, TestResult.Confidence.HIGH, null)); return new YamlTestResult(testResults, null); } - - boolean workflowFound = workflowFilter(); - if (!workflowFound) { - List testResults = new ArrayList<>(); - testResults.add(new TestResult(null, rawApi.getOriginalMessage(), Collections.singletonList("Request API failed to satisfy workflow_selection_filters block, skipping execution"), 0, false, TestResult.Confidence.HIGH, null)); - return new YamlTestResult(testResults, null); - } - YamlTestResult attempts = executor(debug, testLogs); if(attempts == null || attempts.getTestResults().isEmpty()){ List res = new ArrayList<>(); @@ -161,19 +152,4 @@ public void setLogId(String logId) { this.logId = logId; } - public Map getApiNameToApiInfoKey() { - return apiNameToApiInfoKey; - } - - public void setApiNameToApiInfoKey(Map apiNameToApiInfoKey) { - this.apiNameToApiInfoKey = apiNameToApiInfoKey; - } - - public Map getWorkFlowSelectionFilters() { - return workFlowSelectionFilters; - } - - public void setWorkFlowSelectionFilters(Map workFlowSelectionFilters) { - this.workFlowSelectionFilters = workFlowSelectionFilters; - } } diff --git a/apps/testing/src/main/java/com/akto/testing/yaml_tests/YamlTestTemplate.java b/apps/testing/src/main/java/com/akto/testing/yaml_tests/YamlTestTemplate.java index fbc6c5c0b4..1674328e6b 100644 --- a/apps/testing/src/main/java/com/akto/testing/yaml_tests/YamlTestTemplate.java +++ b/apps/testing/src/main/java/com/akto/testing/yaml_tests/YamlTestTemplate.java @@ -1,25 +1,17 @@ package com.akto.testing.yaml_tests; -import com.akto.dao.SampleDataDao; import com.akto.dto.ApiInfo; import com.akto.dto.CustomAuthType; import com.akto.dto.OriginalHttpResponse; import com.akto.dto.RawApi; import com.akto.dto.test_editor.*; import com.akto.dto.testing.*; -import com.akto.dto.traffic.SampleData; import com.akto.log.LoggerMaker; import com.akto.rules.TestPlugin; import com.akto.test_editor.auth.AuthValidator; import com.akto.test_editor.execution.Executor; -import com.akto.test_editor.execution.Memory; import com.akto.testing.StatusCodeAnalyser; -import com.mongodb.BasicDBObject; -import com.mongodb.client.model.Projections; -import com.mongodb.client.model.Sorts; -import java.util.ArrayList; -import java.util.HashMap; import java.util.List; import java.util.Map; @@ -30,8 +22,8 @@ public class YamlTestTemplate extends SecurityTestTemplate { public YamlTestTemplate(ApiInfo.ApiInfoKey apiInfoKey, FilterNode filterNode, FilterNode validatorNode, ExecutorNode executorNode, RawApi rawApi, Map varMap, Auth auth, AuthMechanism authMechanism, String logId, TestingRunConfig testingRunConfig, - List customAuthTypes, Strategy strategy, Map workFlowSelectionFilters) { - super(apiInfoKey, filterNode, validatorNode, executorNode ,rawApi, varMap, auth, authMechanism, logId, testingRunConfig, strategy, workFlowSelectionFilters); + List customAuthTypes, Strategy strategy) { + super(apiInfoKey, filterNode, validatorNode, executorNode ,rawApi, varMap, auth, authMechanism, logId, testingRunConfig, strategy); this.customAuthTypes = customAuthTypes; } @@ -56,10 +48,6 @@ public boolean filter() { return isValid; } - @Override - public boolean workflowFilter() { - return true; - } @Override public boolean checkAuthBeforeExecution(boolean debug, List testLogs) { @@ -83,7 +71,7 @@ public YamlTestResult executor(boolean debug, List tes // loggerMaker.infoAndAddToDb("executor started" + logId, LogDb.TESTING); YamlTestResult results = new Executor().execute(this.executorNode, this.rawApi, this.varMap, this.logId, this.authMechanism, this.validatorNode, this.apiInfoKey, this.testingRunConfig, this.customAuthTypes, - debug, testLogs, memory, apiNameToApiInfoKey); + debug, testLogs, memory); // loggerMaker.infoAndAddToDb("execution result size " + results.size() + " " + logId, LogDb.TESTING); return results; } diff --git a/libs/dao/src/main/java/com/akto/dao/test_editor/TestConfigYamlParser.java b/libs/dao/src/main/java/com/akto/dao/test_editor/TestConfigYamlParser.java index 835670f192..2406e56ecb 100644 --- a/libs/dao/src/main/java/com/akto/dao/test_editor/TestConfigYamlParser.java +++ b/libs/dao/src/main/java/com/akto/dao/test_editor/TestConfigYamlParser.java @@ -1,6 +1,5 @@ package com.akto.dao.test_editor; -import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -57,33 +56,21 @@ public static TestConfig parseConfig(Map config) throws Exceptio Parser authParser = new Parser(); auth = authParser.parse(authMap); if (auth == null) { - return new TestConfig(id, info, null, null, null, null, null, null, null); + return new TestConfig(id, info, null, null, null, null, null, null); } } Object filterMap = config.get("api_selection_filters"); if (filterMap == null) { // todo: should not be null, throw error - return new TestConfig(id, info, auth, null, null, null, null, null, null); - } - - Object workflowSelectionFilterObj = config.get("workflow_selection_filters"); - Map workFlowSelectionFilters = new HashMap<>(); - if (workflowSelectionFilterObj != null) { - Map workflowSelectionFilterMap = (Map) workflowSelectionFilterObj; - for (String apiName: workflowSelectionFilterMap.keySet()) { - Object o = workflowSelectionFilterMap.get(apiName); - ConfigParserResult childFilter = new ConfigParser().parse(o); - workFlowSelectionFilters.put(apiName, childFilter); - } - // todo: should not be null, throw error + return new TestConfig(id, info, auth, null, null, null, null, null); } ConfigParser configParser = new ConfigParser(); ConfigParserResult filters = configParser.parse(filterMap); if (filters == null) { // todo: throw error - new TestConfig(id, info, auth, null, null, null, null, null, workFlowSelectionFilters); + new TestConfig(id, info, auth, null, null, null, null, null); } Map> wordListMap = new HashMap<>(); @@ -92,44 +79,44 @@ public static TestConfig parseConfig(Map config) throws Exceptio wordListMap = (Map) config.get("wordLists"); } } catch (Exception e) { - return new TestConfig(id, info, null, null, null, null, null, null, workFlowSelectionFilters); + return new TestConfig(id, info, null, null, null, null, null, null); } Object executionMap = config.get("execute"); if (executionMap == null) { // todo: should not be null, throw error - return new TestConfig(id, info, auth, filters, wordListMap, null, null, null, workFlowSelectionFilters); + return new TestConfig(id, info, auth, filters, wordListMap, null, null, null); } com.akto.dao.test_editor.executor.ConfigParser executorConfigParser = new com.akto.dao.test_editor.executor.ConfigParser(); ExecutorConfigParserResult executeOperations = executorConfigParser.parseConfigMap(executionMap); if (executeOperations == null) { // todo: throw error - new TestConfig(id, info, auth, filters, wordListMap, null, null, null, workFlowSelectionFilters); + new TestConfig(id, info, auth, filters, wordListMap, null, null, null); } Object validationMap = config.get("validate"); if (validationMap == null) { // todo: should not be null, throw error - return new TestConfig(id, info, auth, filters, wordListMap, executeOperations, null, null, workFlowSelectionFilters); + return new TestConfig(id, info, auth, filters, wordListMap, executeOperations, null, null); } ConfigParserResult validations = configParser.parse(validationMap); if (validations == null) { // todo: throw error - new TestConfig(id, info, auth, filters, wordListMap, executeOperations, null, null, workFlowSelectionFilters); + new TestConfig(id, info, auth, filters, wordListMap, executeOperations, null, null); } Object strategyObject = config.get("strategy"); if (strategyObject == null) { - return new TestConfig(id, info, auth, filters, wordListMap, executeOperations, validations, null, workFlowSelectionFilters); + return new TestConfig(id, info, auth, filters, wordListMap, executeOperations, validations, null); } StrategyParser strategyParser = new StrategyParser(); Strategy strategy = strategyParser.parse(strategyObject); - testConfig = new TestConfig(id, info, auth, filters, wordListMap, executeOperations, validations, strategy, workFlowSelectionFilters); + testConfig = new TestConfig(id, info, auth, filters, wordListMap, executeOperations, validations, strategy); return testConfig; } diff --git a/libs/dao/src/main/java/com/akto/dto/test_editor/TestConfig.java b/libs/dao/src/main/java/com/akto/dto/test_editor/TestConfig.java index 2f4d77fb1b..09bb3c8d13 100644 --- a/libs/dao/src/main/java/com/akto/dto/test_editor/TestConfig.java +++ b/libs/dao/src/main/java/com/akto/dto/test_editor/TestConfig.java @@ -14,7 +14,6 @@ public class TestConfig { private Auth auth; private ConfigParserResult apiSelectionFilters; - private Map workFlowSelectionFilters; private Map> wordlists; @@ -30,7 +29,7 @@ public class TestConfig { private String author; public TestConfig(String id, Info info, Auth auth, ConfigParserResult apiSelectionFilters, Map> wordlists, ExecutorConfigParserResult execute, - ConfigParserResult validation, Strategy strategy, Map workFlowSelectionFilters) { + ConfigParserResult validation, Strategy strategy) { this.id = id; info.setSubCategory(id); @@ -41,7 +40,6 @@ public TestConfig(String id, Info info, Auth auth, ConfigParserResult apiSelecti this.execute = execute; this.validation = validation; this.strategy = strategy; - this.workFlowSelectionFilters = workFlowSelectionFilters; } public TestConfig() { } @@ -150,14 +148,6 @@ public void setAuthor(String author) { this.author = author; } - public Map getWorkFlowSelectionFilters() { - return workFlowSelectionFilters; - } - - public void setWorkFlowSelectionFilters(Map workFlowSelectionFilters) { - this.workFlowSelectionFilters = workFlowSelectionFilters; - } - public boolean isInactive() { return inactive; } From a3f89c6a61623b57e96542194a99f5274fffe714 Mon Sep 17 00:00:00 2001 From: Ankush Jain <91221068+ankush-jain-akto@users.noreply.github.com> Date: Tue, 16 Apr 2024 15:10:22 +0530 Subject: [PATCH 07/16] remove bolt --- .github/workflows/staging.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.github/workflows/staging.yml b/.github/workflows/staging.yml index dddc0b0e31..6d251e571a 100644 --- a/.github/workflows/staging.yml +++ b/.github/workflows/staging.yml @@ -13,8 +13,6 @@ jobs: outputs: IMAGE_TAG: ${{ steps.docker_tag.outputs.IMAGE_TAG }} steps: - - name: Setup Bolt - uses: koalalab-inc/bolt@v1 - uses: actions/checkout@v2 - uses: actions/setup-java@v2 with: @@ -112,8 +110,6 @@ jobs: outputs: lb_name: ${{ steps.deploy_cluster.outputs.lb_name }} steps: - - name: Setup Bolt - uses: koalalab-inc/bolt@v1 #revert branch name - name: Setup mongo, akto-k8s-agent kube yaml's run: | @@ -187,8 +183,6 @@ jobs: needs: deploy runs-on: ubuntu-latest steps: - - name: Setup Bolt - uses: koalalab-inc/bolt@v1 - uses: actions/checkout@v2 - uses: actions/setup-node@v2 with: @@ -227,8 +221,6 @@ jobs: FLASH_NLB_DNS: ${{ secrets.FLASH_NLB_DNS }} FLASH_MONGO_CONN: ${{ secrets.FLASH_MONGO_CONN }} steps: - - name: Setup Bolt - uses: koalalab-inc/bolt@v1 - name: Fetch kube yaml #revert branch name From 08694886fa9335e06516d1874d85d0345f075ca0 Mon Sep 17 00:00:00 2001 From: ayushaga14 Date: Tue, 16 Apr 2024 16:17:33 +0530 Subject: [PATCH 08/16] support variables in override url input --- .../akto/test_editor/execution/Executor.java | 92 +++++++++++-------- .../java/com/akto/testing/TestExecutor.java | 8 +- .../java/com/akto/testing/ApiExecutor.java | 43 +++++++++ 3 files changed, 102 insertions(+), 41 deletions(-) diff --git a/apps/testing/src/main/java/com/akto/test_editor/execution/Executor.java b/apps/testing/src/main/java/com/akto/test_editor/execution/Executor.java index f468d763a4..c54066b386 100644 --- a/apps/testing/src/main/java/com/akto/test_editor/execution/Executor.java +++ b/apps/testing/src/main/java/com/akto/test_editor/execution/Executor.java @@ -4,6 +4,8 @@ import com.akto.dao.billing.OrganizationsDao; import com.akto.dao.billing.TokensDao; import java.util.*; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import com.akto.dao.CustomAuthTypeDao; import com.akto.dao.context.Context; @@ -44,12 +46,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import java.net.URI; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.UUID; import org.json.JSONObject; @@ -76,7 +72,7 @@ public class Executor { public YamlTestResult execute(ExecutorNode node, RawApi rawApi, Map varMap, String logId, AuthMechanism authMechanism, FilterNode validatorNode, ApiInfo.ApiInfoKey apiInfoKey, TestingRunConfig testingRunConfig, List customAuthTypes, boolean debug, List testLogs) { List result = new ArrayList<>(); - + ExecutionListBuilder executionListBuilder = new ExecutionListBuilder(); List executorNodes = new ArrayList<>(); ExecutionOrderResp executionOrderResp = executionListBuilder.parseExecuteOperations(node, executorNodes); @@ -171,35 +167,6 @@ public YamlTestResult execute(ExecutorNode node, RawApi rawApi, Map originalHostHeaders = rawApi.getRequest().getHeaders().getOrDefault(_HOST, new ArrayList<>()); - List attemptHostHeaders = testReq.getRequest().getHeaders().getOrDefault(_HOST, new ArrayList<>()); - - if (!originalHostHeaders.isEmpty() - && originalHostHeaders.get(0) != null - && !attemptHostHeaders.isEmpty() - && attemptHostHeaders.get(0) != null - && originalHostHeaders.get(0).equals(attemptHostHeaders.get(0)) - && testingRunConfig != null - && !StringUtils.isEmpty(testingRunConfig.getOverriddenTestAppUrl())) { - - String url = ApiExecutor.prepareUrl(testReq.getRequest(), testingRunConfig); - URI uri = new URI(url); - String host = uri.getHost(); - if (uri.getPort() != -1) { - host += ":" + uri.getPort(); - } - testReq.getRequest().getHeaders().put(_HOST, Collections.singletonList(host)); - } - - } catch (Exception e) { - testLogs.add(new TestingRunResult.TestLog(TestingRunResult.TestLogType.ERROR, "unable to update host header for overridden test URL")); - loggerMaker.errorAndAddToDb(e,"unable to update host header for overridden test URL", - LogDb.TESTING); - } - // follow redirects = true for now testResponse = ApiExecutor.sendRequest(testReq.getRequest(), followRedirect, testingRunConfig, debug, testLogs); requestSent = true; @@ -230,6 +197,59 @@ public YamlTestResult execute(ExecutorNode node, RawApi rawApi, Map originalHostHeaders = rawApi.getRequest().getHeaders().getOrDefault(_HOST, new ArrayList<>()); + if (!originalHostHeaders.isEmpty() && testingRunConfig != null + && !StringUtils.isEmpty(testingRunConfig.getOverriddenTestAppUrl())) { + + Pattern pattern = Pattern.compile("\\$\\{[^}]*\\}"); + Matcher matcher = pattern.matcher(testingRunConfig.getOverriddenTestAppUrl()); + if (matcher.find()) { + String match = matcher.group(0); + match = match.substring(2, match.length()); + match = match.substring(0, match.length() - 1); + String[] params = match.split("\\+"); + for (int i = 0; i < params.length; i++) { + url += resolveParam(params[i], rawApi); + } + testingRunConfig.setOverriddenTestAppUrl(url); + } else { + url = testingRunConfig.getOverriddenTestAppUrl(); + } + + String newUrl = ApiExecutor.replacePathFromConfig(rawApi.getRequest().getUrl(), testingRunConfig); + URI uri = new URI(newUrl); + String host = uri.getHost(); + if (uri.getPort() != -1) { + host += ":" + uri.getPort(); + } + rawApi.getRequest().getHeaders().put(_HOST, Collections.singletonList(host)); + rawApi.getRequest().setUrl(newUrl); + + } + + } catch (Exception e) { + //testLogs.add(new TestingRunResult.TestLog(TestingRunResult.TestLogType.ERROR, "unable to update host header for overridden test URL")); + loggerMaker.errorAndAddToDb(e,"unable to update host header for overridden test URL", + LogDb.TESTING); + } + } + + public String resolveParam(String param, RawApi rawApi) { + param = param.trim(); + String[] params = param.split("\\."); + + if (params.length == 1) { + return params[0]; + } + + String key = params[params.length - 1]; + String val = rawApi.getRequest().getHeaders().get(key).get(0); + return val; + } + private void overrideAuth(RawApi rawApi, AuthMechanism authMechanism) { List authParams = authMechanism.getAuthParams(); if (authParams == null || authParams.isEmpty()) { diff --git a/apps/testing/src/main/java/com/akto/testing/TestExecutor.java b/apps/testing/src/main/java/com/akto/testing/TestExecutor.java index a4222c151e..aa124d4192 100644 --- a/apps/testing/src/main/java/com/akto/testing/TestExecutor.java +++ b/apps/testing/src/main/java/com/akto/testing/TestExecutor.java @@ -34,6 +34,7 @@ import com.akto.store.AuthMechanismStore; import com.akto.store.SampleMessageStore; import com.akto.store.TestingUtil; +import com.akto.test_editor.execution.Executor; import com.akto.test_editor.execution.VariableResolver; import com.akto.testing.yaml_tests.YamlTestTemplate; import com.akto.testing_issues.TestingIssuesHandler; @@ -46,11 +47,6 @@ import com.mongodb.MongoInterruptedException; import org.apache.commons.lang3.StringUtils; import com.mongodb.BasicDBObject; -import com.mongodb.client.model.Aggregates; -import com.mongodb.client.model.Filters; -import com.mongodb.client.model.Projections; -import com.mongodb.client.model.Sorts; -import com.mongodb.client.model.Updates; import com.mongodb.client.model.*; import org.bson.conversions.Bson; @@ -678,6 +674,8 @@ public TestingRunResult runTestNew(ApiInfo.ApiInfoKey apiInfoKey, ObjectId testR List customAuthTypes = testingUtil.getCustomAuthTypes(); // TestingUtil -> authMechanism // TestingConfig -> auth + com.akto.test_editor.execution.Executor executor = new Executor(); + executor.overrideTestUrl(rawApi, testingRunConfig); YamlTestTemplate yamlTestTemplate = new YamlTestTemplate(apiInfoKey,filterNode, validatorNode, executorNode, rawApi, varMap, auth, testingUtil.getAuthMechanism(), testExecutionLogId, testingRunConfig, customAuthTypes, testConfig.getStrategy()); YamlTestResult testResults = yamlTestTemplate.run(debug, testLogs); diff --git a/libs/utils/src/main/java/com/akto/testing/ApiExecutor.java b/libs/utils/src/main/java/com/akto/testing/ApiExecutor.java index 65900b0083..6c8b1c9639 100644 --- a/libs/utils/src/main/java/com/akto/testing/ApiExecutor.java +++ b/libs/utils/src/main/java/com/akto/testing/ApiExecutor.java @@ -144,6 +144,49 @@ public static String replaceHostFromConfig(String url, TestingRunConfig testingR return url; } + public static String replacePathFromConfig(String url, TestingRunConfig testingRunConfig) { + if (testingRunConfig != null && !StringUtils.isEmpty(testingRunConfig.getOverriddenTestAppUrl())) { + URI typedUrl = null; + try { + typedUrl = new URI(url); + + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } + + String newUrl = testingRunConfig.getOverriddenTestAppUrl(); + + URI newUri = null; + try { + newUri = new URI(newUrl); + + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } + + String newPath = newUri.getPath(); + + if (newPath.equals("") || newPath.equals("/")) { + newPath = typedUrl.getPath(); + } + + String newHost = newUri.getHost(); + if (newUri.getHost().equals("")) { + newHost = typedUrl.getHost(); + } + + try { + String newScheme = newUri.getScheme() == null ? typedUrl.getScheme() : newUri.getScheme(); + int newPort = newUri.getPort() == -1 ? typedUrl.getPort() : newUri.getPort(); + + url = new URI(newScheme, null, newHost, newPort, newPath, typedUrl.getQuery(), typedUrl.getFragment()).toString(); + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } + } + return url; + } + public static String prepareUrl(OriginalHttpRequest request, TestingRunConfig testingRunConfig) throws Exception{ String url = request.getUrl(); url = url.trim(); From dd1d118c164f3fb154b4a6fa89ee774028a66ef8 Mon Sep 17 00:00:00 2001 From: ayushaga14 Date: Tue, 16 Apr 2024 16:18:20 +0530 Subject: [PATCH 09/16] allow variable string in run test input --- apps/dashboard/src/main/java/com/akto/utils/Utils.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/apps/dashboard/src/main/java/com/akto/utils/Utils.java b/apps/dashboard/src/main/java/com/akto/utils/Utils.java index 5731a4dc20..91370c4ca4 100644 --- a/apps/dashboard/src/main/java/com/akto/utils/Utils.java +++ b/apps/dashboard/src/main/java/com/akto/utils/Utils.java @@ -158,6 +158,11 @@ public static boolean isValidURL(String url) { new URL(url).toURI(); return true; } catch (MalformedURLException | URISyntaxException e) { + Pattern pattern = Pattern.compile("\\$\\{[^}]*\\}"); + Matcher matcher = pattern.matcher(url); + if (matcher.find()) { + return true; + } return false; } } From 866081bced708cf5aec7f558301ae790ace2f5ce Mon Sep 17 00:00:00 2001 From: Avneesh Hota Date: Tue, 16 Apr 2024 17:11:56 +0530 Subject: [PATCH 10/16] rever har action changes and null checks for memory.java --- .../main/java/com/akto/action/HarAction.java | 183 +++--------------- .../observe/api_collections/ApiDependency.jsx | 4 +- .../YamlNodeExecutor.java | 2 +- .../akto/dto/dependency_flow/TreeHelper.java | 2 +- 4 files changed, 29 insertions(+), 162 deletions(-) diff --git a/apps/dashboard/src/main/java/com/akto/action/HarAction.java b/apps/dashboard/src/main/java/com/akto/action/HarAction.java index 296409bf75..4d0db6e7b6 100644 --- a/apps/dashboard/src/main/java/com/akto/action/HarAction.java +++ b/apps/dashboard/src/main/java/com/akto/action/HarAction.java @@ -1,20 +1,14 @@ package com.akto.action; import com.akto.DaoInit; -import com.akto.action.test_editor.SaveTestEditorAction; +import com.akto.analyser.ResourceAnalyser; import com.akto.dao.ApiCollectionsDao; import com.akto.dao.BurpPluginInfoDao; -import com.akto.dao.SampleDataDao; +import com.akto.dao.RuntimeFilterDao; import com.akto.dao.context.Context; import com.akto.dao.file.FilesDao; import com.akto.dto.ApiCollection; -import com.akto.dto.ApiInfo; import com.akto.dto.HttpResponseParams; -import com.akto.dto.dependency_flow.DependencyFlow; -import com.akto.dto.testing.TestingRunResult; -import com.akto.dto.testing.YamlTestResult; -import com.akto.dto.traffic.SampleData; -import com.akto.dto.type.URLMethods; import com.akto.har.HAR; import com.akto.listener.InitializerListener; import com.akto.listener.KafkaListener; @@ -26,8 +20,6 @@ import com.akto.har.HAR; import com.akto.log.LoggerMaker; import com.akto.dto.ApiToken.Utility; -import com.akto.test_editor.execution.Executor; -import com.akto.testing.yaml_tests.YamlTestTemplate; import com.akto.util.DashboardMode; import com.akto.utils.GzipUtils; import com.akto.utils.Utils; @@ -42,9 +34,10 @@ import java.io.File; import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.util.*; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.UUID; public class HarAction extends UserAction { private String harString; @@ -57,134 +50,6 @@ public class HarAction extends UserAction { private byte[] tcpContent; private static final LoggerMaker loggerMaker = new LoggerMaker(HarAction.class); - public static void main(String[] args) throws IOException { - DaoInit.init(new ConnectionString("mongodb://localhost:27017")); - Context.accountId.set(1_000_000); - - DependencyFlow dependencyFlow = new DependencyFlow(); - dependencyFlow.run(); - dependencyFlow.syncWithDb(); - } - - public static void main3(String[] args) throws IOException { - DaoInit.init(new ConnectionString("mongodb://localhost:27017")); - Context.accountId.set(1_000_000); - - String filePath = "/Users/avneesh/Downloads/juiceshop_address_1.har"; - String content; - try { - content = new String(Files.readAllBytes(Paths.get(filePath))); - } catch (IOException e) { - e.printStackTrace(); - return; - } - - System.out.println(content.length()); - - HarAction harAction = new HarAction(); - harAction.setHarString(content); - harAction.setApiCollectionName("a3"); - harAction.skipKafka = true; - harAction.execute(); - } - - public static void main2(String[] args) { - DaoInit.init(new ConnectionString("mongodb://localhost:27017/admini")); - Context.accountId.set(1_000_000); - - ApiInfo.ApiInfoKey apiInfoKey = new ApiInfo.ApiInfoKey(1712980524, "https://juiceshop.akto.io/api/Cards/INTEGER", URLMethods.Method.DELETE); - - SampleData sampleData = SampleDataDao.instance.findOne( - Filters.and( - Filters.eq("_id.apiCollectionId", apiInfoKey.getApiCollectionId()), - Filters.eq("_id.method",apiInfoKey.getMethod().name()), - Filters.eq("_id.url",apiInfoKey.getUrl()) - ) - ); - - BasicDBObject apiInfoKeyObj = new BasicDBObject() - .append("apiCollectionId", apiInfoKey.getApiCollectionId()) - .append("url", apiInfoKey.getUrl()) - .append("method", apiInfoKey.getMethod().name()); - - SaveTestEditorAction saveTestEditorAction = new SaveTestEditorAction(); - saveTestEditorAction.setApiInfoKey(apiInfoKeyObj); - saveTestEditorAction.setContent(testContent); - saveTestEditorAction.setSampleDataList(Collections.singletonList(sampleData)); - String result = saveTestEditorAction.runTestForGivenTemplate(); - System.out.println(result); - - TestingRunResult testingRunResult = saveTestEditorAction.getTestingRunResult(); - System.out.println(testingRunResult.getTestResults().size()); - for (TestingRunResult.TestLog testLog: testingRunResult.getTestLogs()) { - System.out.println(testLog.getMessage()); - } - } - - public static void main1(String[] args) { - System.out.println(testContent); - } - - - public static String testContent = "id: REPORT_GENERATION_DOS\n" + - "\n" + - "info:\n" + - " name: \"Denial of Service Test on Report Generation Endpoint\"\n" + - " description: \"A Denial of Service (DoS) test\"\n" + - " details: \"In this test.\"\n" + - " impact: \"A\"\n" + - " category:\n" + - " name: RL\n" + - " shortName: Lack of Resources & Rate Limiting\n" + - " displayName: Lack of Resources & Rate Limiting (RL)\n" + - " subCategory: REPORT_GENERATION_DOS\n" + - " severity: HIGH\n" + - " tags:\n" + - " - Business logic\n" + - " - OWASP top 10\n" + - " - HackerOne top 10\n" + - " references:\n" + - " - \"https://github.com/OWASP/API-Security/blob/master/2019/en/src/0xa4-lack-of-resources-and-rate-limiting.md#scenario-2\"\n" + - " cwe:\n" + - " - CWE-400\n" + - " cve:\n" + - " - CVE-2023-4647\n" + - " - CVE-2023-38254\n" + - "\n" + - "\n" + - "api_selection_filters:\n" + - " response_code:\n" + - " gte: 200\n" + - " lt: 300\n" + - "\n" + - "execute:\n" + - " type: graph\n" + - " requests:\n" + - " - req:\n" + - " - replace_auth_header: true\n" + - " - validate:\n" + - " percentage_match:\n" + - " gte: 90\n" + - " - success: x2\n" + - " - failure: x2\n" + - " - req:\n" + - " - api: get_asset_api\n" + - " - validate: \n" + - " response_code: 4xx\n" + - " success: vulnerability\n" + - " failure: exit\n" + - "\n" + - "\n" + - "\n" + - "validate:\n" + - " and:\n" + - " - compare_greater:\n" + - " - ${x2.response.stats.median_response_time}\n" + - " - 3001\n" + - " - compare_greater:\n" + - " - ${x2.response.stats.median_response_time}\n" + - " - ${x1.response.stats.median_response_time} * 3"; - public String executeWithSkipKafka(boolean skipKafka) throws IOException { this.skipKafka = skipKafka; execute(); @@ -203,7 +68,7 @@ public String execute() throws IOException { apiCollection = ApiCollectionsDao.instance.findByName(apiCollectionName); if (apiCollection == null) { ApiCollectionsAction apiCollectionsAction = new ApiCollectionsAction(); -// apiCollectionsAction.setSession(this.getSession()); + apiCollectionsAction.setSession(this.getSession()); apiCollectionsAction.setCollectionName(apiCollectionName); String result = apiCollectionsAction.createCollection(); if (result.equalsIgnoreCase(Action.SUCCESS)) { @@ -215,7 +80,7 @@ public String execute() throws IOException { return ERROR.toUpperCase(); } } else { - Collection actionErrors = apiCollectionsAction.getActionErrors(); + Collection actionErrors = apiCollectionsAction.getActionErrors(); if (actionErrors != null && actionErrors.size() > 0) { for (String actionError: actionErrors) { addActionError(actionError); @@ -260,16 +125,16 @@ public String execute() throws IOException { return ERROR.toUpperCase(); } -// if (getSession().getOrDefault("utility","").equals(Utility.BURP.toString())) { -// BurpPluginInfoDao.instance.updateLastDataSentTimestamp(getSUser().getLogin()); -// } + if (getSession().getOrDefault("utility","").equals(Utility.BURP.toString())) { + BurpPluginInfoDao.instance.updateLastDataSentTimestamp(getSUser().getLogin()); + } try { HAR har = new HAR(); loggerMaker.infoAndAddToDb("Har file upload processing for collectionId:" + apiCollectionId, LoggerMaker.LogDb.DASHBOARD); -// String zippedString = GzipUtils.zipString(harString); -// com.akto.dto.files.File file = new com.akto.dto.files.File(HttpResponseParams.Source.HAR.toString(),zippedString); -// FilesDao.instance.insertOne(file); + String zippedString = GzipUtils.zipString(harString); + com.akto.dto.files.File file = new com.akto.dto.files.File(HttpResponseParams.Source.HAR.toString(),zippedString); + FilesDao.instance.insertOne(file); List messages = har.getMessages(harString, apiCollectionId, Context.accountId.get()); harErrors = har.getErrors(); Utils.pushDataToKafka(apiCollectionId, topic, messages, harErrors, skipKafka); @@ -313,7 +178,7 @@ public void setTcpContent(byte[] tcpContent) { Awesome awesome = null; public String uploadTcp() { - + File tmpDir = FileUtils.getTempDirectory(); String filename = UUID.randomUUID().toString() + ".pcap"; File tcpDump = new File(tmpDir, filename); @@ -323,23 +188,23 @@ public String uploadTcp() { Awesome.GoString.ByValue str = new Awesome.GoString.ByValue(); str.p = tcpDump.getAbsolutePath(); str.n = str.p.length(); - + Awesome.GoString.ByValue str2 = new Awesome.GoString.ByValue(); str2.p = System.getenv("AKTO_KAFKA_BROKER_URL"); str2.n = str2.p.length(); - + awesome.readTcpDumpFile(str, str2 , apiCollectionId); - - return Action.SUCCESS.toUpperCase(); + + return Action.SUCCESS.toUpperCase(); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); - return Action.ERROR.toUpperCase(); + return Action.ERROR.toUpperCase(); } } - interface Awesome extends Library { + interface Awesome extends Library { public static class GoString extends Structure { /** C type : const char* */ public String p; @@ -359,8 +224,8 @@ public GoString(String p, long n) { public static class ByReference extends GoString implements Structure.ByReference {} public static class ByValue extends GoString implements Structure.ByValue {} } - + public void readTcpDumpFile(GoString.ByValue filepath, GoString.ByValue kafkaURL, long apiCollectionId); - + } -} +} \ No newline at end of file diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/ApiDependency.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/ApiDependency.jsx index 5971ec1d2a..0ceab0b9a9 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/ApiDependency.jsx +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/ApiDependency.jsx @@ -97,7 +97,9 @@ function formatRawNodeData(nodes, currentApiCollectionId, currentEndpoint, curre let connections = node["connections"] let edgesMap = new Map() Object.values(connections).forEach(connection => { - let edge = connection["edges"][0] // todo: null check + let edge = connection["edges"][0] + + if (!edge) return let source = calculateNodeId(edge["apiCollectionId"], edge["url"], edge["method"]); let edgeId = source + "-" + id; diff --git a/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/YamlNodeExecutor.java b/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/YamlNodeExecutor.java index 0f16a8fb53..8625630f76 100644 --- a/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/YamlNodeExecutor.java +++ b/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/YamlNodeExecutor.java @@ -139,7 +139,7 @@ public NodeResult processNode(Node node, Map varMap, Boolean all try { tsBeforeReq = Context.nowInMillis(); testResponse = ApiExecutor.sendRequest(testReq.getRequest(), followRedirect, testingRunConfig, debug, testLogs, Main.SKIP_SSRF_CHECK); - if (apiInfoKey != null) { + if (apiInfoKey != null && memory != null) { memory.fillResponse(testReq.getRequest(), testResponse, apiInfoKey.getApiCollectionId(), apiInfoKey.getUrl(), apiInfoKey.getMethod().name()); memory.reset(apiInfoKey.getApiCollectionId(), apiInfoKey.getUrl(), apiInfoKey.getMethod().name()); } diff --git a/libs/dao/src/main/java/com/akto/dto/dependency_flow/TreeHelper.java b/libs/dao/src/main/java/com/akto/dto/dependency_flow/TreeHelper.java index 1917d2108f..bb04a3fbca 100644 --- a/libs/dao/src/main/java/com/akto/dto/dependency_flow/TreeHelper.java +++ b/libs/dao/src/main/java/com/akto/dto/dependency_flow/TreeHelper.java @@ -38,7 +38,7 @@ public void buildTree(String apiCollectionId, String url, String method) { result.put(node.hashCode(), node); - if (node.getMaxDepth() == 0) return; // todo: check implication on front end graph + if (node.getMaxDepth() == 0) return; Map connections = node.getConnections(); From d0d6f48d22453c316fcfcc381f6c59baea177185 Mon Sep 17 00:00:00 2001 From: aktoboy <111744450+aktoboy@users.noreply.github.com> Date: Tue, 16 Apr 2024 17:37:25 +0530 Subject: [PATCH 11/16] Update Constants.java --- libs/utils/src/main/java/com/akto/onprem/Constants.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/libs/utils/src/main/java/com/akto/onprem/Constants.java b/libs/utils/src/main/java/com/akto/onprem/Constants.java index 1b581fb795..d249e3060b 100644 --- a/libs/utils/src/main/java/com/akto/onprem/Constants.java +++ b/libs/utils/src/main/java/com/akto/onprem/Constants.java @@ -11,6 +11,8 @@ import java.security.NoSuchAlgorithmException; import java.util.Arrays; import java.util.Base64; +import org.bson.types.ObjectId; +import com.akto.dto.billing.Organization; public class Constants { public static final DashboardMode DEFAULT_DASHBOARD_MODE = DashboardMode.LOCAL_DEPLOY; @@ -88,4 +90,8 @@ public static Config.SendgridConfig getSendgridConfig() { } return sendgridConfig; } + + public static void sendTestResults(ObjectId summaryId, Organization organization) { + + } } From c62e46b03e1f7b690484360f138b9eaf39bf4785 Mon Sep 17 00:00:00 2001 From: Avneesh Hota Date: Tue, 16 Apr 2024 18:00:04 +0530 Subject: [PATCH 12/16] fixed unit test --- .../java/com/akto/dependency/TestDependencyAnalyser.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/api-runtime/src/test/java/com/akto/dependency/TestDependencyAnalyser.java b/apps/api-runtime/src/test/java/com/akto/dependency/TestDependencyAnalyser.java index 489e079248..20a3afe5fd 100644 --- a/apps/api-runtime/src/test/java/com/akto/dependency/TestDependencyAnalyser.java +++ b/apps/api-runtime/src/test/java/com/akto/dependency/TestDependencyAnalyser.java @@ -94,7 +94,7 @@ public void testAnalyse1() { TreeHelper treeHelper = new TreeHelper(); treeHelper.buildTree("1000", "/api/m7", "POST"); Map result = treeHelper.result; - assertEquals(2, result.size()); // this is because /api/m6 gets best value from /api/m1 + assertEquals(3, result.size()); // this is because /api/m6 gets best value from /api/m1 Map connections = result.get(Objects.hash("1000", "/api/m7", "POST")).getConnections(); assertEquals(1, connections.size()); @@ -161,7 +161,7 @@ public void testAnalyse3() { TreeHelper treeHelper = new TreeHelper(); treeHelper.buildTree("1000", "/api/m7", "POST"); Map result = treeHelper.result; - assertEquals(6, result.size()); // this is because /api/m6 has 2 parameters getting data + assertEquals(7, result.size()); // this is because /api/m6 has 2 parameters getting data Map connections = result.get(Objects.hash("1000", "/api/m7", "POST")).getConnections(); assertEquals(1, connections.size()); @@ -197,7 +197,7 @@ public void testAnalyse4() { TreeHelper treeHelper = new TreeHelper(); treeHelper.buildTree("1000", "api/cars/INTEGER", "POST"); Map result = treeHelper.result; - assertEquals(1, result.size()); // this is because /api/m6 has 2 parameters getting data + assertEquals(2, result.size()); // this is because /api/m6 has 2 parameters getting data Map connections = result.get(Objects.hash("1000", "api/cars/INTEGER", "POST")).getConnections(); assertEquals(1, connections.size()); From 6b7696381cb5cd1ab76064aabe4d460ba0715a03 Mon Sep 17 00:00:00 2001 From: Avneesh Hota Date: Tue, 16 Apr 2024 18:32:01 +0530 Subject: [PATCH 13/16] added graceful error handling to workflow tests --- .../testing/workflow_node_executor/YamlNodeExecutor.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/YamlNodeExecutor.java b/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/YamlNodeExecutor.java index 8625630f76..6f8eb0d787 100644 --- a/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/YamlNodeExecutor.java +++ b/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/YamlNodeExecutor.java @@ -74,10 +74,18 @@ public NodeResult processNode(Node node, Map varMap, Boolean all String apiType = firstChildNode.getValues().toString(); if (apiType.equalsIgnoreCase("get_asset_api")) { rawApi = memory.findAssetGetterRequest(apiInfoKey); + if (rawApi == null) { + testErrors.add("Couldn't find corresponding getter api"); + new WorkflowTestResult.NodeResult("[]",false, testErrors); + } } childNodes.remove(0); } else { OriginalHttpRequest request = memory.run(apiInfoKey.getApiCollectionId(), apiInfoKey.getUrl(), apiInfoKey.getMethod().name()); + if (request == null) { + testErrors.add("Failed getting request from dependency graph"); + new WorkflowTestResult.NodeResult("[]",false, testErrors); + } rawApi.setRequest(request); } } From a2895d387612928234718dadc3f615d1d34dd4d4 Mon Sep 17 00:00:00 2001 From: ayushaga14 Date: Wed, 17 Apr 2024 10:06:02 +0530 Subject: [PATCH 14/16] add logs --- .../src/main/java/com/akto/test_editor/execution/Executor.java | 1 + libs/utils/src/main/java/com/akto/testing/ApiExecutor.java | 3 +++ 2 files changed, 4 insertions(+) diff --git a/apps/testing/src/main/java/com/akto/test_editor/execution/Executor.java b/apps/testing/src/main/java/com/akto/test_editor/execution/Executor.java index c54066b386..672463b188 100644 --- a/apps/testing/src/main/java/com/akto/test_editor/execution/Executor.java +++ b/apps/testing/src/main/java/com/akto/test_editor/execution/Executor.java @@ -200,6 +200,7 @@ public YamlTestResult execute(ExecutorNode node, RawApi rawApi, Map originalHostHeaders = rawApi.getRequest().getHeaders().getOrDefault(_HOST, new ArrayList<>()); if (!originalHostHeaders.isEmpty() && testingRunConfig != null && !StringUtils.isEmpty(testingRunConfig.getOverriddenTestAppUrl())) { diff --git a/libs/utils/src/main/java/com/akto/testing/ApiExecutor.java b/libs/utils/src/main/java/com/akto/testing/ApiExecutor.java index 6c8b1c9639..c4bf4e0d6c 100644 --- a/libs/utils/src/main/java/com/akto/testing/ApiExecutor.java +++ b/libs/utils/src/main/java/com/akto/testing/ApiExecutor.java @@ -151,6 +151,7 @@ public static String replacePathFromConfig(String url, TestingRunConfig testingR typedUrl = new URI(url); } catch (URISyntaxException e) { + loggerMaker.errorAndAddToDb(e, "error converting req url to uri " + url, LogDb.TESTING); throw new RuntimeException(e); } @@ -161,6 +162,7 @@ public static String replacePathFromConfig(String url, TestingRunConfig testingR newUri = new URI(newUrl); } catch (URISyntaxException e) { + loggerMaker.errorAndAddToDb(e, "error converting override url to uri " + url, LogDb.TESTING); throw new RuntimeException(e); } @@ -181,6 +183,7 @@ public static String replacePathFromConfig(String url, TestingRunConfig testingR url = new URI(newScheme, null, newHost, newPort, newPath, typedUrl.getQuery(), typedUrl.getFragment()).toString(); } catch (URISyntaxException e) { + loggerMaker.errorAndAddToDb(e, "error building new url using override url", LogDb.TESTING); throw new RuntimeException(e); } } From 1f30fb414f11821b0ec4ee78e8983cd43a34594c Mon Sep 17 00:00:00 2001 From: aktoboy Date: Wed, 17 Apr 2024 10:45:31 +0530 Subject: [PATCH 15/16] Added changes for test results telemetry --- .../akto/listener/InitializerListener.java | 3 + .../main/java/com/akto/utils/GzipUtils.java | 49 ---------------- .../akto/utils/billing/OrganizationUtils.java | 5 ++ .../src/main/java/com/akto/testing/Main.java | 28 +++++++++- .../com/akto/dto/billing/Organization.java | 11 ++++ .../parser/parameter_parser/CommonParser.java | 2 + .../main/java/com/akto/utils/GzipUtils.java | 56 +++++++++++++++++++ 7 files changed, 104 insertions(+), 50 deletions(-) delete mode 100644 apps/dashboard/src/main/java/com/akto/utils/GzipUtils.java create mode 100644 libs/utils/src/main/java/com/akto/utils/GzipUtils.java diff --git a/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java b/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java index 327821530c..0b4fc09e3c 100644 --- a/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java +++ b/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java @@ -1795,6 +1795,8 @@ public static Organization fetchAndSaveFeatureWiseAllowed(Organization organizat hotjarSiteId = OrganizationUtils.fetchHotjarSiteId(metaData); boolean telemetryEnabled = OrganizationUtils.fetchTelemetryEnabled(metaData); setTelemetrySettings(organization, telemetryEnabled); + boolean testTelemetryEnabled = OrganizationUtils.fetchTestTelemetryEnabled(metaData); + organization.setTestTelemetryEnabled(testTelemetryEnabled); loggerMaker.infoAndAddToDb("Processed org metadata",LogDb.DASHBOARD); @@ -1812,6 +1814,7 @@ public static Organization fetchAndSaveFeatureWiseAllowed(Organization organizat Updates.set(Organization.FEATURE_WISE_ALLOWED, featureWiseAllowed), Updates.set(Organization.GRACE_PERIOD, gracePeriod), Updates.set(Organization.HOTJAR_SITE_ID, hotjarSiteId), + Updates.set(Organization.TEST_TELEMETRY_ENABLED, testTelemetryEnabled), Updates.set(Organization.LAST_FEATURE_MAP_UPDATE, lastFeatureMapUpdate))); loggerMaker.infoAndAddToDb("Updated org",LogDb.DASHBOARD); diff --git a/apps/dashboard/src/main/java/com/akto/utils/GzipUtils.java b/apps/dashboard/src/main/java/com/akto/utils/GzipUtils.java deleted file mode 100644 index 121911a7e8..0000000000 --- a/apps/dashboard/src/main/java/com/akto/utils/GzipUtils.java +++ /dev/null @@ -1,49 +0,0 @@ -package com.akto.utils; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.util.Base64; -import java.util.zip.GZIPInputStream; -import java.util.zip.GZIPOutputStream; - -public class GzipUtils { - - public static String zipString(String input) { - if (input == null || input.isEmpty()) { - return input; - } - - try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); - GZIPOutputStream gzipOutputStream = new GZIPOutputStream(byteArrayOutputStream)) { - gzipOutputStream.write(input.getBytes(StandardCharsets.UTF_8)); - gzipOutputStream.close(); - return Base64.getEncoder().encodeToString(byteArrayOutputStream.toByteArray()); - } catch (IOException e) { - throw new RuntimeException("Failed to zip content", e); - } - } - - public static String unzipString(String zippedBase64Str) { - if (zippedBase64Str == null || zippedBase64Str.isEmpty()) { - return zippedBase64Str; - } - - byte[] decodedBytes = Base64.getDecoder().decode(zippedBase64Str); - - try (ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(decodedBytes); - GZIPInputStream gzipInputStream = new GZIPInputStream(byteArrayInputStream); - ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream()) { - - byte[] buffer = new byte[1024]; - int len; - while ((len = gzipInputStream.read(buffer)) != -1) { - byteArrayOutputStream.write(buffer, 0, len); - } - return byteArrayOutputStream.toString(StandardCharsets.UTF_8.name()); - } catch (IOException e) { - throw new RuntimeException("Failed to unzip content", e); - } - } -} diff --git a/apps/dashboard/src/main/java/com/akto/utils/billing/OrganizationUtils.java b/apps/dashboard/src/main/java/com/akto/utils/billing/OrganizationUtils.java index 158a637e1a..661c47ec1f 100644 --- a/apps/dashboard/src/main/java/com/akto/utils/billing/OrganizationUtils.java +++ b/apps/dashboard/src/main/java/com/akto/utils/billing/OrganizationUtils.java @@ -286,4 +286,9 @@ public static boolean fetchTelemetryEnabled(BasicDBObject metadata) { BasicDBObject additionalMetaData = (BasicDBObject) metadata.getOrDefault("additionalMetaData", new BasicDBObject()); return additionalMetaData.getString("ENABLE_TELEMETRY", "NA").equalsIgnoreCase("ENABLED"); } + + public static boolean fetchTestTelemetryEnabled(BasicDBObject metadata) { + BasicDBObject additionalMetaData = (BasicDBObject) metadata.getOrDefault("additionalMetaData", new BasicDBObject()); + return additionalMetaData.getString("ENABLE_TEST_TELEMETRY", "NA").equalsIgnoreCase("ENABLED"); + } } diff --git a/apps/testing/src/main/java/com/akto/testing/Main.java b/apps/testing/src/main/java/com/akto/testing/Main.java index f11501f3b7..c11413f1f5 100644 --- a/apps/testing/src/main/java/com/akto/testing/Main.java +++ b/apps/testing/src/main/java/com/akto/testing/Main.java @@ -6,6 +6,7 @@ import com.akto.dao.AccountsDao; import com.akto.dao.SetupDao; import com.akto.dao.MCollection; +import com.akto.dao.billing.OrganizationsDao; import com.akto.dao.context.Context; import com.akto.dao.testing.TestingRunConfigDao; import com.akto.dao.testing.TestingRunDao; @@ -16,6 +17,7 @@ import com.akto.dto.AccountSettings; import com.akto.dto.ApiInfo; import com.akto.dto.Setup; +import com.akto.dto.billing.Organization; import com.akto.dto.test_run_findings.TestingIssuesId; import com.akto.dto.test_run_findings.TestingRunIssues; import com.akto.dto.testing.TestResult; @@ -65,6 +67,9 @@ public class Main { public static final ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(2); + + public static final ScheduledExecutorService testTelemetryScheduler = Executors.newScheduledThreadPool(2); + public static final ScheduledExecutorService schedulerAccessMatrix = Executors.newScheduledThreadPool(2); public static final boolean SKIP_SSRF_CHECK = "true".equalsIgnoreCase(System.getenv("SKIP_SSRF_CHECK")); @@ -191,7 +196,7 @@ private static void setTestingRunConfig(TestingRun testingRun, TestingRunResultS } public static void main(String[] args) throws InterruptedException { - String mongoURI = System.getenv("AKTO_MONGO_CONN");; + String mongoURI = "mongodb://localhost:27017";; DaoInit.init(new ConnectionString(mongoURI)); boolean connectedToMongo = false; @@ -396,6 +401,27 @@ public void run() { } loggerMaker.infoAndAddToDb("Tests completed in " + (Context.now() - start) + " seconds for account: " + accountId, LogDb.TESTING); + + Organization organization = OrganizationsDao.instance.findOne( + Filters.in(Organization.ACCOUNTS, Context.accountId.get())); + + if(organization.getTestTelemetryEnabled()){ + loggerMaker.infoAndAddToDb("Test telemetry enabled for account: " + accountId + ", sending results", LogDb.TESTING); + ObjectId finalSummaryId = summaryId; + testTelemetryScheduler.execute(() -> { + Context.accountId.set(accountId); + try { + com.akto.onprem.Constants.sendTestResults(finalSummaryId, organization); + loggerMaker.infoAndAddToDb("Test telemetry sent for account: " + accountId, LogDb.TESTING); + } catch (Exception e) { + loggerMaker.errorAndAddToDb(e, "Error in sending test telemetry for account: " + accountId); + } + }); + + } else { + loggerMaker.infoAndAddToDb("Test telemetry disabled for account: " + accountId, LogDb.TESTING); + } + }, "testing"); Thread.sleep(1000); } diff --git a/libs/dao/src/main/java/com/akto/dto/billing/Organization.java b/libs/dao/src/main/java/com/akto/dto/billing/Organization.java index f355d5d932..4f0d0706f9 100644 --- a/libs/dao/src/main/java/com/akto/dto/billing/Organization.java +++ b/libs/dao/src/main/java/com/akto/dto/billing/Organization.java @@ -32,6 +32,9 @@ public class Organization { public static final String HOTJAR_SITE_ID = "hotjarSiteId"; public String hotjarSiteId = "hotjarSiteId"; + + public static final String TEST_TELEMETRY_ENABLED = "testTelemetryEnabled"; + private boolean testTelemetryEnabled; private int gracePeriod; public Organization() { } @@ -124,4 +127,12 @@ public int getLastFeatureMapUpdate() { public void setLastFeatureMapUpdate(int lastFeatureMapUpdate) { this.lastFeatureMapUpdate = lastFeatureMapUpdate; } + + public boolean getTestTelemetryEnabled() { + return testTelemetryEnabled; + } + + public void setTestTelemetryEnabled(boolean testTelemetryEnabled) { + this.testTelemetryEnabled = testTelemetryEnabled; + } } diff --git a/libs/utils/src/main/java/com/akto/open_api/parser/parameter_parser/CommonParser.java b/libs/utils/src/main/java/com/akto/open_api/parser/parameter_parser/CommonParser.java index 5cb4a94b92..1e7720a603 100644 --- a/libs/utils/src/main/java/com/akto/open_api/parser/parameter_parser/CommonParser.java +++ b/libs/utils/src/main/java/com/akto/open_api/parser/parameter_parser/CommonParser.java @@ -13,6 +13,8 @@ public static String getExistingExample(Parameter parameter) { ret = parameter.getExamples().get(exampleName).getValue().toString(); break; } + } else if (parameter.getSchema() != null && parameter.getSchema().getExample() != null) { + ret = parameter.getSchema().getExample().toString(); } return ret; } diff --git a/libs/utils/src/main/java/com/akto/utils/GzipUtils.java b/libs/utils/src/main/java/com/akto/utils/GzipUtils.java new file mode 100644 index 0000000000..426e5c53e3 --- /dev/null +++ b/libs/utils/src/main/java/com/akto/utils/GzipUtils.java @@ -0,0 +1,56 @@ +package com.akto.utils; + +import com.mongodb.BasicDBObject; + +import java.io.*; +import java.nio.charset.StandardCharsets; +import java.util.Base64; +import java.util.zip.GZIPInputStream; +import java.util.zip.GZIPOutputStream; + + +public class GzipUtils { + + public static String zipString(String input) { + if (input == null || input.isEmpty()) { + return input; + } + + try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + GZIPOutputStream gzipOutputStream = new GZIPOutputStream(byteArrayOutputStream)) { + gzipOutputStream.write(input.getBytes(StandardCharsets.UTF_8)); + gzipOutputStream.close(); + return Base64.getEncoder().encodeToString(byteArrayOutputStream.toByteArray()); + } catch (IOException e) { + throw new RuntimeException("Failed to zip content", e); + } + } + + + public static String unzipString(String base64String) { + if(base64String == null || base64String.isEmpty()){ + return base64String; + } + String safeString = base64String.replace('-', '+').replace('_', '/'); + safeString = safeString.replaceAll("\\s", ""); + + try { + byte[] decodedBytes = Base64.getDecoder().decode(safeString); + ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(decodedBytes); + GZIPInputStream gzipInputStream = new GZIPInputStream(byteArrayInputStream); + ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + byte[] buffer = new byte[1024]; + int len; + while ((len = gzipInputStream.read(buffer)) > 0) { + byteArrayOutputStream.write(buffer, 0, len); + } + String decompressedString = new String(byteArrayOutputStream.toByteArray(), StandardCharsets.UTF_8); + System.out.println("Decompressed string: " + decompressedString); + return decompressedString; + } catch (Exception e) { + System.err.println("Error decompressing and decoding Base64 string: " + e.getMessage()); + } + return null; + } + +} From b6365d57a33990a4263a8bb8403055615d00fabd Mon Sep 17 00:00:00 2001 From: aktoboy Date: Wed, 17 Apr 2024 10:53:36 +0530 Subject: [PATCH 16/16] Fixed mongo connection string --- apps/testing/src/main/java/com/akto/testing/Main.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/testing/src/main/java/com/akto/testing/Main.java b/apps/testing/src/main/java/com/akto/testing/Main.java index c11413f1f5..b5ff59fdd4 100644 --- a/apps/testing/src/main/java/com/akto/testing/Main.java +++ b/apps/testing/src/main/java/com/akto/testing/Main.java @@ -196,7 +196,7 @@ private static void setTestingRunConfig(TestingRun testingRun, TestingRunResultS } public static void main(String[] args) throws InterruptedException { - String mongoURI = "mongodb://localhost:27017";; + String mongoURI = System.getenv("AKTO_MONGO_CONN"); DaoInit.init(new ConnectionString(mongoURI)); boolean connectedToMongo = false;