diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 809055f635..42a88e99cf 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -14,16 +14,13 @@ name: "CodeQL" on: push: branches: [ "master" ] - pull_request: - # The branches below must be a subset of the branches above - branches: [ "master" ] schedule: - cron: '22 0 * * 1' jobs: analyze: name: Analyze - runs-on: ${{ (matrix.language == 'java' && 'macos-latest') || 'ubuntu-latest' }} + runs-on: 'ubuntu-latest' timeout-minutes: ${{ (matrix.language == 'java' && 120) || 360 }} permissions: actions: read diff --git a/.github/workflows/staging.yml b/.github/workflows/staging.yml index dddc0b0e31..6d251e571a 100644 --- a/.github/workflows/staging.yml +++ b/.github/workflows/staging.yml @@ -13,8 +13,6 @@ jobs: outputs: IMAGE_TAG: ${{ steps.docker_tag.outputs.IMAGE_TAG }} steps: - - name: Setup Bolt - uses: koalalab-inc/bolt@v1 - uses: actions/checkout@v2 - uses: actions/setup-java@v2 with: @@ -112,8 +110,6 @@ jobs: outputs: lb_name: ${{ steps.deploy_cluster.outputs.lb_name }} steps: - - name: Setup Bolt - uses: koalalab-inc/bolt@v1 #revert branch name - name: Setup mongo, akto-k8s-agent kube yaml's run: | @@ -187,8 +183,6 @@ jobs: needs: deploy runs-on: ubuntu-latest steps: - - name: Setup Bolt - uses: koalalab-inc/bolt@v1 - uses: actions/checkout@v2 - uses: actions/setup-node@v2 with: @@ -227,8 +221,6 @@ jobs: FLASH_NLB_DNS: ${{ secrets.FLASH_NLB_DNS }} FLASH_MONGO_CONN: ${{ secrets.FLASH_MONGO_CONN }} steps: - - name: Setup Bolt - uses: koalalab-inc/bolt@v1 - name: Fetch kube yaml #revert branch name diff --git a/apps/api-runtime/src/main/java/com/akto/dependency/DependencyAnalyser.java b/apps/api-runtime/src/main/java/com/akto/dependency/DependencyAnalyser.java index c4b83424f4..71a90833cd 100644 --- a/apps/api-runtime/src/main/java/com/akto/dependency/DependencyAnalyser.java +++ b/apps/api-runtime/src/main/java/com/akto/dependency/DependencyAnalyser.java @@ -152,7 +152,10 @@ public void analyse(String message, int finalApiCollectionId) { SingleTypeInfo.SuperType superType = urlTemplate.getTypes()[i]; if (superType == null) continue; int idx = ogUrl.startsWith("http") ? i:i+1; - String s = ogUrlSplit[idx]; // because ogUrl=/api/books/123 while template url=api/books/INTEGER + Object s = ogUrlSplit[idx]; // because ogUrl=/api/books/123 while template url=api/books/INTEGER + if (superType.equals(SingleTypeInfo.SuperType.INTEGER)) { + s = Integer.parseInt(ogUrlSplit[idx]); + } Set val = new HashSet<>(); val.add(s); processRequestParam(i+"", val, combinedUrl, true, false); diff --git a/apps/api-runtime/src/test/java/com/akto/dependency/TestDependencyAnalyser.java b/apps/api-runtime/src/test/java/com/akto/dependency/TestDependencyAnalyser.java index 489e079248..20a3afe5fd 100644 --- a/apps/api-runtime/src/test/java/com/akto/dependency/TestDependencyAnalyser.java +++ b/apps/api-runtime/src/test/java/com/akto/dependency/TestDependencyAnalyser.java @@ -94,7 +94,7 @@ public void testAnalyse1() { TreeHelper treeHelper = new TreeHelper(); treeHelper.buildTree("1000", "/api/m7", "POST"); Map result = treeHelper.result; - assertEquals(2, result.size()); // this is because /api/m6 gets best value from /api/m1 + assertEquals(3, result.size()); // this is because /api/m6 gets best value from /api/m1 Map connections = result.get(Objects.hash("1000", "/api/m7", "POST")).getConnections(); assertEquals(1, connections.size()); @@ -161,7 +161,7 @@ public void testAnalyse3() { TreeHelper treeHelper = new TreeHelper(); treeHelper.buildTree("1000", "/api/m7", "POST"); Map result = treeHelper.result; - assertEquals(6, result.size()); // this is because /api/m6 has 2 parameters getting data + assertEquals(7, result.size()); // this is because /api/m6 has 2 parameters getting data Map connections = result.get(Objects.hash("1000", "/api/m7", "POST")).getConnections(); assertEquals(1, connections.size()); @@ -197,7 +197,7 @@ public void testAnalyse4() { TreeHelper treeHelper = new TreeHelper(); treeHelper.buildTree("1000", "api/cars/INTEGER", "POST"); Map result = treeHelper.result; - assertEquals(1, result.size()); // this is because /api/m6 has 2 parameters getting data + assertEquals(2, result.size()); // this is because /api/m6 has 2 parameters getting data Map connections = result.get(Objects.hash("1000", "api/cars/INTEGER", "POST")).getConnections(); assertEquals(1, connections.size()); diff --git a/apps/dashboard/src/main/java/com/akto/action/DependencyAction.java b/apps/dashboard/src/main/java/com/akto/action/DependencyAction.java index f522c1dd77..0575280189 100644 --- a/apps/dashboard/src/main/java/com/akto/action/DependencyAction.java +++ b/apps/dashboard/src/main/java/com/akto/action/DependencyAction.java @@ -14,7 +14,7 @@ import com.akto.dto.type.URLMethods.Method; import com.akto.log.LoggerMaker; import com.akto.runtime.RelationshipSync; -import com.akto.utils.Build; +import com.akto.test_editor.execution.Build; import com.akto.utils.Utils; import com.mongodb.BasicDBObject; import com.mongodb.ConnectionString; diff --git a/apps/dashboard/src/main/java/com/akto/action/HarAction.java b/apps/dashboard/src/main/java/com/akto/action/HarAction.java index 8b2b42f83a..4d0db6e7b6 100644 --- a/apps/dashboard/src/main/java/com/akto/action/HarAction.java +++ b/apps/dashboard/src/main/java/com/akto/action/HarAction.java @@ -80,7 +80,7 @@ public String execute() throws IOException { return ERROR.toUpperCase(); } } else { - Collection actionErrors = apiCollectionsAction.getActionErrors(); + Collection actionErrors = apiCollectionsAction.getActionErrors(); if (actionErrors != null && actionErrors.size() > 0) { for (String actionError: actionErrors) { addActionError(actionError); @@ -178,7 +178,7 @@ public void setTcpContent(byte[] tcpContent) { Awesome awesome = null; public String uploadTcp() { - + File tmpDir = FileUtils.getTempDirectory(); String filename = UUID.randomUUID().toString() + ".pcap"; File tcpDump = new File(tmpDir, filename); @@ -188,23 +188,23 @@ public String uploadTcp() { Awesome.GoString.ByValue str = new Awesome.GoString.ByValue(); str.p = tcpDump.getAbsolutePath(); str.n = str.p.length(); - + Awesome.GoString.ByValue str2 = new Awesome.GoString.ByValue(); str2.p = System.getenv("AKTO_KAFKA_BROKER_URL"); str2.n = str2.p.length(); - + awesome.readTcpDumpFile(str, str2 , apiCollectionId); - - return Action.SUCCESS.toUpperCase(); + + return Action.SUCCESS.toUpperCase(); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); - return Action.ERROR.toUpperCase(); + return Action.ERROR.toUpperCase(); } } - interface Awesome extends Library { + interface Awesome extends Library { public static class GoString extends Structure { /** C type : const char* */ public String p; @@ -224,8 +224,8 @@ public GoString(String p, long n) { public static class ByReference extends GoString implements Structure.ByReference {} public static class ByValue extends GoString implements Structure.ByValue {} } - + public void readTcpDumpFile(GoString.ByValue filepath, GoString.ByValue kafkaURL, long apiCollectionId); - + } -} +} \ No newline at end of file diff --git a/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java b/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java index 327821530c..0b4fc09e3c 100644 --- a/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java +++ b/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java @@ -1795,6 +1795,8 @@ public static Organization fetchAndSaveFeatureWiseAllowed(Organization organizat hotjarSiteId = OrganizationUtils.fetchHotjarSiteId(metaData); boolean telemetryEnabled = OrganizationUtils.fetchTelemetryEnabled(metaData); setTelemetrySettings(organization, telemetryEnabled); + boolean testTelemetryEnabled = OrganizationUtils.fetchTestTelemetryEnabled(metaData); + organization.setTestTelemetryEnabled(testTelemetryEnabled); loggerMaker.infoAndAddToDb("Processed org metadata",LogDb.DASHBOARD); @@ -1812,6 +1814,7 @@ public static Organization fetchAndSaveFeatureWiseAllowed(Organization organizat Updates.set(Organization.FEATURE_WISE_ALLOWED, featureWiseAllowed), Updates.set(Organization.GRACE_PERIOD, gracePeriod), Updates.set(Organization.HOTJAR_SITE_ID, hotjarSiteId), + Updates.set(Organization.TEST_TELEMETRY_ENABLED, testTelemetryEnabled), Updates.set(Organization.LAST_FEATURE_MAP_UPDATE, lastFeatureMapUpdate))); loggerMaker.infoAndAddToDb("Updated org",LogDb.DASHBOARD); diff --git a/apps/dashboard/src/main/java/com/akto/utils/GzipUtils.java b/apps/dashboard/src/main/java/com/akto/utils/GzipUtils.java deleted file mode 100644 index 121911a7e8..0000000000 --- a/apps/dashboard/src/main/java/com/akto/utils/GzipUtils.java +++ /dev/null @@ -1,49 +0,0 @@ -package com.akto.utils; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.util.Base64; -import java.util.zip.GZIPInputStream; -import java.util.zip.GZIPOutputStream; - -public class GzipUtils { - - public static String zipString(String input) { - if (input == null || input.isEmpty()) { - return input; - } - - try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); - GZIPOutputStream gzipOutputStream = new GZIPOutputStream(byteArrayOutputStream)) { - gzipOutputStream.write(input.getBytes(StandardCharsets.UTF_8)); - gzipOutputStream.close(); - return Base64.getEncoder().encodeToString(byteArrayOutputStream.toByteArray()); - } catch (IOException e) { - throw new RuntimeException("Failed to zip content", e); - } - } - - public static String unzipString(String zippedBase64Str) { - if (zippedBase64Str == null || zippedBase64Str.isEmpty()) { - return zippedBase64Str; - } - - byte[] decodedBytes = Base64.getDecoder().decode(zippedBase64Str); - - try (ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(decodedBytes); - GZIPInputStream gzipInputStream = new GZIPInputStream(byteArrayInputStream); - ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream()) { - - byte[] buffer = new byte[1024]; - int len; - while ((len = gzipInputStream.read(buffer)) != -1) { - byteArrayOutputStream.write(buffer, 0, len); - } - return byteArrayOutputStream.toString(StandardCharsets.UTF_8.name()); - } catch (IOException e) { - throw new RuntimeException("Failed to unzip content", e); - } - } -} diff --git a/apps/dashboard/src/main/java/com/akto/utils/Utils.java b/apps/dashboard/src/main/java/com/akto/utils/Utils.java index b22c2b2f6d..568e1ffca8 100644 --- a/apps/dashboard/src/main/java/com/akto/utils/Utils.java +++ b/apps/dashboard/src/main/java/com/akto/utils/Utils.java @@ -158,6 +158,11 @@ public static boolean isValidURL(String url) { new URL(url).toURI(); return true; } catch (MalformedURLException | URISyntaxException e) { + Pattern pattern = Pattern.compile("\\$\\{[^}]*\\}"); + Matcher matcher = pattern.matcher(url); + if (matcher.find()) { + return true; + } return false; } } @@ -470,7 +475,7 @@ public static void pushDataToKafka(int apiCollectionId, String topic, List { - let edge = connection["edges"][0] // todo: null check + let edge = connection["edges"][0] + + if (!edge) return let source = calculateNodeId(edge["apiCollectionId"], edge["url"], edge["method"]); let edgeId = source + "-" + id; diff --git a/apps/dashboard/src/main/java/com/akto/utils/Build.java b/apps/testing/src/main/java/com/akto/test_editor/execution/Build.java similarity index 93% rename from apps/dashboard/src/main/java/com/akto/utils/Build.java rename to apps/testing/src/main/java/com/akto/test_editor/execution/Build.java index 1f7f981fd9..3ecab13e43 100644 --- a/apps/dashboard/src/main/java/com/akto/utils/Build.java +++ b/apps/testing/src/main/java/com/akto/test_editor/execution/Build.java @@ -1,30 +1,26 @@ -package com.akto.utils; +package com.akto.test_editor.execution; -import com.akto.DaoInit; import com.akto.dao.DependencyFlowNodesDao; -import com.akto.dao.ModifyHostDetailsDao; -import com.akto.dao.ReplaceDetailsDao; import com.akto.dao.SampleDataDao; import com.akto.dao.context.Context; -import com.akto.dto.*; +import com.akto.dto.ApiInfo; +import com.akto.dto.OriginalHttpRequest; +import com.akto.dto.OriginalHttpResponse; +import com.akto.dto.RawApi; import com.akto.dto.dependency_flow.*; import com.akto.dto.testing.TestingRunConfig; import com.akto.dto.traffic.Key; import com.akto.dto.traffic.SampleData; import com.akto.dto.type.URLMethods; import com.akto.log.LoggerMaker; -import com.akto.parsers.HttpCallParser; import com.akto.runtime.policies.AuthPolicy; -import com.akto.test_editor.execution.Operations; import com.akto.testing.ApiExecutor; import com.akto.util.Constants; import com.akto.util.HttpRequestResponseUtils; import com.akto.util.JSONUtils; - import com.akto.util.modifier.SetValueModifier; import com.fasterxml.jackson.core.JsonFactory; import com.fasterxml.jackson.databind.ObjectMapper; -import com.mongodb.ConnectionString; import com.mongodb.client.model.Filters; import joptsimple.internal.Strings; import org.bson.conversions.Bson; @@ -37,12 +33,11 @@ public class Build { - private Map parentToChildMap = new HashMap<>(); + private Map parentToChildMap = new HashMap<>(); private static final LoggerMaker loggerMaker = new LoggerMaker(Build.class); - private void buildParentToChildMap(List nodes) { - parentToChildMap = new HashMap<>(); + public static void buildParentToChildMap(Collection nodes, Map parentToChildMap) { for (Node node: nodes) { if (node.getConnections() == null) continue; for (Connection connection: node.getConnections().values()) { @@ -69,7 +64,7 @@ private void buildParentToChildMap(List nodes) { } - private Map> buildLevelsToSampleDataMap(List nodes) { + public static Map> buildLevelsToSampleDataMap(List nodes) { // divide them into levels Map> levelsToSampleDataMap = new HashMap<>(); @@ -158,11 +153,11 @@ public void setIsSuccess(boolean success) { this.success = success; } - + } Set apisReplayedSet = new HashSet<>(); - public List runPerLevel(List sdList, Map modifyHostDetailMap, Map replaceDetailsMap) { + public static List runPerLevel(List sdList, Map modifyHostDetailMap, Map replaceDetailsMap, Map parentToChildMap, Set apisReplayedSet) { List runResults = new ArrayList<>(); for (SampleData sampleData: sdList) { Key id = sampleData.getId(); @@ -249,7 +244,7 @@ public List run(List apiCollectionsIds, List nodes = DependencyFlowNodesDao.instance.findNodesForCollectionIds(apiCollectionsIds,false,0, 10_000); - buildParentToChildMap(nodes); + buildParentToChildMap(nodes, parentToChildMap); Map> levelsToSampleDataMap = buildLevelsToSampleDataMap(nodes); List runResults = new ArrayList<>(); @@ -261,7 +256,7 @@ public List run(List apiCollectionsIds, List runResultsPerLevel = runPerLevel(sdList, modifyHostDetailMap, replaceDetailsMap); + List runResultsPerLevel = runPerLevel(sdList, modifyHostDetailMap, replaceDetailsMap, parentToChildMap, apisReplayedSet); runResults.addAll(runResultsPerLevel); loggerMaker.infoAndAddToDb("Finished running level " + level, LoggerMaker.LogDb.DASHBOARD); } catch (Exception e) { @@ -282,7 +277,7 @@ public List run(List apiCollectionsIds, List runResultsAll = runPerLevel(filtered, modifyHostDetailMap, replaceDetailsMap); + List runResultsAll = runPerLevel(filtered, modifyHostDetailMap, replaceDetailsMap, parentToChildMap, apisReplayedSet); runResults.addAll(runResultsAll); skip += limit; if (all.size() < limit) break; @@ -340,7 +335,7 @@ public static void modifyRequest(OriginalHttpRequest request, ReplaceDetail repl } - public List fillSdList(List sdList) { + public static List fillSdList(List sdList) { if (sdList == null || sdList.isEmpty()) return new ArrayList<>(); List filters = new ArrayList<>(); @@ -359,30 +354,12 @@ public List fillSdList(List sdList) { static ObjectMapper mapper = new ObjectMapper(); static JsonFactory factory = mapper.getFactory(); - public void fillReplaceDetailsMap(ReverseNode reverseNode, OriginalHttpResponse response, Map replaceDetailsMap) { + public static void fillReplaceDetailsMap(ReverseNode reverseNode, OriginalHttpResponse response, Map replaceDetailsMap) { if (reverseNode == null) return; Map deltaReplaceDetailsMap = new HashMap<>(); - String respPayload = response.getBody(); - Map> valuesMap = extractValuesFromPayload(respPayload); - - Map> responseHeaders = response.getHeaders(); - for (String headerKey: responseHeaders.keySet()) { - List values = responseHeaders.get(headerKey); - if (values == null) continue; - - if (headerKey.equalsIgnoreCase("set-cookie")) { - Map cookieMap = AuthPolicy.parseCookie(values); - for (String cookieKey : cookieMap.keySet()) { - String cookieVal = cookieMap.get(cookieKey); - valuesMap.put(cookieKey, new HashSet<>(Collections.singletonList(cookieVal))); - } - } else { - valuesMap.put(headerKey, new HashSet<>(values)); - } - - } + Map> valuesMap = getValuesMap(response); Map connections = reverseNode.getReverseConnections(); for (ReverseConnection reverseConnection: connections.values()) { @@ -421,4 +398,27 @@ public void fillReplaceDetailsMap(ReverseNode reverseNode, OriginalHttpResponse } + public static Map> getValuesMap(OriginalHttpResponse response) { + String respPayload = response.getBody(); + Map> valuesMap = extractValuesFromPayload(respPayload); + + Map> responseHeaders = response.getHeaders(); + for (String headerKey: responseHeaders.keySet()) { + List values = responseHeaders.get(headerKey); + if (values == null) continue; + + if (headerKey.equalsIgnoreCase("set-cookie")) { + Map cookieMap = AuthPolicy.parseCookie(values); + for (String cookieKey : cookieMap.keySet()) { + String cookieVal = cookieMap.get(cookieKey); + valuesMap.put(cookieKey, new HashSet<>(Collections.singletonList(cookieVal))); + } + } else { + valuesMap.put(headerKey, new HashSet<>(values)); + } + + } + + return valuesMap; + } } diff --git a/apps/testing/src/main/java/com/akto/test_editor/execution/Executor.java b/apps/testing/src/main/java/com/akto/test_editor/execution/Executor.java index f468d763a4..df25a5dbca 100644 --- a/apps/testing/src/main/java/com/akto/test_editor/execution/Executor.java +++ b/apps/testing/src/main/java/com/akto/test_editor/execution/Executor.java @@ -4,6 +4,8 @@ import com.akto.dao.billing.OrganizationsDao; import com.akto.dao.billing.TokensDao; import java.util.*; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import com.akto.dao.CustomAuthTypeDao; import com.akto.dao.context.Context; @@ -44,12 +46,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import java.net.URI; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.UUID; import org.json.JSONObject; @@ -74,9 +70,11 @@ public class Executor { public final String _HOST = "host"; public YamlTestResult execute(ExecutorNode node, RawApi rawApi, Map varMap, String logId, - AuthMechanism authMechanism, FilterNode validatorNode, ApiInfo.ApiInfoKey apiInfoKey, TestingRunConfig testingRunConfig, List customAuthTypes, boolean debug, List testLogs) { + AuthMechanism authMechanism, FilterNode validatorNode, ApiInfo.ApiInfoKey apiInfoKey, TestingRunConfig testingRunConfig, + List customAuthTypes, boolean debug, List testLogs, + Memory memory) { List result = new ArrayList<>(); - + ExecutionListBuilder executionListBuilder = new ExecutionListBuilder(); List executorNodes = new ArrayList<>(); ExecutionOrderResp executionOrderResp = executionListBuilder.parseExecuteOperations(node, executorNodes); @@ -143,9 +141,14 @@ public YamlTestResult execute(ExecutorNode node, RawApi rawApi, Map apiInfoKeys = new ArrayList<>(); + apiInfoKeys.add(apiInfoKey); + memory = new Memory(apiInfoKeys, new HashMap<>()); + } + workflowTest = buildWorkflowGraph(reqNodes, rawApi, authMechanism, customAuthTypes, apiInfoKey, varMap, validatorNode); + result.add(triggerMultiExecution(workflowTest, reqNodes, rawApi, authMechanism, customAuthTypes, apiInfoKey, varMap, validatorNode, debug, testLogs, memory)); yamlTestResult = new YamlTestResult(result, workflowTest); return yamlTestResult; @@ -171,35 +174,6 @@ public YamlTestResult execute(ExecutorNode node, RawApi rawApi, Map originalHostHeaders = rawApi.getRequest().getHeaders().getOrDefault(_HOST, new ArrayList<>()); - List attemptHostHeaders = testReq.getRequest().getHeaders().getOrDefault(_HOST, new ArrayList<>()); - - if (!originalHostHeaders.isEmpty() - && originalHostHeaders.get(0) != null - && !attemptHostHeaders.isEmpty() - && attemptHostHeaders.get(0) != null - && originalHostHeaders.get(0).equals(attemptHostHeaders.get(0)) - && testingRunConfig != null - && !StringUtils.isEmpty(testingRunConfig.getOverriddenTestAppUrl())) { - - String url = ApiExecutor.prepareUrl(testReq.getRequest(), testingRunConfig); - URI uri = new URI(url); - String host = uri.getHost(); - if (uri.getPort() != -1) { - host += ":" + uri.getPort(); - } - testReq.getRequest().getHeaders().put(_HOST, Collections.singletonList(host)); - } - - } catch (Exception e) { - testLogs.add(new TestingRunResult.TestLog(TestingRunResult.TestLogType.ERROR, "unable to update host header for overridden test URL")); - loggerMaker.errorAndAddToDb(e,"unable to update host header for overridden test URL", - LogDb.TESTING); - } - // follow redirects = true for now testResponse = ApiExecutor.sendRequest(testReq.getRequest(), followRedirect, testingRunConfig, debug, testLogs); requestSent = true; @@ -230,6 +204,60 @@ public YamlTestResult execute(ExecutorNode node, RawApi rawApi, Map originalHostHeaders = rawApi.getRequest().getHeaders().getOrDefault(_HOST, new ArrayList<>()); + if (!originalHostHeaders.isEmpty() && testingRunConfig != null + && !StringUtils.isEmpty(testingRunConfig.getOverriddenTestAppUrl())) { + + Pattern pattern = Pattern.compile("\\$\\{[^}]*\\}"); + Matcher matcher = pattern.matcher(testingRunConfig.getOverriddenTestAppUrl()); + if (matcher.find()) { + String match = matcher.group(0); + match = match.substring(2, match.length()); + match = match.substring(0, match.length() - 1); + String[] params = match.split("\\+"); + for (int i = 0; i < params.length; i++) { + url += resolveParam(params[i], rawApi); + } + testingRunConfig.setOverriddenTestAppUrl(url); + } else { + url = testingRunConfig.getOverriddenTestAppUrl(); + } + + String newUrl = ApiExecutor.replacePathFromConfig(rawApi.getRequest().getUrl(), testingRunConfig); + URI uri = new URI(newUrl); + String host = uri.getHost(); + if (uri.getPort() != -1) { + host += ":" + uri.getPort(); + } + rawApi.getRequest().getHeaders().put(_HOST, Collections.singletonList(host)); + rawApi.getRequest().setUrl(newUrl); + + } + + } catch (Exception e) { + //testLogs.add(new TestingRunResult.TestLog(TestingRunResult.TestLogType.ERROR, "unable to update host header for overridden test URL")); + loggerMaker.errorAndAddToDb(e,"unable to update host header for overridden test URL", + LogDb.TESTING); + } + } + + public String resolveParam(String param, RawApi rawApi) { + param = param.trim(); + String[] params = param.split("\\."); + + if (params.length == 1) { + return params[0]; + } + + String key = params[params.length - 1]; + String val = rawApi.getRequest().getHeaders().get(key).get(0); + return val; + } + private void overrideAuth(RawApi rawApi, AuthMechanism authMechanism) { List authParams = authMechanism.getAuthParams(); if (authParams == null || authParams.isEmpty()) { @@ -257,7 +285,7 @@ public WorkflowTest buildWorkflowGraph(ExecutorNode reqNodes, RawApi rawApi, Aut } public MultiExecTestResult triggerMultiExecution(WorkflowTest workflowTest, ExecutorNode reqNodes, RawApi rawApi, AuthMechanism authMechanism, - List customAuthTypes, ApiInfo.ApiInfoKey apiInfoKey, Map varMap, FilterNode validatorNode, boolean debug, List testLogs) { + List customAuthTypes, ApiInfo.ApiInfoKey apiInfoKey, Map varMap, FilterNode validatorNode, boolean debug, List testLogs, Memory memory) { ApiWorkflowExecutor apiWorkflowExecutor = new ApiWorkflowExecutor(); Graph graph = new Graph(); @@ -266,7 +294,7 @@ public MultiExecTestResult triggerMultiExecution(WorkflowTest workflowTest, Exec List executionOrder = new ArrayList<>(); WorkflowTestResult workflowTestResult = new WorkflowTestResult(id, workflowTest.getId(), new HashMap<>(), null, null); GraphExecutorRequest graphExecutorRequest = new GraphExecutorRequest(graph, graph.getNode("x1"), workflowTest, null, null, varMap, "conditional", workflowTestResult, new HashMap<>(), executionOrder); - GraphExecutorResult graphExecutorResult = apiWorkflowExecutor.init(graphExecutorRequest, debug, testLogs); + GraphExecutorResult graphExecutorResult = apiWorkflowExecutor.init(graphExecutorRequest, debug, testLogs, memory); return new MultiExecTestResult(graphExecutorResult.getWorkflowTestResult().getNodeResultMap(), graphExecutorResult.getVulnerable(), Confidence.HIGH, graphExecutorRequest.getExecutionOrder()); } diff --git a/apps/testing/src/main/java/com/akto/test_editor/execution/Memory.java b/apps/testing/src/main/java/com/akto/test_editor/execution/Memory.java new file mode 100644 index 0000000000..a66cb2bcc2 --- /dev/null +++ b/apps/testing/src/main/java/com/akto/test_editor/execution/Memory.java @@ -0,0 +1,271 @@ +package com.akto.test_editor.execution; + +import com.akto.dao.DependencyFlowNodesDao; +import com.akto.dao.SampleDataDao; +import com.akto.dto.ApiInfo; +import com.akto.dto.OriginalHttpRequest; +import com.akto.dto.OriginalHttpResponse; +import com.akto.dto.RawApi; +import com.akto.dto.dependency_flow.*; +import com.akto.dto.testing.TestingRunConfig; +import com.akto.dto.traffic.Key; +import com.akto.dto.traffic.SampleData; +import com.akto.dto.type.SingleTypeInfo; +import com.akto.testing.ApiExecutor; +import com.akto.types.CappedSet; +import com.akto.util.Constants; +import com.mongodb.client.model.Filters; +import org.bson.conversions.Bson; + +import java.util.*; + +import static com.akto.test_editor.execution.Build.*; + + +public class Memory { + + Map resultMap = new HashMap<>(); + private final Map parentToChildMap = new HashMap<>(); + private final Map nodesMap = new HashMap<>(); + + Map sampleDataMap = new HashMap<>(); + private Map assetsMap = new HashMap<>(); + + + private Map replaceDetailsMap = new HashMap<>(); + + public void findAssets(ApiInfo.ApiInfoKey apiInfoKey) { + List results = new ArrayList<>(); + Node node = DependencyFlowNodesDao.instance.findOne( + Filters.and( + Filters.eq("apiCollectionId", apiInfoKey.getApiCollectionId()+""), + Filters.eq("url", apiInfoKey.getUrl()), + Filters.eq("method", apiInfoKey.getMethod().name()) + ) + ); + + if (node == null || node.getConnections() == null) return; + + Map connections = node.getConnections(); + + for (String key: connections.keySet()) { + Connection connection = connections.get(key); + if (connection == null) continue; + if (connection.getIsHeader()) continue; + + SingleTypeInfo.ParamId paramId = new SingleTypeInfo.ParamId(apiInfoKey.getUrl(), apiInfoKey.getMethod().name(), -1, connection.getIsHeader(), connection.getParam(), SingleTypeInfo.GENERIC, apiInfoKey.getApiCollectionId(), connection.getIsUrlParam()); + results.add(paramId); + } + + if (!results.isEmpty()) { + SingleTypeInfo.ParamId paramId = results.get(0); + assetsMap.put(apiInfoKey, paramId); + } + + } + + public Memory(List apiInfoKeys, Map replaceDetailsMap) { + if (apiInfoKeys == null || apiInfoKeys.isEmpty()) return; + this.replaceDetailsMap = replaceDetailsMap; + + // find all parent APIs + TreeHelper treeHelper = new TreeHelper(); + for (ApiInfo.ApiInfoKey apiInfoKey: apiInfoKeys) { + treeHelper.buildTree(apiInfoKey.getApiCollectionId()+"", apiInfoKey.getUrl(), apiInfoKey.getMethod().name()); + } + Collection nodes = treeHelper.result.values(); + List filters = new ArrayList<>(); + for (Node node: nodes) { + nodesMap.put(node.hashCode(), node); + filters.add(Filters.and( + Filters.eq("_id.apiCollectionId", Integer.parseInt(node.getApiCollectionId())), + Filters.eq("_id.url", node.getUrl()), + Filters.eq("_id.method", node.getMethod()) + )); + } + + // fetch sample data + List sdList = SampleDataDao.instance.findAll(Filters.or(filters)); + for (SampleData sampleData: sdList) { + Key id = sampleData.getId(); + sampleDataMap.put(Objects.hash(id.getApiCollectionId(), id.getUrl(), id.getMethod().name()), sampleData); + } + + buildParentToChildMap(nodes, parentToChildMap); + + for (ApiInfo.ApiInfoKey apiInfoKey: apiInfoKeys) findAssets(apiInfoKey); + } + + + public OriginalHttpRequest run(int apiCollectionId, String url, String method) { + int hash = Objects.hash(apiCollectionId+"", url, method); + if (resultMap.get(hash) != null) return resultMap.get(hash).getRequest(); + + // todo: optimise this.. no need to make db calls everytime + TreeHelper treeHelper = new TreeHelper(); + treeHelper.buildTree(apiCollectionId+"", url, method); + List nodes = new ArrayList<>(treeHelper.result.values()); + + nodes.sort(Comparator.comparingInt(Node::getMaxDepth)); + + List sampleDataList = new ArrayList<>(); + for (Node node: nodes) { + Integer nodeHash = Objects.hash(Integer.parseInt(node.getApiCollectionId()), node.getUrl(), node.getMethod()); + sampleDataList.add(sampleDataMap.get(nodeHash)); + } + + return execute(sampleDataList); + } + + public RawApi findAssetGetterRequest(ApiInfo.ApiInfoKey apiInfoKey) { + SingleTypeInfo.ParamId paramId = assetsMap.get(apiInfoKey); + // find getter API + Node node = DependencyFlowNodesDao.instance.findOne( + Filters.and( + Filters.eq("apiCollectionId", apiInfoKey.getApiCollectionId()+""), + Filters.eq("url", apiInfoKey.getUrl()), + Filters.eq("method", apiInfoKey.getMethod().name()) + ) + ); + if (node == null || node.getConnections() == null) return null; + + Map connections = node.getConnections(); + + int apiCollectionId = 0; + String url = null; + String method = null; + + for (String key: connections.keySet()) { + Connection connection = connections.get(key); + if (connection == null) continue; + if (connection.getIsHeader()) continue; + + String connectionParam = connection.getParam(); + String param = paramId.getParam(); + if (!param.equals(connectionParam)) continue; + + if ((paramId.getIsUrlParam() && connection.getIsUrlParam()) || (!paramId.getIsUrlParam() || !connection.getIsUrlParam())) { + List edges = connection.getEdges(); + if (edges.isEmpty()) continue; + + Edge edge = edges.get(0); + apiCollectionId = Integer.parseInt(edge.getApiCollectionId()); + url = edge.getUrl(); + method = edge.getMethod(); + } + } + + if (url == null) return null; + + // find sample message from result map + int hash = Objects.hash(apiCollectionId+"", url, method); + + // return the request + return resultMap.get(hash); + } + + + private OriginalHttpRequest execute(List sdList) { + int idx = 0; + for (SampleData sampleData: sdList) { + idx++; + boolean isFinal = sdList.size() == idx; // todo: find a better way + Key id = sampleData.getId(); + int hash = Objects.hash(id.getApiCollectionId()+"", id.getUrl(), id.getMethod().name()); + if (resultMap.containsKey(hash)) continue; + try { + List samples = sampleData.getSamples(); + if (samples.isEmpty()) continue;; + + String sample = samples.get(0); + OriginalHttpRequest request = new OriginalHttpRequest(); + request.buildFromSampleMessage(sample); + // todo: String newHost = findNewHost(request, modifyHostDetailMap); + String newHost = null; + + OriginalHttpResponse originalHttpResponse = new OriginalHttpResponse(); + originalHttpResponse.buildFromSampleMessage(sample); + + // do modifications + Node node = nodesMap.get(hash); + ReplaceDetail finalReplaceDetail = getReplaceDetail(node); + modifyRequest(request, finalReplaceDetail); + + if (isFinal) return request; + + TestingRunConfig testingRunConfig = new TestingRunConfig(0, new HashMap<>(), new ArrayList<>(), null,newHost, null); + + OriginalHttpResponse response = null; + try { + response = ApiExecutor.sendRequest(request,false, testingRunConfig, false, new ArrayList<>()); + request.getHeaders().remove(Constants.AKTO_IGNORE_FLAG); + RawApi rawApi = new RawApi(request, response, ""); + resultMap.put(hash, rawApi); + } catch (Exception e) { + e.printStackTrace(); + } + } catch (Exception e) { + e.printStackTrace(); + } + + } + + return null; + } + + + public void fillResponse(OriginalHttpRequest request, OriginalHttpResponse response, int apiCollectionId, String url, String method) { + int hash = Objects.hash(apiCollectionId+"", url, method); + RawApi rawApi = new RawApi(request, response, ""); + resultMap.put(hash, rawApi); + } + + public void reset(int apiCollectionId, String url, String method) { + // find all children + int hash = Objects.hash(apiCollectionId+"", url, method); + ReverseNode reverseNode = parentToChildMap.get(hash); + if (reverseNode == null) return; + + for (ReverseConnection reverseConnection: reverseNode.getReverseConnections().values()) { + for (ReverseEdge reverseEdge: reverseConnection.getReverseEdges()) { + int childApiCollectionId = Integer.parseInt(reverseEdge.getApiCollectionId()); + String childUrl = reverseEdge.getUrl(); + String childMethod = reverseEdge.getMethod(); + + int childHash = Objects.hash(childApiCollectionId+"", childUrl, childMethod); + resultMap.remove(childHash); + reset(childApiCollectionId, childUrl, childMethod); + } + } + } + + + public ReplaceDetail getReplaceDetail(Node node) { + ReplaceDetail replaceDetail = new ReplaceDetail(Integer.parseInt(node.getApiCollectionId()), node.getUrl(), node.getMethod(), new ArrayList<>()); + Map connections = node.getConnections(); + for (Connection connection: connections.values()) { + String requestParam = connection.getParam(); + List edges = connection.getEdges(); + if (edges.isEmpty()) continue; + Edge edge = edges.get(0); + String responseParam = edge.getParam(); + String parentApiCollectionId = edge.getApiCollectionId(); + String parentUrl = edge.getUrl(); + String parentMethod = edge.getMethod(); + int parentHash = Objects.hash(parentApiCollectionId, parentUrl, parentMethod); + RawApi rawApi = resultMap.get(parentHash); + if (rawApi == null) continue; + Map> valuesMap = getValuesMap(rawApi.getResponse()); + Set values = valuesMap.get(responseParam); + Object value = values != null && values.size() > 0 ? values.toArray()[0] : null; // todo: + if (value == null) continue; + + KVPair.KVType type = value instanceof Integer ? KVPair.KVType.INTEGER : KVPair.KVType.STRING; + KVPair kvPair = new KVPair(requestParam, value.toString(), connection.getIsHeader(), connection.getIsUrlParam(), type); + replaceDetail.addIfNotExist(kvPair); + } + + return replaceDetail; + } + +} diff --git a/apps/testing/src/main/java/com/akto/testing/ApiWorkflowExecutor.java b/apps/testing/src/main/java/com/akto/testing/ApiWorkflowExecutor.java index eaf243170f..caaa6fa05c 100644 --- a/apps/testing/src/main/java/com/akto/testing/ApiWorkflowExecutor.java +++ b/apps/testing/src/main/java/com/akto/testing/ApiWorkflowExecutor.java @@ -1,16 +1,19 @@ package com.akto.testing; +import com.akto.dto.ApiInfo; import com.akto.dto.testing.*; +import com.akto.test_editor.execution.Memory; import com.akto.testing.workflow_node_executor.GraphExecutor; import com.akto.testing.workflow_node_executor.GraphExecutorFactory; import java.util.List; +import java.util.Map; public class ApiWorkflowExecutor { - public GraphExecutorResult init(GraphExecutorRequest graphExecutorRequest, boolean debug, List testLogs) { + public GraphExecutorResult init(GraphExecutorRequest graphExecutorRequest, boolean debug, List testLogs, Memory memory) { GraphExecutor graphExecutor = GraphExecutorFactory.fetchExecutor(graphExecutorRequest); - GraphExecutorResult graphExecutorResult = graphExecutor.executeGraph(graphExecutorRequest,debug,testLogs); + GraphExecutorResult graphExecutorResult = graphExecutor.executeGraph(graphExecutorRequest,debug,testLogs, memory); return graphExecutorResult; } diff --git a/apps/testing/src/main/java/com/akto/testing/Main.java b/apps/testing/src/main/java/com/akto/testing/Main.java index f11501f3b7..b5ff59fdd4 100644 --- a/apps/testing/src/main/java/com/akto/testing/Main.java +++ b/apps/testing/src/main/java/com/akto/testing/Main.java @@ -6,6 +6,7 @@ import com.akto.dao.AccountsDao; import com.akto.dao.SetupDao; import com.akto.dao.MCollection; +import com.akto.dao.billing.OrganizationsDao; import com.akto.dao.context.Context; import com.akto.dao.testing.TestingRunConfigDao; import com.akto.dao.testing.TestingRunDao; @@ -16,6 +17,7 @@ import com.akto.dto.AccountSettings; import com.akto.dto.ApiInfo; import com.akto.dto.Setup; +import com.akto.dto.billing.Organization; import com.akto.dto.test_run_findings.TestingIssuesId; import com.akto.dto.test_run_findings.TestingRunIssues; import com.akto.dto.testing.TestResult; @@ -65,6 +67,9 @@ public class Main { public static final ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(2); + + public static final ScheduledExecutorService testTelemetryScheduler = Executors.newScheduledThreadPool(2); + public static final ScheduledExecutorService schedulerAccessMatrix = Executors.newScheduledThreadPool(2); public static final boolean SKIP_SSRF_CHECK = "true".equalsIgnoreCase(System.getenv("SKIP_SSRF_CHECK")); @@ -191,7 +196,7 @@ private static void setTestingRunConfig(TestingRun testingRun, TestingRunResultS } public static void main(String[] args) throws InterruptedException { - String mongoURI = System.getenv("AKTO_MONGO_CONN");; + String mongoURI = System.getenv("AKTO_MONGO_CONN"); DaoInit.init(new ConnectionString(mongoURI)); boolean connectedToMongo = false; @@ -396,6 +401,27 @@ public void run() { } loggerMaker.infoAndAddToDb("Tests completed in " + (Context.now() - start) + " seconds for account: " + accountId, LogDb.TESTING); + + Organization organization = OrganizationsDao.instance.findOne( + Filters.in(Organization.ACCOUNTS, Context.accountId.get())); + + if(organization.getTestTelemetryEnabled()){ + loggerMaker.infoAndAddToDb("Test telemetry enabled for account: " + accountId + ", sending results", LogDb.TESTING); + ObjectId finalSummaryId = summaryId; + testTelemetryScheduler.execute(() -> { + Context.accountId.set(accountId); + try { + com.akto.onprem.Constants.sendTestResults(finalSummaryId, organization); + loggerMaker.infoAndAddToDb("Test telemetry sent for account: " + accountId, LogDb.TESTING); + } catch (Exception e) { + loggerMaker.errorAndAddToDb(e, "Error in sending test telemetry for account: " + accountId); + } + }); + + } else { + loggerMaker.infoAndAddToDb("Test telemetry disabled for account: " + accountId, LogDb.TESTING); + } + }, "testing"); Thread.sleep(1000); } diff --git a/apps/testing/src/main/java/com/akto/testing/TestExecutor.java b/apps/testing/src/main/java/com/akto/testing/TestExecutor.java index a4222c151e..9a8b2a87a4 100644 --- a/apps/testing/src/main/java/com/akto/testing/TestExecutor.java +++ b/apps/testing/src/main/java/com/akto/testing/TestExecutor.java @@ -17,10 +17,7 @@ import com.akto.dto.OriginalHttpRequest; import com.akto.dto.RawApi; import com.akto.dto.api_workflow.Graph; -import com.akto.dto.test_editor.Auth; -import com.akto.dto.test_editor.ExecutorNode; -import com.akto.dto.test_editor.FilterNode; -import com.akto.dto.test_editor.TestConfig; +import com.akto.dto.test_editor.*; import com.akto.dto.testing.*; import com.akto.dto.testing.TestResult.Confidence; import com.akto.dto.testing.TestResult.TestError; @@ -34,6 +31,7 @@ import com.akto.store.AuthMechanismStore; import com.akto.store.SampleMessageStore; import com.akto.store.TestingUtil; +import com.akto.test_editor.execution.Executor; import com.akto.test_editor.execution.VariableResolver; import com.akto.testing.yaml_tests.YamlTestTemplate; import com.akto.testing_issues.TestingIssuesHandler; @@ -46,11 +44,6 @@ import com.mongodb.MongoInterruptedException; import org.apache.commons.lang3.StringUtils; import com.mongodb.BasicDBObject; -import com.mongodb.client.model.Aggregates; -import com.mongodb.client.model.Filters; -import com.mongodb.client.model.Projections; -import com.mongodb.client.model.Sorts; -import com.mongodb.client.model.Updates; import com.mongodb.client.model.*; import org.bson.conversions.Bson; @@ -110,7 +103,7 @@ public void workflowInit (TestingRun testingRun, ObjectId summaryId, boolean deb Graph graph = new Graph(); graph.buildGraph(workflowTest); GraphExecutorRequest graphExecutorRequest = new GraphExecutorRequest(graph, workflowTest, testingRun.getId(), summaryId, valuesMap, false, "linear"); - GraphExecutorResult graphExecutorResult = apiWorkflowExecutor.init(graphExecutorRequest, debug, testLogs); + GraphExecutorResult graphExecutorResult = apiWorkflowExecutor.init(graphExecutorRequest, debug, testLogs, null); WorkflowTestResultsDao.instance.insertOne(graphExecutorResult.getWorkflowTestResult()); } catch (Exception e) { loggerMaker.errorAndAddToDb("Error while executing workflow test " + e, LogDb.TESTING); @@ -678,6 +671,8 @@ public TestingRunResult runTestNew(ApiInfo.ApiInfoKey apiInfoKey, ObjectId testR List customAuthTypes = testingUtil.getCustomAuthTypes(); // TestingUtil -> authMechanism // TestingConfig -> auth + com.akto.test_editor.execution.Executor executor = new Executor(); + executor.overrideTestUrl(rawApi, testingRunConfig); YamlTestTemplate yamlTestTemplate = new YamlTestTemplate(apiInfoKey,filterNode, validatorNode, executorNode, rawApi, varMap, auth, testingUtil.getAuthMechanism(), testExecutionLogId, testingRunConfig, customAuthTypes, testConfig.getStrategy()); YamlTestResult testResults = yamlTestTemplate.run(debug, testLogs); diff --git a/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/ApiNodeExecutor.java b/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/ApiNodeExecutor.java index c79885d67c..92645aa6c6 100644 --- a/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/ApiNodeExecutor.java +++ b/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/ApiNodeExecutor.java @@ -5,6 +5,7 @@ import java.util.Map; import java.util.Collections; +import com.akto.dto.ApiInfo; import com.akto.dto.OriginalHttpRequest; import com.akto.dto.OriginalHttpResponse; import com.akto.dto.api_workflow.Node; @@ -15,6 +16,7 @@ import com.akto.dto.testing.WorkflowTestResult.NodeResult; import com.akto.log.LoggerMaker; import com.akto.log.LoggerMaker.LogDb; +import com.akto.test_editor.execution.Memory; import com.akto.testing.ApiExecutor; import com.akto.testing.Main; import com.akto.testing.Utils; @@ -24,7 +26,7 @@ public class ApiNodeExecutor extends NodeExecutor { private static final LoggerMaker loggerMaker = new LoggerMaker(ApiNodeExecutor.class); - public NodeResult processNode(Node node, Map valuesMap, Boolean allowAllStatusCodes, boolean debug, List testLogs) { + public NodeResult processNode(Node node, Map valuesMap, Boolean allowAllStatusCodes, boolean debug, List testLogs, Memory memory) { loggerMaker.infoAndAddToDb("\n", LogDb.TESTING); loggerMaker.infoAndAddToDb("NODE: " + node.getId(), LogDb.TESTING); List testErrors = new ArrayList<>(); diff --git a/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/ConditionalGraphExecutor.java b/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/ConditionalGraphExecutor.java index 903921508b..80d68cc439 100644 --- a/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/ConditionalGraphExecutor.java +++ b/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/ConditionalGraphExecutor.java @@ -5,16 +5,18 @@ import java.util.Map; import com.akto.dao.test_editor.TestEditorEnums; +import com.akto.dto.ApiInfo; import com.akto.dto.api_workflow.Node; import com.akto.dto.test_editor.DataOperandsFilterResponse; import com.akto.dto.test_editor.ExecutorNode; import com.akto.dto.test_editor.FilterNode; import com.akto.dto.testing.*; +import com.akto.test_editor.execution.Memory; import com.akto.test_editor.filter.Filter; public class ConditionalGraphExecutor extends GraphExecutor { - public GraphExecutorResult executeGraph(GraphExecutorRequest graphExecutorRequest,boolean debug, List testLogs) { + public GraphExecutorResult executeGraph(GraphExecutorRequest graphExecutorRequest, boolean debug, List testLogs, Memory memory) { Map visitedMap = graphExecutorRequest.getVisitedMap(); List errors = new ArrayList<>(); @@ -30,7 +32,7 @@ public GraphExecutorResult executeGraph(GraphExecutorRequest graphExecutorReques boolean success = false; WorkflowTestResult.NodeResult nodeResult; - nodeResult = Utils.executeNode(node, graphExecutorRequest.getValuesMap(), debug, testLogs); + nodeResult = Utils.executeNode(node, graphExecutorRequest.getValuesMap(), debug, testLogs, memory); graphExecutorRequest.getWorkflowTestResult().getNodeResultMap().put(node.getId(), nodeResult); graphExecutorRequest.getExecutionOrder().add(node.getId()); @@ -77,7 +79,7 @@ public GraphExecutorResult executeGraph(GraphExecutorRequest graphExecutorReques boolean vulnerable = success; if (childNode != null) { GraphExecutorRequest childExecReq = new GraphExecutorRequest(graphExecutorRequest, childNode, graphExecutorRequest.getWorkflowTestResult(), visitedMap, graphExecutorRequest.getExecutionOrder()); - GraphExecutorResult childExecResult = executeGraph(childExecReq, debug, testLogs); + GraphExecutorResult childExecResult = executeGraph(childExecReq, debug, testLogs, memory); vulnerable = childExecResult.getVulnerable(); return new GraphExecutorResult(graphExecutorRequest.getWorkflowTestResult(), vulnerable, errors); } else { diff --git a/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/GraphExecutor.java b/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/GraphExecutor.java index 42faeac0a7..cc876d2a3c 100644 --- a/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/GraphExecutor.java +++ b/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/GraphExecutor.java @@ -1,13 +1,16 @@ package com.akto.testing.workflow_node_executor; +import com.akto.dto.ApiInfo; import com.akto.dto.testing.GraphExecutorRequest; import com.akto.dto.testing.GraphExecutorResult; import com.akto.dto.testing.TestingRunResult; +import com.akto.test_editor.execution.Memory; import java.util.List; +import java.util.Map; public abstract class GraphExecutor { - public abstract GraphExecutorResult executeGraph(GraphExecutorRequest graphExecutorRequest, boolean debug, List testLogs); + public abstract GraphExecutorResult executeGraph(GraphExecutorRequest graphExecutorRequest, boolean debug, List testLogs, Memory memory); } diff --git a/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/LinearGraphExecutor.java b/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/LinearGraphExecutor.java index 6f269502bf..5cc7ace500 100644 --- a/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/LinearGraphExecutor.java +++ b/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/LinearGraphExecutor.java @@ -6,15 +6,17 @@ import java.util.Map; import com.akto.dao.context.Context; +import com.akto.dto.ApiInfo; import com.akto.dto.api_workflow.Node; import com.akto.dto.testing.GraphExecutorRequest; import com.akto.dto.testing.GraphExecutorResult; import com.akto.dto.testing.TestingRunResult; import com.akto.dto.testing.WorkflowTestResult; +import com.akto.test_editor.execution.Memory; public class LinearGraphExecutor extends GraphExecutor { - public GraphExecutorResult executeGraph(GraphExecutorRequest graphExecutorRequest, boolean debug, List testLogs) { + public GraphExecutorResult executeGraph(GraphExecutorRequest graphExecutorRequest, boolean debug, List testLogs, Memory memory) { List nodes = graphExecutorRequest.getGraph().sort(); int id = Context.now(); @@ -22,7 +24,7 @@ public GraphExecutorResult executeGraph(GraphExecutorRequest graphExecutorReques Map testResultMap = workflowTestResult.getNodeResultMap(); for (Node node: nodes) { WorkflowTestResult.NodeResult nodeResult; - nodeResult = Utils.executeNode(node, graphExecutorRequest.getValuesMap(), debug, testLogs); + nodeResult = Utils.executeNode(node, graphExecutorRequest.getValuesMap(), debug, testLogs, memory); testResultMap.put(node.getId(), nodeResult); if (nodeResult.getErrors().size() > 0) break; if (graphExecutorRequest.getSkipIfNotVulnerable() && !nodeResult.isVulnerable()) { diff --git a/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/NodeExecutor.java b/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/NodeExecutor.java index 5fd8c29f34..68e5b00c00 100644 --- a/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/NodeExecutor.java +++ b/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/NodeExecutor.java @@ -3,11 +3,13 @@ import java.util.List; import java.util.Map; +import com.akto.dto.ApiInfo; import com.akto.dto.api_workflow.Node; import com.akto.dto.testing.TestingRunResult; import com.akto.dto.testing.WorkflowTestResult; +import com.akto.test_editor.execution.Memory; public abstract class NodeExecutor { - public abstract WorkflowTestResult.NodeResult processNode(Node node, Map valuesMap, Boolean allowAllStatusCodes, boolean debug, List testLogs); + public abstract WorkflowTestResult.NodeResult processNode(Node node, Map valuesMap, Boolean allowAllStatusCodes, boolean debug, List testLogs, Memory memory); } diff --git a/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/Utils.java b/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/Utils.java index e2cfcb3256..216d4afec7 100644 --- a/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/Utils.java +++ b/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/Utils.java @@ -14,7 +14,9 @@ import javax.script.ScriptEngineManager; import javax.script.ScriptException; +import com.akto.dto.ApiInfo; import com.akto.dto.testing.*; +import com.akto.test_editor.execution.Memory; import org.apache.commons.lang3.StringUtils; import org.bson.conversions.Bson; import org.json.JSONObject; @@ -187,7 +189,7 @@ public static String fetchToken(RecordedLoginFlowInput recordedLoginFlowInput, i return token; } - public static WorkflowTestResult.NodeResult processNode(Node node, Map valuesMap, Boolean allowAllStatusCodes, boolean debug, List testLogs) { + public static WorkflowTestResult.NodeResult processNode(Node node, Map valuesMap, Boolean allowAllStatusCodes, boolean debug, List testLogs, Memory memory) { if (node.getWorkflowNodeDetails().getType() == WorkflowNodeDetails.Type.RECORDED) { return processRecorderNode(node, valuesMap); } @@ -195,22 +197,22 @@ else if (node.getWorkflowNodeDetails().getType() == WorkflowNodeDetails.Type.OTP return processOtpNode(node, valuesMap); } else { - return processApiNode(node, valuesMap, allowAllStatusCodes, debug, testLogs); + return processApiNode(node, valuesMap, allowAllStatusCodes, debug, testLogs, memory); } } - public static WorkflowTestResult.NodeResult processApiNode(Node node, Map valuesMap, Boolean allowAllStatusCodes, boolean debug, List testLogs) { + public static WorkflowTestResult.NodeResult processApiNode(Node node, Map valuesMap, Boolean allowAllStatusCodes, boolean debug, List testLogs, Memory memory) { NodeExecutorFactory nodeExecutorFactory = new NodeExecutorFactory(); NodeExecutor nodeExecutor = nodeExecutorFactory.getExecutor(node); - return nodeExecutor.processNode(node, valuesMap, allowAllStatusCodes, debug, testLogs); + return nodeExecutor.processNode(node, valuesMap, allowAllStatusCodes, debug, testLogs, memory); } - public static WorkflowTestResult.NodeResult executeNode(Node node, Map valuesMap,boolean debug, List testLogs) { + public static WorkflowTestResult.NodeResult executeNode(Node node, Map valuesMap,boolean debug, List testLogs, Memory memory) { WorkflowTestResult.NodeResult nodeResult; try { - nodeResult = Utils.processNode(node, valuesMap, true, debug, testLogs); + nodeResult = Utils.processNode(node, valuesMap, true, debug, testLogs, memory); } catch (Exception e) { ; List testErrors = new ArrayList<>(); @@ -244,7 +246,7 @@ public static LoginFlowResponse runLoginFlow(WorkflowTest workflowTest, AuthMech if (authMechanism.getRequestData() != null && authMechanism.getRequestData().size() > 0 && authMechanism.getRequestData().get(index).getAllowAllStatusCodes()) { allowAllStatusCodes = authMechanism.getRequestData().get(0).getAllowAllStatusCodes(); } - nodeResult = processNode(node, valuesMap, allowAllStatusCodes, false, new ArrayList<>()); + nodeResult = processNode(node, valuesMap, allowAllStatusCodes, false, new ArrayList<>(), null); } catch (Exception e) { ; List testErrors = new ArrayList<>(); diff --git a/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/YamlNodeExecutor.java b/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/YamlNodeExecutor.java index 5a42505a90..6f8eb0d787 100644 --- a/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/YamlNodeExecutor.java +++ b/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/YamlNodeExecutor.java @@ -7,16 +7,16 @@ import java.util.List; import java.util.Map; +import com.akto.dto.type.SingleTypeInfo; import com.akto.testing.Main; +import com.akto.dto.*; +import com.akto.dto.type.URLMethods; +import com.akto.test_editor.execution.Memory; import org.json.JSONObject; import com.akto.dao.context.Context; import com.akto.dao.test_editor.TestEditorEnums; import com.akto.dao.test_editor.YamlTemplateDao; -import com.akto.dto.ApiInfo; -import com.akto.dto.CustomAuthType; -import com.akto.dto.OriginalHttpResponse; -import com.akto.dto.RawApi; import com.akto.dto.api_workflow.Node; import com.akto.dto.test_editor.ConfigParserResult; import com.akto.dto.test_editor.ExecuteAlgoObj; @@ -49,7 +49,8 @@ public class YamlNodeExecutor extends NodeExecutor { private static final Gson gson = new Gson(); - public NodeResult processNode(Node node, Map varMap, Boolean allowAllStatusCodes, boolean debug, List testLogs) { + + public NodeResult processNode(Node node, Map varMap, Boolean allowAllStatusCodes, boolean debug, List testLogs, Memory memory) { List testErrors = new ArrayList<>(); YamlNodeDetails yamlNodeDetails = (YamlNodeDetails) node.getWorkflowNodeDetails(); @@ -57,17 +58,43 @@ public NodeResult processNode(Node node, Map varMap, Boolean all if (yamlNodeDetails.getTestId() != null && yamlNodeDetails.getTestId().length() > 0) { return processYamlNode(node, varMap, allowAllStatusCodes, yamlNodeDetails, debug, testLogs); } - + RawApi rawApi = yamlNodeDetails.getRawApi(); RawApi sampleRawApi = rawApi.copy(); - List rawApis = new ArrayList<>(); - rawApis.add(rawApi.copy()); Executor executor = new Executor(); ExecutorNode executorNode = yamlNodeDetails.getExecutorNode(); FilterNode validatorNode = yamlNodeDetails.getValidatorNode(); + List childNodes = executorNode.getChildNodes(); + + ApiInfo.ApiInfoKey apiInfoKey = ((YamlNodeDetails) node.getWorkflowNodeDetails()).getApiInfoKey(); + ExecutorNode firstChildNode = childNodes.get(0); // todo check for length + if (memory != null) { + if (firstChildNode.getOperationType().equalsIgnoreCase("api")) { + String apiType = firstChildNode.getValues().toString(); + if (apiType.equalsIgnoreCase("get_asset_api")) { + rawApi = memory.findAssetGetterRequest(apiInfoKey); + if (rawApi == null) { + testErrors.add("Couldn't find corresponding getter api"); + new WorkflowTestResult.NodeResult("[]",false, testErrors); + } + } + childNodes.remove(0); + } else { + OriginalHttpRequest request = memory.run(apiInfoKey.getApiCollectionId(), apiInfoKey.getUrl(), apiInfoKey.getMethod().name()); + if (request == null) { + testErrors.add("Failed getting request from dependency graph"); + new WorkflowTestResult.NodeResult("[]",false, testErrors); + } + rawApi.setRequest(request); + } + } - for (ExecutorNode execNode: executorNode.getChildNodes()) { + + List rawApis = new ArrayList<>(); + rawApis.add(rawApi.copy()); + + for (ExecutorNode execNode: childNodes) { if (execNode.getNodeType().equalsIgnoreCase(TestEditorEnums.ValidateExecutorDataOperands.Validate.toString())) { validatorNode = (FilterNode) execNode.getChildNodes().get(0).getValues(); } @@ -120,6 +147,10 @@ public NodeResult processNode(Node node, Map varMap, Boolean all try { tsBeforeReq = Context.nowInMillis(); testResponse = ApiExecutor.sendRequest(testReq.getRequest(), followRedirect, testingRunConfig, debug, testLogs, Main.SKIP_SSRF_CHECK); + if (apiInfoKey != null && memory != null) { + memory.fillResponse(testReq.getRequest(), testResponse, apiInfoKey.getApiCollectionId(), apiInfoKey.getUrl(), apiInfoKey.getMethod().name()); + memory.reset(apiInfoKey.getApiCollectionId(), apiInfoKey.getUrl(), apiInfoKey.getMethod().name()); + } tsAfterReq = Context.nowInMillis(); responseTimeArr.add(tsAfterReq - tsBeforeReq); ExecutionResult attempt = new ExecutionResult(singleReq.getSuccess(), singleReq.getErrMsg(), testReq.getRequest(), testResponse); diff --git a/apps/testing/src/main/java/com/akto/testing/yaml_tests/SecurityTestTemplate.java b/apps/testing/src/main/java/com/akto/testing/yaml_tests/SecurityTestTemplate.java index c32c517024..9027116dbf 100644 --- a/apps/testing/src/main/java/com/akto/testing/yaml_tests/SecurityTestTemplate.java +++ b/apps/testing/src/main/java/com/akto/testing/yaml_tests/SecurityTestTemplate.java @@ -8,6 +8,7 @@ import com.akto.dto.test_editor.Strategy; import com.akto.dto.testing.*; import com.akto.dto.testing.TestResult.TestError; +import com.akto.test_editor.execution.Memory; import java.util.Collections; import java.util.ArrayList; @@ -31,6 +32,8 @@ public abstract class SecurityTestTemplate { TestingRunConfig testingRunConfig; Strategy strategy; + Memory memory; + public SecurityTestTemplate(ApiInfo.ApiInfoKey apiInfoKey, FilterNode filterNode, FilterNode validatorNode, ExecutorNode executorNode ,RawApi rawApi, Map varMap, Auth auth, AuthMechanism authMechanism, String logId, TestingRunConfig testingRunConfig, Strategy strategy) { this.apiInfoKey = apiInfoKey; this.filterNode = filterNode; @@ -148,5 +151,5 @@ public String getLogId() { public void setLogId(String logId) { this.logId = logId; } - + } diff --git a/apps/testing/src/main/java/com/akto/testing/yaml_tests/YamlTestTemplate.java b/apps/testing/src/main/java/com/akto/testing/yaml_tests/YamlTestTemplate.java index e49fd0983c..1674328e6b 100644 --- a/apps/testing/src/main/java/com/akto/testing/yaml_tests/YamlTestTemplate.java +++ b/apps/testing/src/main/java/com/akto/testing/yaml_tests/YamlTestTemplate.java @@ -4,11 +4,7 @@ import com.akto.dto.CustomAuthType; import com.akto.dto.OriginalHttpResponse; import com.akto.dto.RawApi; -import com.akto.dto.test_editor.Auth; -import com.akto.dto.test_editor.ExecutionResult; -import com.akto.dto.test_editor.ExecutorNode; -import com.akto.dto.test_editor.FilterNode; -import com.akto.dto.test_editor.Strategy; +import com.akto.dto.test_editor.*; import com.akto.dto.testing.*; import com.akto.log.LoggerMaker; import com.akto.rules.TestPlugin; @@ -25,7 +21,8 @@ public class YamlTestTemplate extends SecurityTestTemplate { private final List customAuthTypes; public YamlTestTemplate(ApiInfo.ApiInfoKey apiInfoKey, FilterNode filterNode, FilterNode validatorNode, ExecutorNode executorNode, RawApi rawApi, Map varMap, Auth auth, - AuthMechanism authMechanism, String logId, TestingRunConfig testingRunConfig, List customAuthTypes, Strategy strategy) { + AuthMechanism authMechanism, String logId, TestingRunConfig testingRunConfig, + List customAuthTypes, Strategy strategy) { super(apiInfoKey, filterNode, validatorNode, executorNode ,rawApi, varMap, auth, authMechanism, logId, testingRunConfig, strategy); this.customAuthTypes = customAuthTypes; } @@ -51,6 +48,7 @@ public boolean filter() { return isValid; } + @Override public boolean checkAuthBeforeExecution(boolean debug, List testLogs) { if (this.auth != null && this.auth.getAuthenticated() != null && this.auth.getAuthenticated() == true) { @@ -72,7 +70,8 @@ public boolean checkAuthBeforeExecution(boolean debug, List testLogs) { // loggerMaker.infoAndAddToDb("executor started" + logId, LogDb.TESTING); YamlTestResult results = new Executor().execute(this.executorNode, this.rawApi, this.varMap, this.logId, - this.authMechanism, this.validatorNode, this.apiInfoKey, this.testingRunConfig, this.customAuthTypes, debug, testLogs); + this.authMechanism, this.validatorNode, this.apiInfoKey, this.testingRunConfig, this.customAuthTypes, + debug, testLogs, memory); // loggerMaker.infoAndAddToDb("execution result size " + results.size() + " " + logId, LogDb.TESTING); return results; } diff --git a/libs/dao/src/main/java/com/akto/dao/test_editor/TestConfigYamlParser.java b/libs/dao/src/main/java/com/akto/dao/test_editor/TestConfigYamlParser.java index e719d7e419..2406e56ecb 100644 --- a/libs/dao/src/main/java/com/akto/dao/test_editor/TestConfigYamlParser.java +++ b/libs/dao/src/main/java/com/akto/dao/test_editor/TestConfigYamlParser.java @@ -65,7 +65,7 @@ public static TestConfig parseConfig(Map config) throws Exceptio // todo: should not be null, throw error return new TestConfig(id, info, auth, null, null, null, null, null); } - + ConfigParser configParser = new ConfigParser(); ConfigParserResult filters = configParser.parse(filterMap); if (filters == null) { diff --git a/libs/dao/src/main/java/com/akto/dao/test_editor/TestEditorEnums.java b/libs/dao/src/main/java/com/akto/dao/test_editor/TestEditorEnums.java index 083b00231a..129dbb6f3b 100644 --- a/libs/dao/src/main/java/com/akto/dao/test_editor/TestEditorEnums.java +++ b/libs/dao/src/main/java/com/akto/dao/test_editor/TestEditorEnums.java @@ -198,6 +198,7 @@ public enum RequestParentOperand { } public enum TerminalExecutorDataOperands { + API, DELETE_HEADER, DELETE_BODY_PARAM, DELETE_QUERY_PARAM, diff --git a/libs/dao/src/main/java/com/akto/dto/billing/Organization.java b/libs/dao/src/main/java/com/akto/dto/billing/Organization.java index f355d5d932..4f0d0706f9 100644 --- a/libs/dao/src/main/java/com/akto/dto/billing/Organization.java +++ b/libs/dao/src/main/java/com/akto/dto/billing/Organization.java @@ -32,6 +32,9 @@ public class Organization { public static final String HOTJAR_SITE_ID = "hotjarSiteId"; public String hotjarSiteId = "hotjarSiteId"; + + public static final String TEST_TELEMETRY_ENABLED = "testTelemetryEnabled"; + private boolean testTelemetryEnabled; private int gracePeriod; public Organization() { } @@ -124,4 +127,12 @@ public int getLastFeatureMapUpdate() { public void setLastFeatureMapUpdate(int lastFeatureMapUpdate) { this.lastFeatureMapUpdate = lastFeatureMapUpdate; } + + public boolean getTestTelemetryEnabled() { + return testTelemetryEnabled; + } + + public void setTestTelemetryEnabled(boolean testTelemetryEnabled) { + this.testTelemetryEnabled = testTelemetryEnabled; + } } diff --git a/libs/dao/src/main/java/com/akto/dto/dependency_flow/TreeHelper.java b/libs/dao/src/main/java/com/akto/dto/dependency_flow/TreeHelper.java index 1c094b02a4..bb04a3fbca 100644 --- a/libs/dao/src/main/java/com/akto/dto/dependency_flow/TreeHelper.java +++ b/libs/dao/src/main/java/com/akto/dto/dependency_flow/TreeHelper.java @@ -34,10 +34,13 @@ public void buildTree(String apiCollectionId, String url, String method) { ) ); - if (node == null || node.getMaxDepth() == 0) return; + if (node == null) return; result.put(node.hashCode(), node); + if (node.getMaxDepth() == 0) return; + + Map connections = node.getConnections(); for (Connection connection: connections.values()) { List edges = connection.getEdges(); diff --git a/libs/dao/src/main/java/com/akto/dto/test_editor/TestConfig.java b/libs/dao/src/main/java/com/akto/dto/test_editor/TestConfig.java index 347b742561..09bb3c8d13 100644 --- a/libs/dao/src/main/java/com/akto/dto/test_editor/TestConfig.java +++ b/libs/dao/src/main/java/com/akto/dto/test_editor/TestConfig.java @@ -148,4 +148,7 @@ public void setAuthor(String author) { this.author = author; } + public boolean isInactive() { + return inactive; + } } diff --git a/libs/utils/src/main/java/com/akto/onprem/Constants.java b/libs/utils/src/main/java/com/akto/onprem/Constants.java index 1b581fb795..d249e3060b 100644 --- a/libs/utils/src/main/java/com/akto/onprem/Constants.java +++ b/libs/utils/src/main/java/com/akto/onprem/Constants.java @@ -11,6 +11,8 @@ import java.security.NoSuchAlgorithmException; import java.util.Arrays; import java.util.Base64; +import org.bson.types.ObjectId; +import com.akto.dto.billing.Organization; public class Constants { public static final DashboardMode DEFAULT_DASHBOARD_MODE = DashboardMode.LOCAL_DEPLOY; @@ -88,4 +90,8 @@ public static Config.SendgridConfig getSendgridConfig() { } return sendgridConfig; } + + public static void sendTestResults(ObjectId summaryId, Organization organization) { + + } } diff --git a/libs/utils/src/main/java/com/akto/open_api/parser/parameter_parser/CommonParser.java b/libs/utils/src/main/java/com/akto/open_api/parser/parameter_parser/CommonParser.java index 5cb4a94b92..1e7720a603 100644 --- a/libs/utils/src/main/java/com/akto/open_api/parser/parameter_parser/CommonParser.java +++ b/libs/utils/src/main/java/com/akto/open_api/parser/parameter_parser/CommonParser.java @@ -13,6 +13,8 @@ public static String getExistingExample(Parameter parameter) { ret = parameter.getExamples().get(exampleName).getValue().toString(); break; } + } else if (parameter.getSchema() != null && parameter.getSchema().getExample() != null) { + ret = parameter.getSchema().getExample().toString(); } return ret; } diff --git a/libs/utils/src/main/java/com/akto/testing/ApiExecutor.java b/libs/utils/src/main/java/com/akto/testing/ApiExecutor.java index 65900b0083..c4bf4e0d6c 100644 --- a/libs/utils/src/main/java/com/akto/testing/ApiExecutor.java +++ b/libs/utils/src/main/java/com/akto/testing/ApiExecutor.java @@ -144,6 +144,52 @@ public static String replaceHostFromConfig(String url, TestingRunConfig testingR return url; } + public static String replacePathFromConfig(String url, TestingRunConfig testingRunConfig) { + if (testingRunConfig != null && !StringUtils.isEmpty(testingRunConfig.getOverriddenTestAppUrl())) { + URI typedUrl = null; + try { + typedUrl = new URI(url); + + } catch (URISyntaxException e) { + loggerMaker.errorAndAddToDb(e, "error converting req url to uri " + url, LogDb.TESTING); + throw new RuntimeException(e); + } + + String newUrl = testingRunConfig.getOverriddenTestAppUrl(); + + URI newUri = null; + try { + newUri = new URI(newUrl); + + } catch (URISyntaxException e) { + loggerMaker.errorAndAddToDb(e, "error converting override url to uri " + url, LogDb.TESTING); + throw new RuntimeException(e); + } + + String newPath = newUri.getPath(); + + if (newPath.equals("") || newPath.equals("/")) { + newPath = typedUrl.getPath(); + } + + String newHost = newUri.getHost(); + if (newUri.getHost().equals("")) { + newHost = typedUrl.getHost(); + } + + try { + String newScheme = newUri.getScheme() == null ? typedUrl.getScheme() : newUri.getScheme(); + int newPort = newUri.getPort() == -1 ? typedUrl.getPort() : newUri.getPort(); + + url = new URI(newScheme, null, newHost, newPort, newPath, typedUrl.getQuery(), typedUrl.getFragment()).toString(); + } catch (URISyntaxException e) { + loggerMaker.errorAndAddToDb(e, "error building new url using override url", LogDb.TESTING); + throw new RuntimeException(e); + } + } + return url; + } + public static String prepareUrl(OriginalHttpRequest request, TestingRunConfig testingRunConfig) throws Exception{ String url = request.getUrl(); url = url.trim(); diff --git a/libs/utils/src/main/java/com/akto/utils/GzipUtils.java b/libs/utils/src/main/java/com/akto/utils/GzipUtils.java new file mode 100644 index 0000000000..426e5c53e3 --- /dev/null +++ b/libs/utils/src/main/java/com/akto/utils/GzipUtils.java @@ -0,0 +1,56 @@ +package com.akto.utils; + +import com.mongodb.BasicDBObject; + +import java.io.*; +import java.nio.charset.StandardCharsets; +import java.util.Base64; +import java.util.zip.GZIPInputStream; +import java.util.zip.GZIPOutputStream; + + +public class GzipUtils { + + public static String zipString(String input) { + if (input == null || input.isEmpty()) { + return input; + } + + try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + GZIPOutputStream gzipOutputStream = new GZIPOutputStream(byteArrayOutputStream)) { + gzipOutputStream.write(input.getBytes(StandardCharsets.UTF_8)); + gzipOutputStream.close(); + return Base64.getEncoder().encodeToString(byteArrayOutputStream.toByteArray()); + } catch (IOException e) { + throw new RuntimeException("Failed to zip content", e); + } + } + + + public static String unzipString(String base64String) { + if(base64String == null || base64String.isEmpty()){ + return base64String; + } + String safeString = base64String.replace('-', '+').replace('_', '/'); + safeString = safeString.replaceAll("\\s", ""); + + try { + byte[] decodedBytes = Base64.getDecoder().decode(safeString); + ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(decodedBytes); + GZIPInputStream gzipInputStream = new GZIPInputStream(byteArrayInputStream); + ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + byte[] buffer = new byte[1024]; + int len; + while ((len = gzipInputStream.read(buffer)) > 0) { + byteArrayOutputStream.write(buffer, 0, len); + } + String decompressedString = new String(byteArrayOutputStream.toByteArray(), StandardCharsets.UTF_8); + System.out.println("Decompressed string: " + decompressedString); + return decompressedString; + } catch (Exception e) { + System.err.println("Error decompressing and decoding Base64 string: " + e.getMessage()); + } + return null; + } + +}