diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/test_editor/api.js b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/test_editor/api.js index e6bdab86e6..4dce55eb9c 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/test_editor/api.js +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/test_editor/api.js @@ -12,13 +12,6 @@ const testEditorRequests = { } }) }, - fetchAllSubCategories(mode, skip, limit) { - return request({ - url: 'api/fetchAllSubCategories', - method: 'post', - data: { mode, skip, limit } - }) - }, fetchVulnerableRequests(skip, limit) { return request({ diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/TestRoleAccessMatrix/TestRoleAccessMatrix.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/TestRoleAccessMatrix/TestRoleAccessMatrix.jsx index e4d58cc76e..0eae274604 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/TestRoleAccessMatrix/TestRoleAccessMatrix.jsx +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/TestRoleAccessMatrix/TestRoleAccessMatrix.jsx @@ -9,7 +9,7 @@ import { useState } from 'react'; function TestRoleAccessMatrix() { const location = useLocation() - const name = location?.state?.name || null + const [name, setName] = useState(location?.state?.name || null); const [roleToUrls, setRoleToUrls] = useState([]) diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/transform.js b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/transform.js index 76f92d9344..8bfa2b011c 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/transform.js +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/transform.js @@ -595,6 +595,7 @@ const transform = { }, async getAllSubcategoriesData(fetchActive,type){ let finalDataSubCategories = [], promises = [], categories = []; + let testSourceConfigs = [] const limit = 50; for(var i = 0 ; i < 20; i++){ promises.push( @@ -613,15 +614,22 @@ const transform = { categories.push(...result.value.categories); } } + + if (result?.value?.testSourceConfigs && + result?.value?.testSourceConfigs !== undefined && + result?.value?.testSourceConfigs.length > 0) { + testSourceConfigs = result?.value?.testSourceConfigs + } } } return { categories: categories, - subCategories: finalDataSubCategories + subCategories: finalDataSubCategories, + testSourceConfigs: testSourceConfigs } }, async setTestMetadata() { - const resp = await api.fetchAllSubCategories(false, "Dashboard"); + const resp = await this.getAllSubcategoriesData(false, "Dashboard") let subCategoryMap = {}; resp.subCategories.forEach((x) => { func.trimContentFromSubCategory(x) diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/vulnerability_report/Issue.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/vulnerability_report/Issue.jsx index 6d24acea82..e3e2e4280b 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/vulnerability_report/Issue.jsx +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/vulnerability_report/Issue.jsx @@ -19,7 +19,7 @@ function Issue({ index, issueDetails, isLast }) { const testRunResultSummaryHexId = firstVulnerableApi.testRunHexId const testRunId = firstVulnerableApi.hexId - return `/dashboard/testing/${testRunResultSummaryHexId}/result/${testRunId}` + return `/dashboard/testing/${testRunResultSummaryHexId}?result=${testRunId}` } const testResultLink = getTestResultLink() diff --git a/apps/testing/src/main/java/com/akto/rules/BFLATest.java b/apps/testing/src/main/java/com/akto/rules/BFLATest.java index b93daf996a..03d3c56226 100644 --- a/apps/testing/src/main/java/com/akto/rules/BFLATest.java +++ b/apps/testing/src/main/java/com/akto/rules/BFLATest.java @@ -6,7 +6,6 @@ import com.akto.dto.OriginalHttpResponse; import com.akto.dto.RawApi; import com.akto.dto.testing.*; -import com.akto.dto.testing.info.BFLATestInfo; import com.akto.dto.testing.sources.AuthWithCond; import com.akto.log.LoggerMaker.LogDb; import com.akto.store.TestingUtil; @@ -21,7 +20,6 @@ import org.bson.conversions.Bson; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.Map; @@ -74,6 +72,9 @@ public List updateAllowedRoles(RawApi rawApi, ApiInfo.ApiInfoKey apiInfo if (allHeadersMatched) { AuthMechanism authMechanismForRole = authWithCond.getAuthMechanism(); if (authMechanismForRole.getType().equalsIgnoreCase(LoginFlowEnums.AuthMechanismTypes.LOGIN_REQUEST.name())) { + if (authWithCond.getRecordedLoginFlowInput() != null) { + authMechanismForRole.setRecordedLoginFlowInput(authWithCond.getRecordedLoginFlowInput()); + } LoginFlowResponse loginFlowResponse = TestExecutor.executeLoginFlow(authMechanismForRole, null); if (!loginFlowResponse.getSuccess()) throw new Exception(loginFlowResponse.getError()); diff --git a/apps/testing/src/main/java/com/akto/rules/TestPlugin.java b/apps/testing/src/main/java/com/akto/rules/TestPlugin.java index 3df12d3df3..e4aa1acf6f 100644 --- a/apps/testing/src/main/java/com/akto/rules/TestPlugin.java +++ b/apps/testing/src/main/java/com/akto/rules/TestPlugin.java @@ -41,7 +41,7 @@ public abstract class TestPlugin { static ObjectMapper mapper = new ObjectMapper(); static JsonFactory factory = mapper.getFactory(); - static final LoggerMaker loggerMaker = new LoggerMaker(TestPlugin.class); + static final LoggerMaker loggerMaker = new LoggerMaker(TestPlugin.class, LogDb.TESTING); private static final Logger logger = LoggerFactory.getLogger(TestPlugin.class); private static final Gson gson = new Gson(); @@ -52,7 +52,8 @@ public abstract class TestPlugin { public abstract String subTestName(); public static boolean isStatusGood(int statusCode) { - return statusCode >= 200 && statusCode<300; + // TODO: 250 status code is for a client. To be verified later. + return statusCode >= 200 && statusCode < 300 && statusCode != 250; } public static void extractAllValuesFromPayload(String payload, Map> payloadMap) throws Exception{ diff --git a/apps/testing/src/main/java/com/akto/test_editor/execution/Executor.java b/apps/testing/src/main/java/com/akto/test_editor/execution/Executor.java index db57bd6150..9dd96167cb 100644 --- a/apps/testing/src/main/java/com/akto/test_editor/execution/Executor.java +++ b/apps/testing/src/main/java/com/akto/test_editor/execution/Executor.java @@ -2,6 +2,7 @@ import com.akto.dao.billing.OrganizationsDao; import java.util.*; +import java.util.concurrent.ConcurrentHashMap; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -27,6 +28,7 @@ import com.akto.rules.TestPlugin; import com.akto.test_editor.Utils; import com.akto.util.Constants; +import com.akto.util.JSONUtils; import com.akto.util.modifier.JWTPayloadReplacer; import com.fasterxml.jackson.databind.ObjectMapper; @@ -48,7 +50,7 @@ public class Executor { - private static final LoggerMaker loggerMaker = new LoggerMaker(Executor.class); + private static final LoggerMaker loggerMaker = new LoggerMaker(Executor.class, LogDb.TESTING); public final String _HOST = "host"; @@ -504,15 +506,23 @@ private ExecutorSingleOperationResp modifyAuthTokenInRawApi(TestRoles testRole, if (authWithCond.getRecordedLoginFlowInput() != null) { // handle json recording RecordedLoginFlowInput recordedLoginFlowInput = authWithCond.getRecordedLoginFlowInput(); + Map valuesMap = new HashMap<>(); String token = com.akto.testing.workflow_node_executor.Utils.fetchToken(recordedLoginFlowInput, 5); if (token == null) { return new ExecutorSingleOperationResp(false, "Failed to replace roles_access_context: "); + } else { + loggerMaker.infoAndAddToDb("flattened here: " + token); + BasicDBObject flattened = JSONUtils.flattenWithDots(BasicDBObject.parse(token)); + + for (String param: flattened.keySet()) { + String key = "x1.response.body." + param; + valuesMap.put(key, flattened.get(param)); + loggerMaker.infoAndAddToDb("kv pair: " + key + " " + flattened.get(param)); + } + } - Map valuesMap = new HashMap<>(); - valuesMap.put("x1.response.body.token", token); - for (AuthParam param : authMechanismForRole.getAuthParams()) { try { String value = com.akto.testing.workflow_node_executor.Utils.executeCode(param.getValue(), valuesMap); @@ -526,6 +536,12 @@ private ExecutorSingleOperationResp modifyAuthTokenInRawApi(TestRoles testRole, } authMechanismForRole.setType(LoginFlowEnums.AuthMechanismTypes.HARDCODED.name()); + /* + * We set the recorded login flow null + * so that we calculate the tokens only once + * and use them every time. + */ + authWithCond.setRecordedLoginFlowInput(null); } else { if (AuthMechanismTypes.LOGIN_REQUEST.toString().equalsIgnoreCase(authMechanismForRole.getType())) { try { @@ -548,7 +564,12 @@ private ExecutorSingleOperationResp modifyAuthTokenInRawApi(TestRoles testRole, if (!authParamList.isEmpty()) { ExecutorSingleOperationResp ret = null; for (AuthParam authParam1: authParamList) { - ret = Operations.modifyHeader(rawApi, authParam1.getKey().toLowerCase(), authParam1.getValue()); + if(authParam1.authTokenPresent(rawApi.getRequest())){ + authParam1.addAuthTokens(rawApi.getRequest()); + ret = new ExecutorSingleOperationResp(true, ""); + } else { + ret = new ExecutorSingleOperationResp(true, "key not present " + authParam1.getKey().toLowerCase()); + } } return ret; @@ -559,6 +580,26 @@ private ExecutorSingleOperationResp modifyAuthTokenInRawApi(TestRoles testRole, return null; } + private static ConcurrentHashMap roleCache = new ConcurrentHashMap<>(); + + public static void clearRoleCache() { + if (roleCache != null) { + roleCache.clear(); + } + } + + private synchronized static TestRoles fetchOrFindTestRole(String name) { + if (roleCache == null) { + roleCache = new ConcurrentHashMap<>(); + } + if (roleCache.containsKey(name)) { + return roleCache.get(name); + } + TestRoles testRole = TestRolesDao.instance.findOne(TestRoles.NAME, name); + roleCache.put(name, testRole); + return roleCache.get(name); + } + public ExecutorSingleOperationResp runOperation(String operationType, RawApi rawApi, Object key, Object value, Map varMap, AuthMechanism authMechanism, List customAuthTypes, ApiInfo.ApiInfoKey apiInfoKey) { switch (operationType.toLowerCase()) { case "send_ssrf_req": @@ -590,12 +631,14 @@ public ExecutorSingleOperationResp runOperation(String operationType, RawApi raw keyStr = keyStr.replace(ACCESS_ROLES_CONTEXT, ""); keyStr = keyStr.substring(0,keyStr.length()-1).trim(); - TestRoles testRole = TestRolesDao.instance.findOne(TestRoles.NAME, keyStr); + TestRoles testRole = fetchOrFindTestRole(keyStr); if (testRole == null) { return new ExecutorSingleOperationResp(false, "Test Role " + keyStr + " Doesn't Exist "); } - - ExecutorSingleOperationResp insertedAuthResp = modifyAuthTokenInRawApi(testRole, rawApi); + ExecutorSingleOperationResp insertedAuthResp = new ExecutorSingleOperationResp(true, ""); + synchronized (testRole) { + insertedAuthResp = modifyAuthTokenInRawApi(testRole, rawApi); + } if (insertedAuthResp != null) { return insertedAuthResp; } @@ -682,7 +725,9 @@ public ExecutorSingleOperationResp runOperation(String operationType, RawApi raw for (AuthParam authParam: authMechanism.getAuthParams()) { authVal = authParam.getValue(); - ExecutorSingleOperationResp result = Operations.modifyHeader(rawApi, authParam.getKey(), authVal, true); + ExecutorSingleOperationResp result = new ExecutorSingleOperationResp(true, ""); + boolean ret = authParam.addAuthTokens(rawApi.getRequest()); + result.setSuccess(ret); modifiedAtLeastOne = modifiedAtLeastOne || result.getSuccess(); } } diff --git a/apps/testing/src/main/java/com/akto/testing/Main.java b/apps/testing/src/main/java/com/akto/testing/Main.java index 9b1150ca47..c754e94ca8 100644 --- a/apps/testing/src/main/java/com/akto/testing/Main.java +++ b/apps/testing/src/main/java/com/akto/testing/Main.java @@ -35,6 +35,7 @@ import com.akto.notifications.slack.SlackSender; import com.akto.rules.RequiredConfigs; import com.akto.task.Cluster; +import com.akto.test_editor.execution.Executor; import com.akto.util.AccountTask; import com.akto.util.Constants; import com.akto.util.DashboardMode; @@ -319,6 +320,12 @@ public void run() { // saving the initial usageLeft, to calc delta later. int usageLeft = syncLimit.getUsageLeft(); + /* + * Since the role cache is static + * so to prevent it from being shared across accounts. + */ + Executor.clearRoleCache(); + try { fillTestingEndpoints(testingRun); // continuous testing condition diff --git a/apps/testing/src/main/java/com/akto/testing/TestExecutor.java b/apps/testing/src/main/java/com/akto/testing/TestExecutor.java index 9a09ba7978..5ea6f719f5 100644 --- a/apps/testing/src/main/java/com/akto/testing/TestExecutor.java +++ b/apps/testing/src/main/java/com/akto/testing/TestExecutor.java @@ -402,12 +402,15 @@ public static WorkflowTest convertToWorkflowGraph(ArrayList request int waitTime = 0; WorkflowNodeDetails.Type nodeType = WorkflowNodeDetails.Type.API; - if (data.getType().equals(LoginFlowEnums.LoginStepTypesEnums.OTP_VERIFICATION.toString()) || data.getUrl().contains("fetchOtpData")) { + if ((data.getType() != null + && data.getType().equals(LoginFlowEnums.LoginStepTypesEnums.OTP_VERIFICATION.toString())) + || (data.getUrl() != null && data.getUrl().contains("fetchOtpData"))) { nodeType = WorkflowNodeDetails.Type.OTP; waitTime = 20; data.setOtpRefUuid(data.getUrl().substring(data.getUrl().lastIndexOf('/') + 1)); } - if (data.getType().equals(LoginFlowEnums.LoginStepTypesEnums.RECORDED_FLOW.toString())) { + if (data.getType() != null + && data.getType().equals(LoginFlowEnums.LoginStepTypesEnums.RECORDED_FLOW.toString())) { nodeType = WorkflowNodeDetails.Type.RECORDED; } WorkflowNodeDetails workflowNodeDetails = new WorkflowNodeDetails(0, data.getUrl(), diff --git a/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/Utils.java b/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/Utils.java index d32b3abffc..245dceffd9 100644 --- a/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/Utils.java +++ b/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/Utils.java @@ -41,7 +41,7 @@ public class Utils { - private static final LoggerMaker loggerMaker = new LoggerMaker(Utils.class); + private static final LoggerMaker loggerMaker = new LoggerMaker(Utils.class, LogDb.TESTING); private static final Gson gson = new Gson(); public static WorkflowTestResult.NodeResult processOtpNode(Node node, Map valuesMap) { @@ -128,15 +128,13 @@ private static String extractOtpCode(String text, String regex) { return verificationCode; } - public static WorkflowTestResult.NodeResult processRecorderNode(Node node, Map valuesMap) { + public static WorkflowTestResult.NodeResult processRecorderNode(Node node, Map valuesMap, RecordedLoginFlowInput recordedLoginFlowInput) { List testErrors = new ArrayList<>(); BasicDBObject resp = new BasicDBObject(); BasicDBObject body = new BasicDBObject(); BasicDBObject data = new BasicDBObject(); String message; - - RecordedLoginFlowInput recordedLoginFlowInput = RecordedLoginInputDao.instance.findOne(new BasicDBObject()); String token = fetchToken(recordedLoginFlowInput, 5); @@ -149,8 +147,17 @@ public static WorkflowTestResult.NodeResult processRecorderNode(Node node, Map valuesMap, Boolean allowAllStatusCodes, boolean debug, List testLogs, Memory memory) { + RecordedLoginFlowInput recordedLoginFlowInput = RecordedLoginInputDao.instance.findOne(new BasicDBObject()); + return processNode(node, valuesMap, allowAllStatusCodes, debug, testLogs, memory, recordedLoginFlowInput); + } + + public static WorkflowTestResult.NodeResult processNode(Node node, Map valuesMap, Boolean allowAllStatusCodes, boolean debug, List testLogs, Memory memory, AuthMechanism authMechanism) { + return processNode(node, valuesMap, allowAllStatusCodes, debug, testLogs, memory, authMechanism.getRecordedLoginFlowInput()); + } + + public static WorkflowTestResult.NodeResult processNode(Node node, Map valuesMap, Boolean allowAllStatusCodes, boolean debug, List testLogs, Memory memory, RecordedLoginFlowInput recordedLoginFlowInput) { if (node.getWorkflowNodeDetails().getType() == WorkflowNodeDetails.Type.RECORDED) { - return processRecorderNode(node, valuesMap); + return processRecorderNode(node, valuesMap, recordedLoginFlowInput); } else if (node.getWorkflowNodeDetails().getType() == WorkflowNodeDetails.Type.OTP) { return processOtpNode(node, valuesMap); @@ -201,7 +217,6 @@ else if (node.getWorkflowNodeDetails().getType() == WorkflowNodeDetails.Type.OTP } } - public static WorkflowTestResult.NodeResult processApiNode(Node node, Map valuesMap, Boolean allowAllStatusCodes, boolean debug, List testLogs, Memory memory) { NodeExecutorFactory nodeExecutorFactory = new NodeExecutorFactory(); @@ -246,7 +261,7 @@ public static LoginFlowResponse runLoginFlow(WorkflowTest workflowTest, AuthMech if (authMechanism.getRequestData() != null && authMechanism.getRequestData().size() > 0 && authMechanism.getRequestData().get(index).getAllowAllStatusCodes()) { allowAllStatusCodes = authMechanism.getRequestData().get(0).getAllowAllStatusCodes(); } - nodeResult = processNode(node, valuesMap, allowAllStatusCodes, false, new ArrayList<>(), null); + nodeResult = processNode(node, valuesMap, allowAllStatusCodes, false, new ArrayList<>(), null, authMechanism); } catch (Exception e) { ; List testErrors = new ArrayList<>(); diff --git a/libs/dao/src/main/java/com/akto/dao/testing/TestRolesDao.java b/libs/dao/src/main/java/com/akto/dao/testing/TestRolesDao.java index 30591eded3..93d21bf919 100644 --- a/libs/dao/src/main/java/com/akto/dao/testing/TestRolesDao.java +++ b/libs/dao/src/main/java/com/akto/dao/testing/TestRolesDao.java @@ -66,7 +66,11 @@ public AuthMechanism fetchAttackerToken(int apiCollectionId, RawApi rawApi) { TestRoles testRoles = TestRolesDao.instance.findOne(TestRoles.NAME, "ATTACKER_TOKEN_ALL"); if (testRoles != null && testRoles.getAuthWithCondList().size() > 0) { List authWithCondList = testRoles.getAuthWithCondList(); - AuthMechanism defaultAuthMechanism = authWithCondList.get(0).getAuthMechanism(); + AuthWithCond firstAuth = authWithCondList.get(0); + AuthMechanism defaultAuthMechanism = firstAuth.getAuthMechanism(); + if(firstAuth.getRecordedLoginFlowInput()!=null){ + defaultAuthMechanism.setRecordedLoginFlowInput(firstAuth.getRecordedLoginFlowInput()); + } if (rawApi == null) { return defaultAuthMechanism; } else { @@ -88,7 +92,11 @@ public AuthMechanism fetchAttackerToken(int apiCollectionId, RawApi rawApi) { } if (allHeadersMatched) { - return authWithCond.getAuthMechanism(); + defaultAuthMechanism = authWithCond.getAuthMechanism(); + if(authWithCond.getRecordedLoginFlowInput()!=null){ + defaultAuthMechanism.setRecordedLoginFlowInput(authWithCond.getRecordedLoginFlowInput()); + } + return defaultAuthMechanism; } } } catch (Exception e) { diff --git a/libs/dao/src/main/java/com/akto/dto/testing/AuthMechanism.java b/libs/dao/src/main/java/com/akto/dto/testing/AuthMechanism.java index d3d6ed867b..0d3013aeb9 100644 --- a/libs/dao/src/main/java/com/akto/dto/testing/AuthMechanism.java +++ b/libs/dao/src/main/java/com/akto/dto/testing/AuthMechanism.java @@ -1,7 +1,9 @@ package com.akto.dto.testing; -import com.akto.dto.HttpRequestParams; import com.akto.dto.OriginalHttpRequest; +import com.akto.dto.RecordedLoginFlowInput; + +import org.bson.codecs.pojo.annotations.BsonIgnore; import org.bson.types.ObjectId; import java.util.ArrayList; @@ -19,6 +21,10 @@ public class AuthMechanism { private String uuid; private List apiCollectionIds; + + @BsonIgnore + private RecordedLoginFlowInput recordedLoginFlowInput; + public AuthMechanism() { } @@ -107,4 +113,12 @@ public List getApiCollectionIds() { public void setApiCollectionIds(List apiCollectionIds) { this.apiCollectionIds = apiCollectionIds; } + + public RecordedLoginFlowInput getRecordedLoginFlowInput() { + return recordedLoginFlowInput; + } + + public void setRecordedLoginFlowInput(RecordedLoginFlowInput recordedLoginFlowInput) { + this.recordedLoginFlowInput = recordedLoginFlowInput; + } } diff --git a/libs/dao/src/main/java/com/akto/util/RecordedLoginFlowUtil.java b/libs/dao/src/main/java/com/akto/util/RecordedLoginFlowUtil.java index 4358e213f3..b5b0c0943e 100644 --- a/libs/dao/src/main/java/com/akto/util/RecordedLoginFlowUtil.java +++ b/libs/dao/src/main/java/com/akto/util/RecordedLoginFlowUtil.java @@ -46,7 +46,11 @@ public static void triggerFlow(String tokenFetchCommand, String payload, String TimeoutObject timeoutObj = new TimeoutObject(300, 300, 300); JsonNode node = ApiRequest.postRequestWithTimeout(new HashMap<>(), url, reqData, timeoutObj); - String token = node.get("token").textValue(); + logger.info("getting token...: " + outputFilePath); + + String token = node.toString(); //node.get("token").textValue(); + + logger.info("token: " + token); FileUtils.writeStringToFile(new File(outputFilePath), token, (String) null);