Merge pull request #1696 from akto-api-security/feature/automated_aut…

…h_fix

Feature/automated auth fix

apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/test_editor/api.js

@@ -12,13 +12,6 @@ const testEditorRequests = {
             }
         })
     },
-    fetchAllSubCategories(mode, skip, limit) {
-        return request({
-            url: 'api/fetchAllSubCategories',
-            method: 'post',
-            data: { mode, skip, limit }
-        })
-    },
 
     fetchVulnerableRequests(skip, limit) {
         return request({

apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/TestRoleAccessMatrix/TestRoleAccessMatrix.jsx

@@ -9,7 +9,7 @@ import { useState } from 'react';
 
 function TestRoleAccessMatrix() {
     const location = useLocation()
-    const name = location?.state?.name || null
+    const [name, setName] = useState(location?.state?.name || null);
 
     const [roleToUrls, setRoleToUrls] = useState([])
 

apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/transform.js

@@ -595,6 +595,7 @@ const transform = {
   },
   async getAllSubcategoriesData(fetchActive,type){
     let finalDataSubCategories = [], promises = [], categories = [];
+    let testSourceConfigs = []
     const limit = 50;
     for(var i = 0 ; i < 20; i++){
       promises.push(
@@ -613,15 +614,22 @@ const transform = {
             categories.push(...result.value.categories);
           }
         }
+
+        if (result?.value?.testSourceConfigs &&
+          result?.value?.testSourceConfigs !== undefined &&
+          result?.value?.testSourceConfigs.length > 0) {
+          testSourceConfigs = result?.value?.testSourceConfigs
+        }
       }
     }
     return {
       categories: categories,
-      subCategories: finalDataSubCategories
+      subCategories: finalDataSubCategories,
+      testSourceConfigs: testSourceConfigs
     }
   },
   async setTestMetadata() {
-    const resp = await api.fetchAllSubCategories(false, "Dashboard");
+    const resp = await this.getAllSubcategoriesData(false, "Dashboard")
     let subCategoryMap = {};
     resp.subCategories.forEach((x) => {
       func.trimContentFromSubCategory(x)

apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/vulnerability_report/Issue.jsx

@@ -19,7 +19,7 @@ function Issue({ index, issueDetails, isLast }) {
         const testRunResultSummaryHexId = firstVulnerableApi.testRunHexId
         const testRunId = firstVulnerableApi.hexId  
 
-        return `/dashboard/testing/${testRunResultSummaryHexId}/result/${testRunId}`
+        return `/dashboard/testing/${testRunResultSummaryHexId}?result=${testRunId}`
     }
     const testResultLink = getTestResultLink()
 

apps/testing/src/main/java/com/akto/rules/BFLATest.java

@@ -6,7 +6,6 @@
 import com.akto.dto.OriginalHttpResponse;
 import com.akto.dto.RawApi;
 import com.akto.dto.testing.*;
-import com.akto.dto.testing.info.BFLATestInfo;
 import com.akto.dto.testing.sources.AuthWithCond;
 import com.akto.log.LoggerMaker.LogDb;
 import com.akto.store.TestingUtil;
@@ -21,7 +20,6 @@
 import org.bson.conversions.Bson;
 
 import java.util.ArrayList;
-import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 
@@ -74,6 +72,9 @@ public List<String> updateAllowedRoles(RawApi rawApi, ApiInfo.ApiInfoKey apiInfo
                 if (allHeadersMatched) {
                     AuthMechanism authMechanismForRole = authWithCond.getAuthMechanism();
                     if (authMechanismForRole.getType().equalsIgnoreCase(LoginFlowEnums.AuthMechanismTypes.LOGIN_REQUEST.name())) {
+                        if (authWithCond.getRecordedLoginFlowInput() != null) {
+                            authMechanismForRole.setRecordedLoginFlowInput(authWithCond.getRecordedLoginFlowInput());
+                        }
                         LoginFlowResponse loginFlowResponse = TestExecutor.executeLoginFlow(authMechanismForRole, null);
                         if (!loginFlowResponse.getSuccess()) throw new Exception(loginFlowResponse.getError());
 

apps/testing/src/main/java/com/akto/rules/TestPlugin.java

@@ -41,7 +41,7 @@
 public abstract class TestPlugin {
     static ObjectMapper mapper = new ObjectMapper();
     static JsonFactory factory = mapper.getFactory();
-    static final LoggerMaker loggerMaker = new LoggerMaker(TestPlugin.class);
+    static final LoggerMaker loggerMaker = new LoggerMaker(TestPlugin.class, LogDb.TESTING);
 
     private static final Logger logger = LoggerFactory.getLogger(TestPlugin.class);
     private static final Gson gson = new Gson();
@@ -52,7 +52,8 @@ public abstract class TestPlugin {
     public abstract String subTestName();
 
     public static boolean isStatusGood(int statusCode) {
-        return statusCode >= 200 && statusCode<300;
+        // TODO: 250 status code is for a client. To be verified later.
+        return statusCode >= 200 && statusCode < 300 && statusCode != 250;
     }
 
     public static void extractAllValuesFromPayload(String payload, Map<String,Set<String>> payloadMap) throws Exception{

apps/testing/src/main/java/com/akto/test_editor/execution/Executor.java

@@ -2,6 +2,7 @@
 
 import com.akto.dao.billing.OrganizationsDao;
 import java.util.*;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
@@ -27,6 +28,7 @@
 import com.akto.rules.TestPlugin;
 import com.akto.test_editor.Utils;
 import com.akto.util.Constants;
+import com.akto.util.JSONUtils;
 import com.akto.util.modifier.JWTPayloadReplacer;
 import com.fasterxml.jackson.databind.ObjectMapper;
 
@@ -48,7 +50,7 @@
 
 public class Executor {
 
-    private static final LoggerMaker loggerMaker = new LoggerMaker(Executor.class);
+    private static final LoggerMaker loggerMaker = new LoggerMaker(Executor.class, LogDb.TESTING);
 
     public final String _HOST = "host";
 
@@ -504,15 +506,23 @@ private ExecutorSingleOperationResp modifyAuthTokenInRawApi(TestRoles testRole,
                 if (authWithCond.getRecordedLoginFlowInput() != null) {
                     // handle json recording
                     RecordedLoginFlowInput recordedLoginFlowInput = authWithCond.getRecordedLoginFlowInput();
+                    Map<String, Object> valuesMap = new HashMap<>();
 
                     String token = com.akto.testing.workflow_node_executor.Utils.fetchToken(recordedLoginFlowInput, 5);
                     if (token == null) {
                         return new ExecutorSingleOperationResp(false, "Failed to replace roles_access_context: ");
+                    } else {
+                        loggerMaker.infoAndAddToDb("flattened here: " + token);
+                        BasicDBObject flattened = JSONUtils.flattenWithDots(BasicDBObject.parse(token));
+
+                        for (String param: flattened.keySet()) {
+                            String key = "x1.response.body." + param;
+                            valuesMap.put(key, flattened.get(param));
+                            loggerMaker.infoAndAddToDb("kv pair: " + key + " " + flattened.get(param));
+                        }	
+
                     }
 
-                    Map<String, Object> valuesMap = new HashMap<>();
-                    valuesMap.put("x1.response.body.token", token);
-
                     for (AuthParam param : authMechanismForRole.getAuthParams()) {
                         try {
                             String value = com.akto.testing.workflow_node_executor.Utils.executeCode(param.getValue(), valuesMap);
@@ -526,6 +536,12 @@ private ExecutorSingleOperationResp modifyAuthTokenInRawApi(TestRoles testRole,
                     }
 
                     authMechanismForRole.setType(LoginFlowEnums.AuthMechanismTypes.HARDCODED.name());
+                    /*
+                     * We set the recorded login flow null
+                     * so that we calculate the tokens only once
+                     * and use them every time.
+                     */
+                    authWithCond.setRecordedLoginFlowInput(null);
                 } else {
                     if (AuthMechanismTypes.LOGIN_REQUEST.toString().equalsIgnoreCase(authMechanismForRole.getType())) {
                         try {
@@ -548,7 +564,12 @@ private ExecutorSingleOperationResp modifyAuthTokenInRawApi(TestRoles testRole,
                 if (!authParamList.isEmpty()) {
                     ExecutorSingleOperationResp ret = null;
                     for (AuthParam authParam1: authParamList) {
-                        ret = Operations.modifyHeader(rawApi, authParam1.getKey().toLowerCase(), authParam1.getValue());
+                        if(authParam1.authTokenPresent(rawApi.getRequest())){
+                            authParam1.addAuthTokens(rawApi.getRequest());
+                            ret = new ExecutorSingleOperationResp(true, "");
+                        } else {
+                            ret = new ExecutorSingleOperationResp(true, "key not present " + authParam1.getKey().toLowerCase());
+                        }
                     }
 
                     return ret;
@@ -559,6 +580,26 @@ private ExecutorSingleOperationResp modifyAuthTokenInRawApi(TestRoles testRole,
         return null;
     }
 
+    private static ConcurrentHashMap<String, TestRoles> roleCache = new ConcurrentHashMap<>();
+
+    public static void clearRoleCache() {
+        if (roleCache != null) {
+            roleCache.clear();
+        }
+    }
+
+    private synchronized static TestRoles fetchOrFindTestRole(String name) {
+        if (roleCache == null) {
+            roleCache = new ConcurrentHashMap<>();
+        }
+        if (roleCache.containsKey(name)) {
+            return roleCache.get(name);
+        }
+        TestRoles testRole = TestRolesDao.instance.findOne(TestRoles.NAME, name);
+        roleCache.put(name, testRole);
+        return roleCache.get(name);
+    }
+
     public ExecutorSingleOperationResp runOperation(String operationType, RawApi rawApi, Object key, Object value, Map<String, Object> varMap, AuthMechanism authMechanism, List<CustomAuthType> customAuthTypes, ApiInfo.ApiInfoKey apiInfoKey) {
         switch (operationType.toLowerCase()) {
             case "send_ssrf_req":
@@ -590,12 +631,14 @@ public ExecutorSingleOperationResp runOperation(String operationType, RawApi raw
 
                     keyStr = keyStr.replace(ACCESS_ROLES_CONTEXT, "");
                     keyStr = keyStr.substring(0,keyStr.length()-1).trim();
-                    TestRoles testRole = TestRolesDao.instance.findOne(TestRoles.NAME, keyStr);
+                    TestRoles testRole = fetchOrFindTestRole(keyStr);
                     if (testRole == null) {
                         return new ExecutorSingleOperationResp(false, "Test Role " + keyStr +  " Doesn't Exist ");
                     }
-
-                    ExecutorSingleOperationResp insertedAuthResp = modifyAuthTokenInRawApi(testRole, rawApi);
+                    ExecutorSingleOperationResp insertedAuthResp = new ExecutorSingleOperationResp(true, "");
+                    synchronized (testRole) {
+                        insertedAuthResp = modifyAuthTokenInRawApi(testRole, rawApi);
+                    }
                     if (insertedAuthResp != null) {
                         return insertedAuthResp;
                     }
@@ -682,7 +725,9 @@ public ExecutorSingleOperationResp runOperation(String operationType, RawApi raw
 
                     for (AuthParam authParam: authMechanism.getAuthParams()) {
                         authVal = authParam.getValue();
-                        ExecutorSingleOperationResp result = Operations.modifyHeader(rawApi, authParam.getKey(), authVal, true);
+                        ExecutorSingleOperationResp result = new ExecutorSingleOperationResp(true, "");
+                        boolean ret = authParam.addAuthTokens(rawApi.getRequest());
+                        result.setSuccess(ret);
                         modifiedAtLeastOne = modifiedAtLeastOne || result.getSuccess();
                     }
                 }

apps/testing/src/main/java/com/akto/testing/Main.java

@@ -35,6 +35,7 @@
 import com.akto.notifications.slack.SlackSender;
 import com.akto.rules.RequiredConfigs;
 import com.akto.task.Cluster;
+import com.akto.test_editor.execution.Executor;
 import com.akto.util.AccountTask;
 import com.akto.util.Constants;
 import com.akto.util.DashboardMode;
@@ -319,6 +320,12 @@ public void run() {
                 // saving the initial usageLeft, to calc delta later.
                 int usageLeft = syncLimit.getUsageLeft();
 
+                /*
+                 * Since the role cache is static
+                 * so to prevent it from being shared across accounts.
+                 */
+                Executor.clearRoleCache();
+
                 try {
                     fillTestingEndpoints(testingRun);
                     // continuous testing condition

apps/testing/src/main/java/com/akto/testing/TestExecutor.java

@@ -402,12 +402,15 @@ public static WorkflowTest convertToWorkflowGraph(ArrayList<RequestData> request
 
             int waitTime = 0;
             WorkflowNodeDetails.Type nodeType = WorkflowNodeDetails.Type.API;
-            if (data.getType().equals(LoginFlowEnums.LoginStepTypesEnums.OTP_VERIFICATION.toString()) || data.getUrl().contains("fetchOtpData")) {
+            if ((data.getType() != null
+                    && data.getType().equals(LoginFlowEnums.LoginStepTypesEnums.OTP_VERIFICATION.toString()))
+                    || (data.getUrl() != null && data.getUrl().contains("fetchOtpData"))) {
                 nodeType = WorkflowNodeDetails.Type.OTP;
                 waitTime = 20;
                 data.setOtpRefUuid(data.getUrl().substring(data.getUrl().lastIndexOf('/') + 1));
             }
-            if (data.getType().equals(LoginFlowEnums.LoginStepTypesEnums.RECORDED_FLOW.toString())) {
+            if (data.getType() != null
+                    && data.getType().equals(LoginFlowEnums.LoginStepTypesEnums.RECORDED_FLOW.toString())) {
                 nodeType = WorkflowNodeDetails.Type.RECORDED;
             }
             WorkflowNodeDetails workflowNodeDetails = new WorkflowNodeDetails(0, data.getUrl(),

apps/testing/src/main/java/com/akto/testing/workflow_node_executor/Utils.java

@@ -41,7 +41,7 @@
 
 public class Utils {
 
-    private static final LoggerMaker loggerMaker = new LoggerMaker(Utils.class);
+    private static final LoggerMaker loggerMaker = new LoggerMaker(Utils.class, LogDb.TESTING);
     private static final Gson gson = new Gson();
 
     public static WorkflowTestResult.NodeResult processOtpNode(Node node, Map<String, Object> valuesMap) {
@@ -128,15 +128,13 @@ private static String extractOtpCode(String text, String regex) {
         return verificationCode;
     }
 
-    public static WorkflowTestResult.NodeResult processRecorderNode(Node node, Map<String, Object> valuesMap) {
+    public static WorkflowTestResult.NodeResult processRecorderNode(Node node, Map<String, Object> valuesMap, RecordedLoginFlowInput recordedLoginFlowInput) {
 
         List<String> testErrors = new ArrayList<>();
         BasicDBObject resp = new BasicDBObject();
         BasicDBObject body = new BasicDBObject();
         BasicDBObject data = new BasicDBObject();
         String message;
-
-        RecordedLoginFlowInput recordedLoginFlowInput = RecordedLoginInputDao.instance.findOne(new BasicDBObject());
 
         String token = fetchToken(recordedLoginFlowInput, 5);
 
@@ -149,8 +147,17 @@ public static WorkflowTestResult.NodeResult processRecorderNode(Node node, Map<S
             return new WorkflowTestResult.NodeResult(resp.toString(), false, testErrors);
         }
 
-        valuesMap.put(node.getId() + ".response.body.token", token);
+        // valuesMap.put(node.getId() + ".response.body.token", token);
 
+
+        BasicDBObject flattened = JSONUtils.flattenWithDots(BasicDBObject.parse(token));
+
+        for (String param: flattened.keySet()) {
+            String key = node.getId() + ".response.body" + "." + param;
+            valuesMap.put(key, flattened.get(param));
+	        loggerMaker.infoAndAddToDb("kv pair: " + key + " " + flattened.get(param));
+        }	
+
         data.put("token", token);
         body.put("body", data);
         resp.put("response", body);
@@ -190,8 +197,17 @@ public static String fetchToken(RecordedLoginFlowInput recordedLoginFlowInput, i
     }
 
     public static WorkflowTestResult.NodeResult processNode(Node node, Map<String, Object> valuesMap, Boolean allowAllStatusCodes, boolean debug, List<TestingRunResult.TestLog> testLogs, Memory memory) {
+        RecordedLoginFlowInput recordedLoginFlowInput = RecordedLoginInputDao.instance.findOne(new BasicDBObject());
+        return processNode(node, valuesMap, allowAllStatusCodes, debug, testLogs, memory, recordedLoginFlowInput);
+    }
+
+    public static WorkflowTestResult.NodeResult processNode(Node node, Map<String, Object> valuesMap, Boolean allowAllStatusCodes, boolean debug, List<TestingRunResult.TestLog> testLogs, Memory memory, AuthMechanism authMechanism) {
+        return processNode(node, valuesMap, allowAllStatusCodes, debug, testLogs, memory, authMechanism.getRecordedLoginFlowInput());
+    }
+
+    public static WorkflowTestResult.NodeResult processNode(Node node, Map<String, Object> valuesMap, Boolean allowAllStatusCodes, boolean debug, List<TestingRunResult.TestLog> testLogs, Memory memory, RecordedLoginFlowInput recordedLoginFlowInput) {
         if (node.getWorkflowNodeDetails().getType() == WorkflowNodeDetails.Type.RECORDED) {
-            return processRecorderNode(node, valuesMap);
+            return processRecorderNode(node, valuesMap, recordedLoginFlowInput);
         }
         else if (node.getWorkflowNodeDetails().getType() == WorkflowNodeDetails.Type.OTP) {
             return processOtpNode(node, valuesMap);
@@ -201,7 +217,6 @@ else if (node.getWorkflowNodeDetails().getType() == WorkflowNodeDetails.Type.OTP
         }
     }
 
-
     public static WorkflowTestResult.NodeResult processApiNode(Node node, Map<String, Object> valuesMap, Boolean allowAllStatusCodes, boolean debug, List<TestingRunResult.TestLog> testLogs, Memory memory) {
 
         NodeExecutorFactory nodeExecutorFactory = new NodeExecutorFactory();
@@ -246,7 +261,7 @@ public static LoginFlowResponse runLoginFlow(WorkflowTest workflowTest, AuthMech
                 if (authMechanism.getRequestData() != null && authMechanism.getRequestData().size() > 0 && authMechanism.getRequestData().get(index).getAllowAllStatusCodes()) {
                     allowAllStatusCodes = authMechanism.getRequestData().get(0).getAllowAllStatusCodes();
                 }
-                nodeResult = processNode(node, valuesMap, allowAllStatusCodes, false, new ArrayList<>(), null);
+                nodeResult = processNode(node, valuesMap, allowAllStatusCodes, false, new ArrayList<>(), null, authMechanism);
             } catch (Exception e) {
                 ;
                 List<String> testErrors = new ArrayList<>();

libs/dao/src/main/java/com/akto/dao/testing/TestRolesDao.java

@@ -66,7 +66,11 @@ public AuthMechanism fetchAttackerToken(int apiCollectionId, RawApi rawApi) {
         TestRoles testRoles = TestRolesDao.instance.findOne(TestRoles.NAME, "ATTACKER_TOKEN_ALL");
         if (testRoles != null && testRoles.getAuthWithCondList().size() > 0) {
             List<AuthWithCond> authWithCondList = testRoles.getAuthWithCondList();
-            AuthMechanism defaultAuthMechanism = authWithCondList.get(0).getAuthMechanism();
+            AuthWithCond firstAuth = authWithCondList.get(0);
+            AuthMechanism defaultAuthMechanism = firstAuth.getAuthMechanism();
+            if(firstAuth.getRecordedLoginFlowInput()!=null){
+                defaultAuthMechanism.setRecordedLoginFlowInput(firstAuth.getRecordedLoginFlowInput());
+            }
             if (rawApi == null) {
                 return defaultAuthMechanism;
             } else {
@@ -88,7 +92,11 @@ public AuthMechanism fetchAttackerToken(int apiCollectionId, RawApi rawApi) {
                         }
 
                         if (allHeadersMatched) {
-                            return authWithCond.getAuthMechanism();
+                            defaultAuthMechanism = authWithCond.getAuthMechanism();
+                            if(authWithCond.getRecordedLoginFlowInput()!=null){
+                                defaultAuthMechanism.setRecordedLoginFlowInput(authWithCond.getRecordedLoginFlowInput());
+                            }
+                            return defaultAuthMechanism;
                         }
                     }
                 } catch (Exception e) {