From 53a70f10d50aabd4850a66f9de7f68f74c39c459 Mon Sep 17 00:00:00 2001 From: Umesh Kumar <166806589+TangoBeeAkto@users.noreply.github.com> Date: Tue, 24 Dec 2024 12:51:47 +0530 Subject: [PATCH 01/40] feat: made automated login test flow editable --- .../com/akto/action/testing/TestRolesAction.java | 2 +- .../pages/testing/TestRoleSettings/ParamsCard.jsx | 2 +- .../testing/TestRoleSettings/TestRoleSettings.jsx | 14 ++++++++++++-- .../web/src/apps/dashboard/pages/testing/api.js | 4 ++-- .../pages/testing/user_config/JsonRecording.jsx | 2 +- .../pages/testing/user_config/LoginStepBuilder.jsx | 2 +- 6 files changed, 18 insertions(+), 8 deletions(-) diff --git a/apps/dashboard/src/main/java/com/akto/action/testing/TestRolesAction.java b/apps/dashboard/src/main/java/com/akto/action/testing/TestRolesAction.java index 5a365b3503..c7e44d06bd 100644 --- a/apps/dashboard/src/main/java/com/akto/action/testing/TestRolesAction.java +++ b/apps/dashboard/src/main/java/com/akto/action/testing/TestRolesAction.java @@ -86,7 +86,7 @@ private AuthWithCond makeAuthWithConditionFromParamData(TestRoles role){ for (AuthParamData authParamDataElem : authParamData) { AuthParam param = null; - if (authAutomationType.equals(LoginFlowEnums.AuthMechanismTypes.HARDCODED.toString())) { + if (authAutomationType.toUpperCase().equals(LoginFlowEnums.AuthMechanismTypes.HARDCODED.toString())) { param = new HardcodedAuthParam(authParamDataElem.getWhere(), authParamDataElem.getKey(), authParamDataElem.getValue(), true); } else { param = new LoginRequestAuthParam(authParamDataElem.getWhere(), authParamDataElem.getKey(), authParamDataElem.getValue(), authParamDataElem.getShowHeader()); diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/TestRoleSettings/ParamsCard.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/TestRoleSettings/ParamsCard.jsx index e7dd96427d..49a7b41ba4 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/TestRoleSettings/ParamsCard.jsx +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/TestRoleSettings/ParamsCard.jsx @@ -69,7 +69,7 @@ function ParamsCard({dataObj, handleDelete, showEdit}) { - {authMechanism?.type?.toLowerCase() === 'hardcoded' ? : null} + diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/TestRoleSettings/TestRoleSettings.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/TestRoleSettings/TestRoleSettings.jsx index dd30d7488e..b6dd25500c 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/TestRoleSettings/TestRoleSettings.jsx +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/TestRoleSettings/TestRoleSettings.jsx @@ -81,6 +81,7 @@ function TestRoleSettings() { const resetFunc = (newItems) => { setChange(false); setRoleName(newItems.name || systemRole || ""); + setAuthMechanism(null) dispatchConditions({type:"replace", conditions:transform.createConditions(newItems.endpoints)}) } useEffect(() => { @@ -186,7 +187,7 @@ function TestRoleSettings() { setAdvancedHeaderSettingsOpen(true) } setShowAuthComponent(true) - setHardcodedOpen(true) + setHardcodedOpen(authObj?.authMechanism?.type === "HardCoded") setEditableDocs(index) } @@ -305,6 +306,8 @@ function TestRoleSettings() { setHeaderKey('') setHeaderValue('') setHardCodeAuthInfo({authParams:[]}) + setAuthMechanism(null) + setHardcodedOpen(true) } const handleSaveAuthMechanism = async() => { @@ -332,8 +335,15 @@ function TestRoleSettings() { errorFilePath: null, } } + + if(editableDoc > -1) { + resp = await api.updateAuthInRole(initialItems.name, apiCond, editableDoc, currentInfo.authParams, automationType, currentInfo.steps, recordedLoginFlowInput) + } else { + resp = await api.addAuthToRole(initialItems.name, apiCond, currentInfo.authParams, automationType, currentInfo.steps, recordedLoginFlowInput) + } + } else { + func.setToast(true, true, "Request data cannot be empty!") } - resp = await api.addAuthToRole(initialItems.name, apiCond, currentInfo.authParams, automationType, currentInfo.steps, recordedLoginFlowInput) } handleCancel() await saveAction(true, resp.selectedRole.authWithCondList) diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/api.js b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/api.js index be973bfa69..abbb7a61ff 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/api.js +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/api.js @@ -256,11 +256,11 @@ export default { data: {roleName, index} }) }, - updateAuthInRole(roleName, apiCond ,index, authParamData, authAutomationType) { + updateAuthInRole(roleName, apiCond ,index, authParamData, authAutomationType, reqData, recordedLoginFlowInput) { return request({ url: '/api/updateAuthInRole', method: 'post', - data: {roleName, apiCond, index, authParamData, authAutomationType} + data: {roleName, apiCond, index, authParamData, authAutomationType, reqData, recordedLoginFlowInput} }) }, deleteTestRuns(testRunIds){ diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/user_config/JsonRecording.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/user_config/JsonRecording.jsx index bc4f0edb60..71f45be56f 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/user_config/JsonRecording.jsx +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/user_config/JsonRecording.jsx @@ -25,7 +25,7 @@ function JsonRecording({extractInformation, showOnlyApi, setStoreData}) { }]) useEffect(() => { - if (!extractInformation) { + if (extractInformation) { if (authMechanism && authMechanism.type === "LOGIN_REQUEST" && authMechanism.requestData[0].type === "RECORDED_FLOW") { setTokenFetchCommand(authMechanism.requestData[0].tokenFetchCommand) setAuthParams(authMechanism.authParams) diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/user_config/LoginStepBuilder.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/user_config/LoginStepBuilder.jsx index a7b249928a..b6f168e955 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/user_config/LoginStepBuilder.jsx +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/user_config/LoginStepBuilder.jsx @@ -49,7 +49,7 @@ function LoginStepBuilder({extractInformation, showOnlyApi, setStoreData}) { useEffect(() => { - if(!extractInformation){ + if(extractInformation){ setIsLoading(true) if (authMechanism && authMechanism.type === "LOGIN_REQUEST" && authMechanism.requestData[0].type !== "RECORDED_FLOW") { setSteps(authMechanism.requestData.map((step, index) => ({ From 74bbddc237e03189b95d79d5d30e3cb8b642a383 Mon Sep 17 00:00:00 2001 From: Umesh Kumar <166806589+TangoBeeAkto@users.noreply.github.com> Date: Tue, 24 Dec 2024 13:28:38 +0530 Subject: [PATCH 02/40] fix: fixed a bug --- .../pages/testing/TestRoleSettings/TestRoleSettings.jsx | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/TestRoleSettings/TestRoleSettings.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/TestRoleSettings/TestRoleSettings.jsx index b6dd25500c..f38242d09a 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/TestRoleSettings/TestRoleSettings.jsx +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/TestRoleSettings/TestRoleSettings.jsx @@ -308,6 +308,7 @@ function TestRoleSettings() { setHardCodeAuthInfo({authParams:[]}) setAuthMechanism(null) setHardcodedOpen(true) + setEditableDocs(-1) } const handleSaveAuthMechanism = async() => { From fba5ee39c8cbcb245bb1910e2b64002b8f2800cb Mon Sep 17 00:00:00 2001 From: Umesh Kumar <166806589+TangoBeeAkto@users.noreply.github.com> Date: Tue, 24 Dec 2024 16:15:23 +0530 Subject: [PATCH 03/40] fix: update login test flow data when user clicks on multiple edit auth token button --- .../apps/dashboard/pages/testing/user_config/JsonRecording.jsx | 2 +- .../dashboard/pages/testing/user_config/LoginStepBuilder.jsx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/user_config/JsonRecording.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/user_config/JsonRecording.jsx index 71f45be56f..df31619336 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/user_config/JsonRecording.jsx +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/user_config/JsonRecording.jsx @@ -35,7 +35,7 @@ function JsonRecording({extractInformation, showOnlyApi, setStoreData}) { } else { return; } - }, []) + }, [authMechanism]) const inputRef = useRef(null); diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/user_config/LoginStepBuilder.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/user_config/LoginStepBuilder.jsx index b6f168e955..57dba6036c 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/user_config/LoginStepBuilder.jsx +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/user_config/LoginStepBuilder.jsx @@ -65,7 +65,7 @@ function LoginStepBuilder({extractInformation, showOnlyApi, setStoreData}) { }else{ return; } - }, []) + }, [authMechanism]) const stepOptions = [ { label: "Call API", value: "LOGIN_FORM" }, From bd8e0b516a7d20c773f2587bcbc265c34d7e3957 Mon Sep 17 00:00:00 2001 From: Umesh Kumar <166806589+TangoBeeAkto@users.noreply.github.com> Date: Wed, 25 Dec 2024 01:10:08 +0530 Subject: [PATCH 04/40] fix: fixed dropdown for auth params --- .../src/apps/dashboard/pages/testing/user_config/AuthParams.jsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/user_config/AuthParams.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/user_config/AuthParams.jsx index 3c49c0afbf..294d155664 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/user_config/AuthParams.jsx +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/user_config/AuthParams.jsx @@ -60,7 +60,7 @@ function AuthParams({ authParams, setAuthParams, hideTitle }) {
handleUpdate(index, "where", authParamLocation)} /> Key: handleUpdate(index, "key", key)} /> From ef671d254dc6f423cb0033ab0d795fe0984227d6 Mon Sep 17 00:00:00 2001 From: Umesh Kumar <166806589+TangoBeeAkto@users.noreply.github.com> Date: Wed, 25 Dec 2024 01:41:39 +0530 Subject: [PATCH 05/40] fix: refresh after attacker token test role update --- .../main/java/com/akto/action/testing/TestRolesAction.java | 4 ++-- .../pages/testing/TestRoleSettings/TestRoleSettings.jsx | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/apps/dashboard/src/main/java/com/akto/action/testing/TestRolesAction.java b/apps/dashboard/src/main/java/com/akto/action/testing/TestRolesAction.java index c7e44d06bd..94536e48bb 100644 --- a/apps/dashboard/src/main/java/com/akto/action/testing/TestRolesAction.java +++ b/apps/dashboard/src/main/java/com/akto/action/testing/TestRolesAction.java @@ -180,8 +180,8 @@ public String updateTestRoles() { isAttackerRole = role.getId().equals(attackerRole.getId()); } if (isAttackerRole) { - addActionError("Unable to update endpoint conditions for attacker role"); - return ERROR.toUpperCase(); + this.orConditions = null; + this.andConditions = null; } Conditions orConditions = null; diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/TestRoleSettings/TestRoleSettings.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/TestRoleSettings/TestRoleSettings.jsx index f38242d09a..115d0780e7 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/TestRoleSettings/TestRoleSettings.jsx +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/TestRoleSettings/TestRoleSettings.jsx @@ -131,7 +131,7 @@ function TestRoleSettings() { const saveAction = async (updatedAuth=false, authWithCondLists = null) => { let andConditions = transform.filterContainsConditions(conditions, 'AND') let orConditions = transform.filterContainsConditions(conditions, 'OR') - if (!(andConditions || orConditions) || roleName.length === 0) { + if (roleName !== 'ATTACKER_TOKEN_ALL' && !(andConditions || orConditions) || roleName.length === 0) { func.setToast(true, true, "Please select valid values for a test role") } else { if (isNew) { @@ -235,7 +235,7 @@ function TestRoleSettings() { } } - const conditionsCard = ( + const conditionsCard = roleName !== 'ATTACKER_TOKEN_ALL' ? ( - ) + ) : (<>) const deleteModalComp = ( Date: Wed, 25 Dec 2024 03:14:39 +0530 Subject: [PATCH 06/40] add hide filter functionality to TrendChart component --- .../pages/testing/SingleTestRunPage/TrendChart.jsx | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/SingleTestRunPage/TrendChart.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/SingleTestRunPage/TrendChart.jsx index 11d18b4670..c5d7e074bb 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/SingleTestRunPage/TrendChart.jsx +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/SingleTestRunPage/TrendChart.jsx @@ -29,6 +29,7 @@ function TrendChart(props) { const [metadataFilterData, setMetadataFilterData] = useState([]); const [totalVulnerabilities, setTotalVulnerabilites] = useState(0); const [collapsible, setCollapsible] = useState(true) + const [hideFilter, setHideFilter] = useState(false) const dateRangeFilter = { @@ -38,7 +39,12 @@ function TrendChart(props) { ( getDate(dateObj)} - setPopoverState={() => { }} + setPopoverState={() => { + setHideFilter(true) + setTimeout(() => { + setHideFilter(false) + }, 10) + }} />), pinned: true } @@ -315,6 +321,7 @@ function TrendChart(props) { filters={[dateRangeFilter, ...metadataFilters]} appliedFilters={appliedFilters} onClearAll={handleFiltersClearAll} + hideFilters={hideFilter} />
From d8fe3b66d0aae10d5d843e126a4c09613b5837a0 Mon Sep 17 00:00:00 2001 From: notshivansh Date: Mon, 30 Dec 2024 14:04:10 +0530 Subject: [PATCH 07/40] update plan type if present --- .../dashboard/pages/settings/billing/Billing.jsx | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/settings/billing/Billing.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/settings/billing/Billing.jsx index 2000315374..b4d86c597b 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/settings/billing/Billing.jsx +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/settings/billing/Billing.jsx @@ -46,7 +46,21 @@ function Billing() { default: } - if (window.PLAN_TYPE) { $('.stigg-subscription-plan-name').text(window.PLAN_TYPE) } + if (window.PLAN_TYPE && window.PLAN_TYPE.length > 0) { + const observer = new MutationObserver((mutationsList) => { + for (const mutation of mutationsList) { + const elements = document.querySelectorAll('.stigg-subscription-plan-name'); + if (elements.length > 0) { + elements.forEach(element => { + element.textContent = window.PLAN_TYPE; + }); + observer.disconnect(); // Stop observing once the element is found + } + } + }); + observer.observe(document.body, { childList: true, subtree: true }); + return () => observer.disconnect(); // Cleanup on unmount + } }) async function refreshUsageData(){ From a1807767f7311a2ebcd05514d33c4cffb3c381a8 Mon Sep 17 00:00:00 2001 From: notshivansh Date: Mon, 30 Dec 2024 15:30:02 +0530 Subject: [PATCH 08/40] update approach for plan check --- .../pages/settings/billing/Billing.jsx | 22 +++++++++---------- 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/settings/billing/Billing.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/settings/billing/Billing.jsx index b4d86c597b..49024d7555 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/settings/billing/Billing.jsx +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/settings/billing/Billing.jsx @@ -47,19 +47,17 @@ function Billing() { } if (window.PLAN_TYPE && window.PLAN_TYPE.length > 0) { - const observer = new MutationObserver((mutationsList) => { - for (const mutation of mutationsList) { - const elements = document.querySelectorAll('.stigg-subscription-plan-name'); - if (elements.length > 0) { - elements.forEach(element => { - element.textContent = window.PLAN_TYPE; - }); - observer.disconnect(); // Stop observing once the element is found - } + const checkForElement = () => { + const elements = document.querySelectorAll('.stigg-subscription-plan-name'); + if (elements.length > 0) { + elements.forEach(element => { + element.textContent = window.PLAN_TYPE; + }); + } else { + setTimeout(() => checkForElement(), 500); } - }); - observer.observe(document.body, { childList: true, subtree: true }); - return () => observer.disconnect(); // Cleanup on unmount + }; + checkForElement(); } }) From 4dc62f7077b5e949ca6b3bd8848def7c854134d3 Mon Sep 17 00:00:00 2001 From: Ark2307 Date: Mon, 30 Dec 2024 16:52:18 +0530 Subject: [PATCH 09/40] Fixing count for api groups --- .../com/akto/action/ApiCollectionsAction.java | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/apps/dashboard/src/main/java/com/akto/action/ApiCollectionsAction.java b/apps/dashboard/src/main/java/com/akto/action/ApiCollectionsAction.java index db9372f096..440c3ad77e 100644 --- a/apps/dashboard/src/main/java/com/akto/action/ApiCollectionsAction.java +++ b/apps/dashboard/src/main/java/com/akto/action/ApiCollectionsAction.java @@ -84,11 +84,25 @@ public List fillApiCollectionsUrlCount(List apiCol int apiCollectionId = apiCollection.getId(); Integer count = countMap.get(apiCollectionId); int fallbackCount = apiCollection.getUrls()!=null ? apiCollection.getUrls().size() : apiCollection.getUrlsCount(); - if (count != null && (apiCollection.getHostName() != null)) { apiCollection.setUrlsCount(count); } else if(ApiCollection.Type.API_GROUP.equals(apiCollection.getType())){ - if (count == null) { + int conditionsCount = 0; + if(!apiCollection.getAutomated()){ + ApiCollection apiCollectionWithCond = ApiCollectionsDao.instance.findOne(Filters.eq(Constants.ID, apiCollection.getId()), Projections.include("conditions")); + if(apiCollectionWithCond.getConditions() != null && apiCollectionWithCond.getConditions().get(0) != null){ + if(apiCollectionWithCond.getConditions().get(0).getType().equals(TestingEndpoints.Type.CUSTOM)){ + CustomTestingEndpoints testingEndpoints = (CustomTestingEndpoints) apiCollectionWithCond.getConditions().get(0); + if(testingEndpoints.getApisList() != null && !testingEndpoints.getApisList().isEmpty()){ + conditionsCount = testingEndpoints.getApisList().size(); + } + } + } + } + + if(conditionsCount != 0){ + count = conditionsCount; + }else if (count == null) { count = fallbackCount; } apiCollection.setUrlsCount(count); From 84f93778d0a16b9d1b2f00ab269aecc32f5369e3 Mon Sep 17 00:00:00 2001 From: Ark2307 Date: Mon, 30 Dec 2024 17:36:40 +0530 Subject: [PATCH 10/40] Added debug logs --- .../src/main/java/com/akto/action/ApiCollectionsAction.java | 3 +++ libs/dao/src/main/java/com/akto/dto/ApiCollection.java | 1 - 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/apps/dashboard/src/main/java/com/akto/action/ApiCollectionsAction.java b/apps/dashboard/src/main/java/com/akto/action/ApiCollectionsAction.java index 440c3ad77e..efb90f2876 100644 --- a/apps/dashboard/src/main/java/com/akto/action/ApiCollectionsAction.java +++ b/apps/dashboard/src/main/java/com/akto/action/ApiCollectionsAction.java @@ -410,8 +410,11 @@ public String addApisToCustomCollection(){ return ERROR.toUpperCase(); } + loggerMaker.infoAndAddToDb("Started adding " + this.apiList.size() + " apis into custom collection.", LogDb.DASHBOARD); + CustomTestingEndpoints condition = new CustomTestingEndpoints(apiList, CustomTestingEndpoints.Operator.OR); apiCollection.addToConditions(condition); + loggerMaker.infoAndAddToDb("Final conditions for collection: " + apiCollection.getName() + " are: " + apiCollection.getConditions().toString()); ApiCollectionUsers.updateApiCollection(apiCollection.getConditions(), apiCollection.getId()); ApiCollectionUsers.addToCollectionsForCollectionId(apiCollection.getConditions(), apiCollection.getId()); diff --git a/libs/dao/src/main/java/com/akto/dto/ApiCollection.java b/libs/dao/src/main/java/com/akto/dto/ApiCollection.java index 56f479ffa0..d98a692060 100644 --- a/libs/dao/src/main/java/com/akto/dto/ApiCollection.java +++ b/libs/dao/src/main/java/com/akto/dto/ApiCollection.java @@ -4,7 +4,6 @@ import java.util.Arrays; import java.util.HashSet; import java.util.List; -import java.util.Objects; import java.util.Set; import org.bson.codecs.pojo.annotations.BsonId; From 4fffdd136e79a00c479b9c8d464f168351a5ce5f Mon Sep 17 00:00:00 2001 From: Umesh Kumar <166806589+TangoBeeAkto@users.noreply.github.com> Date: Mon, 30 Dec 2024 19:26:03 +0530 Subject: [PATCH 11/40] hiding deactivated collections from the dropdown search menu --- .../components/ConditionComponent.jsx | 16 +++++++++----- .../components/shared/DropdownSearch.jsx | 4 ++-- .../observe/api_collections/ApiGroupModal.jsx | 21 +++++++++++++------ .../web/polaris_web/web/src/util/func.js | 4 +++- 4 files changed, 31 insertions(+), 14 deletions(-) diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/ConditionComponent.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/ConditionComponent.jsx index e4f04456ab..5c78143018 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/ConditionComponent.jsx +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/ConditionComponent.jsx @@ -16,12 +16,16 @@ function ConditionComponent(props) { } },[condition]) const allCollections = PersistStore(state => state.allCollections); - const allCollectionsOptions = allCollections.map(collection => { - return { - label: collection.displayName, - value: collection.id + const activatedCollections = allCollections.filter(collection => collection.deactivated === false) + const allCollectionsOptions = [ + { + title: `Search from ${activatedCollections.length} Collection${func.addPlurality(activatedCollections.length)} (type more to refine results)`, + options: activatedCollections.map(collection => ({ + label: collection.displayName, + value: collection.id + })) } - }) + ] const getApiEndpointsOptions = (data) => { return data.map(apiEndpoint => { let str = func.toMethodUrlString(apiEndpoint); @@ -115,6 +119,8 @@ function ConditionComponent(props) { setSelected={(collectionId) => handleCollectionSelected(collectionId)} preSelected={[Number(getCollectionId(field))]} value={mapCollectionIdToName[getCollectionId(field)]} + isNested={true} + dynamicTitle={true} />
diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/shared/DropdownSearch.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/shared/DropdownSearch.jsx index 2882edb20e..c2df5b374c 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/shared/DropdownSearch.jsx +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/shared/DropdownSearch.jsx @@ -6,7 +6,7 @@ function DropdownSearch(props) { const id = props.id ? props.id : "dropdown-search" - const { disabled, label, placeholder, optionsList, setSelected, value , avatarIcon, preSelected, allowMultiple, itemName, dropdownSearchKey, isNested, sliceMaxVal} = props + const { disabled, label, placeholder, optionsList, setSelected, value , avatarIcon, preSelected, allowMultiple, itemName, dropdownSearchKey, isNested, sliceMaxVal, dynamicTitle} = props const deselectedOptions = optionsList const [selectedOptions, setSelectedOptions] = useState(preSelected ? preSelected : []); @@ -81,7 +81,7 @@ function DropdownSearch(props) { ); resultOptions.push({ - title: opt.title, + title: dynamicTitle ? `Showing ${options.length} result${func.addPlurality(options.length)} (type more to refine results)` : opt.title, options, }); }); diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/ApiGroupModal.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/ApiGroupModal.jsx index fb7443e108..fb0a84b553 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/ApiGroupModal.jsx +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/ApiGroupModal.jsx @@ -19,6 +19,7 @@ function ApiGroupModal(props){ const setCollectionsMap = PersistStore(state => state.setCollectionsMap) const allCollections = PersistStore(state => state.allCollections); const setAllCollections = PersistStore(state => state.setAllCollections) + const activatedGroupCollections = allCollections.filter((x) => { return (x.type === 'API_GROUP' && x.deactivated === false) }) const [apiGroupName, setApiGroupName] = useState(currentApiGroupName) @@ -65,14 +66,22 @@ function ApiGroupModal(props){ id={"select-api-group"} label="Select API group" placeholder="Select API group" - optionsList={allCollections.filter((x) => { return x.type === 'API_GROUP' }).map((x) => { - return { - label: x.displayName, - value: x.displayName - } - }) + optionsList={ + [ + { + title: `Search from ${activatedGroupCollections.length} Group${func.addPlurality(activatedGroupCollections.length)} (type more to refine results)`, + options: allCollections.filter((x) => { return (x.type === 'API_GROUP' && x.deactivated === false) }).map((x) => { + return { + label: x.displayName, + value: x.displayName + } + }) + } + ] } setSelected={setApiGroupName} + dynamicTitle={true} + isNested={true} /> ) diff --git a/apps/dashboard/web/polaris_web/web/src/util/func.js b/apps/dashboard/web/polaris_web/web/src/util/func.js index c473eb6a85..652b884201 100644 --- a/apps/dashboard/web/polaris_web/web/src/util/func.js +++ b/apps/dashboard/web/polaris_web/web/src/util/func.js @@ -1246,9 +1246,11 @@ getDeprecatedEndpoints(apiInfoList, unusedEndpoints, apiCollectionId) { getSearchItemsArr(allRoutes,allCollections){ let combinedArr = [] + const activatedColections = allCollections.filter((item) => item.deactivated === false) + let initialStr = "/dashboard/observe/inventory/" - allCollections.forEach((item)=> { + activatedColections.forEach((item)=> { combinedArr.push({content: item.displayName, url: initialStr + item.id, type:'collection'}) }) From 0ea012708fe63d725f757960f7c42d3fa644c319 Mon Sep 17 00:00:00 2001 From: Umesh Kumar <166806589+TangoBeeAkto@users.noreply.github.com> Date: Mon, 30 Dec 2024 19:57:31 +0530 Subject: [PATCH 12/40] showing first 20 items in dropdown --- .../web/src/apps/dashboard/components/shared/DropdownSearch.jsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/shared/DropdownSearch.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/shared/DropdownSearch.jsx index c2df5b374c..28962c8826 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/shared/DropdownSearch.jsx +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/shared/DropdownSearch.jsx @@ -78,7 +78,7 @@ function DropdownSearch(props) { deselectedOptions.forEach((opt) => { const options = opt.options.filter((option) => option[searchKey].match(filterRegex), - ); + ).slice(0, sliceMaxVal || 20); resultOptions.push({ title: dynamicTitle ? `Showing ${options.length} result${func.addPlurality(options.length)} (type more to refine results)` : opt.title, From c4a38feeac8532b6c8fe87bad07149f0dcc9d42a Mon Sep 17 00:00:00 2001 From: Ark2307 Date: Mon, 30 Dec 2024 22:17:34 +0530 Subject: [PATCH 13/40] Calculating count map at the end of testing --- .../src/main/java/com/akto/testing/Main.java | 20 ++++++++- .../java/com/akto/testing/TestExecutor.java | 4 +- .../src/main/java/com/akto/testing/Utils.java | 42 +++++++++++++++++++ 3 files changed, 63 insertions(+), 3 deletions(-) diff --git a/apps/testing/src/main/java/com/akto/testing/Main.java b/apps/testing/src/main/java/com/akto/testing/Main.java index 6b3f940d98..b0fc059786 100644 --- a/apps/testing/src/main/java/com/akto/testing/Main.java +++ b/apps/testing/src/main/java/com/akto/testing/Main.java @@ -278,6 +278,9 @@ public void run() { public static void main(String[] args) throws InterruptedException { String mongoURI = System.getenv("AKTO_MONGO_CONN"); ReadPreference readPreference = ReadPreference.secondary(); + if(DashboardMode.isOnPremDeployment()){ + readPreference = ReadPreference.primary(); + } WriteConcern writeConcern = WriteConcern.W1; DaoInit.init(new ConnectionString(mongoURI), readPreference, writeConcern); @@ -356,9 +359,16 @@ public void run() { loggerMaker.infoAndAddToDb("Testing run stopped"); if (trrs != null) { loggerMaker.infoAndAddToDb("Stopping TRRS: " + trrs.getId()); + + // get count issues here + Map finalCountMap = Utils.finalCountIssuesMap(trrs.getId()); + loggerMaker.infoAndAddToDb("Final count map calculated is " + finalCountMap.toString()); TestingRunResultSummariesDao.instance.updateOneNoUpsert( Filters.eq(Constants.ID, trrs.getId()), - Updates.set(TestingRunResultSummary.STATE, State.STOPPED) + Updates.combine( + Updates.set(TestingRunResultSummary.STATE, State.STOPPED), + Updates.set(TestingRunResultSummary.COUNT_ISSUES, finalCountMap) + ) ); loggerMaker.infoAndAddToDb("Stopped TRRS: " + trrs.getId()); } @@ -441,10 +451,16 @@ public void run() { + testingRunResult.getHexId() + ", TRRS_ID:" + testingRunResultSummary.getHexId() + " TR_ID:" + testingRun.getHexId(), LogDb.TESTING); int countFailedSummaries = (int) TestingRunResultSummariesDao.instance.count(filterCountFailed); - Bson updateForSummary = Updates.set(TestingRunResultSummary.STATE, State.FAILED); + Map finalCountMap = Utils.finalCountIssuesMap(testingRunResultSummary.getId()); + loggerMaker.infoAndAddToDb("Final count map calculated is " + finalCountMap.toString()); + Bson updateForSummary = Updates.combine( + Updates.set(TestingRunResultSummary.STATE, State.FAILED), + Updates.set(TestingRunResultSummary.COUNT_ISSUES, finalCountMap) + ); if(countFailedSummaries >= (MAX_RETRIES_FOR_FAILED_SUMMARIES - 1)){ updateForSummary = Updates.combine( Updates.set(TestingRunResultSummary.STATE, State.COMPLETED), + Updates.set(TestingRunResultSummary.COUNT_ISSUES, finalCountMap), Updates.set(TestingRunResultSummary.END_TIMESTAMP, Context.now()) ); loggerMaker.infoAndAddToDb("Max retries level reached for TRR_ID: " + testingRun.getHexId(), LogDb.TESTING); diff --git a/apps/testing/src/main/java/com/akto/testing/TestExecutor.java b/apps/testing/src/main/java/com/akto/testing/TestExecutor.java index a8a0a7c585..7e1b799bcc 100644 --- a/apps/testing/src/main/java/com/akto/testing/TestExecutor.java +++ b/apps/testing/src/main/java/com/akto/testing/TestExecutor.java @@ -300,11 +300,13 @@ public static void updateTestSummary(ObjectId summaryId){ options.returnDocument(ReturnDocument.AFTER); State updatedState = GetRunningTestsStatus.getRunningTests().isTestRunning(summaryId, true) ? State.COMPLETED : GetRunningTestsStatus.getRunningTests().getCurrentState(summaryId); - + Map finalCountMap = Utils.finalCountIssuesMap(summaryId); + loggerMaker.infoAndAddToDb("Final count map calculated is " + finalCountMap.toString()); TestingRunResultSummary testingRunResultSummary = TestingRunResultSummariesDao.instance.getMCollection().withWriteConcern(WriteConcern.W1).findOneAndUpdate( Filters.eq(Constants.ID, summaryId), Updates.combine( Updates.set(TestingRunResultSummary.END_TIMESTAMP, Context.now()), + Updates.set(TestingRunResultSummary.COUNT_ISSUES, finalCountMap), Updates.set(TestingRunResultSummary.STATE, updatedState)), options); GithubUtils.publishGithubComments(testingRunResultSummary); diff --git a/libs/utils/src/main/java/com/akto/testing/Utils.java b/libs/utils/src/main/java/com/akto/testing/Utils.java index 3a2caaa710..3999cce874 100644 --- a/libs/utils/src/main/java/com/akto/testing/Utils.java +++ b/libs/utils/src/main/java/com/akto/testing/Utils.java @@ -1,5 +1,6 @@ package com.akto.testing; +import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -10,6 +11,9 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; +import org.bson.conversions.Bson; +import org.bson.types.ObjectId; +import com.akto.dao.testing.TestingRunResultDao; import com.akto.dto.ApiInfo.ApiInfoKey; import com.akto.dto.CollectionConditions.ConditionsType; import com.akto.dto.OriginalHttpRequest; @@ -17,6 +21,7 @@ import com.akto.dto.test_editor.DataOperandsFilterResponse; import com.akto.dto.test_editor.FilterNode; import com.akto.dto.test_editor.Util; +import com.akto.dto.testing.TestingRunResult; import com.akto.dto.testing.WorkflowUpdatedSampleData; import com.akto.dto.type.RequestTemplate; import com.akto.log.LoggerMaker; @@ -24,8 +29,14 @@ import com.akto.test_editor.filter.Filter; import com.akto.test_editor.filter.data_operands_impl.ValidationResult; import com.akto.util.JSONUtils; +import com.akto.util.enums.GlobalEnums.Severity; import com.mongodb.BasicDBList; import com.mongodb.BasicDBObject; +import com.mongodb.client.MongoCursor; +import com.mongodb.client.model.Accumulators; +import com.mongodb.client.model.Aggregates; +import com.mongodb.client.model.Filters; +import com.mongodb.client.model.Projections; import okhttp3.MediaType; @@ -412,5 +423,36 @@ public static void modifyHeaderOperations(OriginalHttpRequest httpRequest, List< } + + public static Map finalCountIssuesMap(ObjectId testingRunResultSummaryId){ + Map countIssuesMap = new HashMap<>(); + countIssuesMap.put(Severity.HIGH.toString(), 0); + countIssuesMap.put(Severity.MEDIUM.toString(), 0); + countIssuesMap.put(Severity.LOW.toString(), 0); + + Bson projection = Projections.computed("confidence", Projections.computed("$first", "$testResults.confidence")); + Bson filterQ = Filters.and( + Filters.eq(TestingRunResult.TEST_RUN_RESULT_SUMMARY_ID, testingRunResultSummaryId), + Filters.eq(TestingRunResult.VULNERABLE, true) + ); + BasicDBObject groupId = new BasicDBObject("_id", "$confidence"); + List pipeline = new ArrayList<>(); + + pipeline.add(Aggregates.match(filterQ)); + pipeline.add(Aggregates.project(projection)); + pipeline.add(Aggregates.group(groupId, Accumulators.sum("count", 1))); + + MongoCursor cursor = TestingRunResultDao.instance.getMCollection().aggregate(pipeline, BasicDBObject.class).cursor(); + while(cursor.hasNext()){ + BasicDBObject dbObject = cursor.next(); + BasicDBObject objectId = (BasicDBObject) dbObject.get("_id"); + String id = objectId.getString("_id"); + int val = dbObject.getInt("count"); + + countIssuesMap.put(id, val); + } + + return countIssuesMap; + } } From 43c43092cbc2c05cb0e434b6e0547763d645b9d8 Mon Sep 17 00:00:00 2001 From: Umesh Kumar <166806589+TangoBeeAkto@users.noreply.github.com> Date: Mon, 30 Dec 2024 22:45:13 +0530 Subject: [PATCH 14/40] hiding deactivated collections from the test editor's dropdown search menu and showing title directly inside dropdown comp --- .../components/CollectionComponent.jsx | 3 ++- .../components/ConditionComponent.jsx | 15 ++++---------- .../components/shared/DropdownSearch.jsx | 20 ++++++++++++++----- .../observe/api_collections/ApiGroupModal.jsx | 17 +++++----------- .../test_editor/components/SampleApi.jsx | 3 ++- .../TestRunsPage/TestrunsBannerComponent.jsx | 2 +- 6 files changed, 29 insertions(+), 31 deletions(-) diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/CollectionComponent.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/CollectionComponent.jsx index 8a76a9e200..43ce23584b 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/CollectionComponent.jsx +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/CollectionComponent.jsx @@ -34,7 +34,8 @@ function CollectionComponent(props) { }, [condition]) const allCollections = PersistStore(state => state.allCollections); - const allCollectionsOptions = allCollections.filter(x => x.type !== "API_GROUP") + const activatedCollections = allCollections.filter(collection => collection.deactivated === false) + const allCollectionsOptions = activatedCollections.filter(x => x.type !== "API_GROUP") .map(collection => { return { label: collection.displayName, diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/ConditionComponent.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/ConditionComponent.jsx index 5c78143018..820334b2c2 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/ConditionComponent.jsx +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/ConditionComponent.jsx @@ -17,15 +17,10 @@ function ConditionComponent(props) { },[condition]) const allCollections = PersistStore(state => state.allCollections); const activatedCollections = allCollections.filter(collection => collection.deactivated === false) - const allCollectionsOptions = [ - { - title: `Search from ${activatedCollections.length} Collection${func.addPlurality(activatedCollections.length)} (type more to refine results)`, - options: activatedCollections.map(collection => ({ - label: collection.displayName, - value: collection.id - })) - } - ] + const allCollectionsOptions = activatedCollections.map(collection => ({ + label: collection.displayName, + value: collection.id + })) const getApiEndpointsOptions = (data) => { return data.map(apiEndpoint => { let str = func.toMethodUrlString(apiEndpoint); @@ -119,8 +114,6 @@ function ConditionComponent(props) { setSelected={(collectionId) => handleCollectionSelected(collectionId)} preSelected={[Number(getCollectionId(field))]} value={mapCollectionIdToName[getCollectionId(field)]} - isNested={true} - dynamicTitle={true} />
diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/shared/DropdownSearch.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/shared/DropdownSearch.jsx index 28962c8826..adf632e989 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/shared/DropdownSearch.jsx +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/shared/DropdownSearch.jsx @@ -6,7 +6,7 @@ function DropdownSearch(props) { const id = props.id ? props.id : "dropdown-search" - const { disabled, label, placeholder, optionsList, setSelected, value , avatarIcon, preSelected, allowMultiple, itemName, dropdownSearchKey, isNested, sliceMaxVal, dynamicTitle} = props + const { disabled, label, placeholder, optionsList, setSelected, value , avatarIcon, preSelected, allowMultiple, itemName, dropdownSearchKey, isNested, sliceMaxVal} = props const deselectedOptions = optionsList const [selectedOptions, setSelectedOptions] = useState(preSelected ? preSelected : []); @@ -78,17 +78,27 @@ function DropdownSearch(props) { deselectedOptions.forEach((opt) => { const options = opt.options.filter((option) => option[searchKey].match(filterRegex), - ).slice(0, sliceMaxVal || 20); + ); resultOptions.push({ - title: dynamicTitle ? `Showing ${options.length} result${func.addPlurality(options.length)} (type more to refine results)` : opt.title, + title: opt.title, options, }); }); }else{ + const defaultSliceValue = sliceMaxVal || 20 resultOptions = deselectedOptions.filter((option) => - option[searchKey].match(filterRegex) - ); + option[searchKey].match(filterRegex) + ).slice(0, defaultSliceValue); + + const title = resultOptions.length >= defaultSliceValue + ? `Showing ${resultOptions.length} result${func.addPlurality(resultOptions.length)} only. (type more to refine results)` + : "Showing all results"; + + resultOptions = [{ + title: title, + options: resultOptions + }] } setOptions(resultOptions); setLoading(false); diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/ApiGroupModal.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/ApiGroupModal.jsx index fb0a84b553..3cf756e12e 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/ApiGroupModal.jsx +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/ApiGroupModal.jsx @@ -67,21 +67,14 @@ function ApiGroupModal(props){ label="Select API group" placeholder="Select API group" optionsList={ - [ - { - title: `Search from ${activatedGroupCollections.length} Group${func.addPlurality(activatedGroupCollections.length)} (type more to refine results)`, - options: allCollections.filter((x) => { return (x.type === 'API_GROUP' && x.deactivated === false) }).map((x) => { - return { - label: x.displayName, - value: x.displayName - } - }) + allCollections.filter((x) => { return (x.type === 'API_GROUP' && x.deactivated === false) }).map((x) => { + return { + label: x.displayName, + value: x.displayName } - ] + }) } setSelected={setApiGroupName} - dynamicTitle={true} - isNested={true} /> ) diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/test_editor/components/SampleApi.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/test_editor/components/SampleApi.jsx index 887c9c319c..6c61d73653 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/test_editor/components/SampleApi.jsx +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/test_editor/components/SampleApi.jsx @@ -154,7 +154,8 @@ const SampleApi = () => { } - const allCollectionsOptions = allCollections.map(collection => { + const activatedCollections = allCollections.filter(collection => collection.deactivated === false) + const allCollectionsOptions = activatedCollections.map(collection => { return { label: collection.displayName, value: collection.id diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/TestRunsPage/TestrunsBannerComponent.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/TestRunsPage/TestrunsBannerComponent.jsx index 23adac845d..af7a4d767d 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/TestRunsPage/TestrunsBannerComponent.jsx +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/TestRunsPage/TestrunsBannerComponent.jsx @@ -12,7 +12,7 @@ function SelectCollectionComponent() { const allCollections = PersistStore(state => state.allCollections); const navigate = useNavigate() let urlsCount = 0 - const allCollectionsOptions = allCollections.filter(x => x.type !== "API_GROUP") + const allCollectionsOptions = allCollections.filter(x => (x.type !== "API_GROUP" && x.deactivated === false)) .map(collection => { urlsCount += collection.urlsCount return { From ef5b0a5a0f39c70721fc727a7f9f510393d1fae7 Mon Sep 17 00:00:00 2001 From: notshivansh Date: Mon, 30 Dec 2024 22:54:02 +0530 Subject: [PATCH 15/40] replace API severity graph with issue severity graph --- .../action/testing_issues/IssuesAction.java | 31 +++++++++++++++++-- apps/dashboard/src/main/resources/struts.xml | 21 +++++++++++++ .../pages/dashboard/HomeDashboard.jsx | 25 ++++++++++----- .../src/apps/dashboard/pages/testing/api.js | 7 +++++ .../vulnerability_report/ReportSummary.jsx | 2 +- .../VulnerabilityReport.jsx | 4 ++- .../testing/vulnerability_report/transform.js | 26 +++------------- .../TestingRunIssuesDao.java | 22 ++++++++++--- 8 files changed, 101 insertions(+), 37 deletions(-) diff --git a/apps/dashboard/src/main/java/com/akto/action/testing_issues/IssuesAction.java b/apps/dashboard/src/main/java/com/akto/action/testing_issues/IssuesAction.java index 883aa251ce..cd9aa6f8fb 100644 --- a/apps/dashboard/src/main/java/com/akto/action/testing_issues/IssuesAction.java +++ b/apps/dashboard/src/main/java/com/akto/action/testing_issues/IssuesAction.java @@ -41,6 +41,7 @@ import com.mongodb.client.MongoCursor; import com.mongodb.client.model.*; import com.mongodb.client.result.InsertOneResult; +import com.opensymphony.xwork2.Action; import org.bson.Document; import org.bson.conversions.Bson; @@ -55,7 +56,7 @@ public class IssuesAction extends UserAction { - private static final LoggerMaker loggerMaker = new LoggerMaker(IssuesAction.class); + private static final LoggerMaker loggerMaker = new LoggerMaker(IssuesAction.class, LogDb.DASHBOARD); private static final Logger logger = LoggerFactory.getLogger(IssuesAction.class); private List issues; private TestingIssuesId issueId; @@ -74,6 +75,8 @@ public class IssuesAction extends UserAction { private List similarlyAffectedIssues; private int startEpoch; long endTimeStamp; + private Map> severityInfo = new HashMap<>(); + private Bson createFilters (boolean useFilterStatus) { Bson filters = Filters.empty(); if (useFilterStatus && filterStatus != null && !filterStatus.isEmpty()) { @@ -89,8 +92,12 @@ private Bson createFilters (boolean useFilterStatus) { filters = Filters.and(filters, Filters.in(ID + "." + TestingIssuesId.TEST_SUB_CATEGORY, filterSubCategory)); } - if (startEpoch != 0 && endTimeStamp != 0) { + + if (startEpoch != 0) { filters = Filters.and(filters, Filters.gte(TestingRunIssues.CREATION_TIME, startEpoch)); + } + + if(endTimeStamp != 0){ filters = Filters.and(filters, Filters.lt(TestingRunIssues.CREATION_TIME, endTimeStamp)); } @@ -628,6 +635,18 @@ public String getReportFilters () { return SUCCESS.toUpperCase(); } + public String fetchSeverityInfoForIssues() { + Bson filter = createFilters(true); + + if (issuesIds != null && !issuesIds.isEmpty()) { + filter = Filters.and(filter, Filters.in(Constants.ID, issuesIds)); + } + + this.severityInfo = TestingRunIssuesDao.instance.getSeveritiesMapForCollections(filter, false); + return Action.SUCCESS.toUpperCase(); + } + + public List getIssues() { return issues; } @@ -877,4 +896,12 @@ public void setIssuesIdsForReport(List issuesIdsForReport) { public BasicDBObject getResponse() { return response; } + + public Map> getSeverityInfo() { + return severityInfo; + } + + public void setSeverityInfo(Map> severityInfo) { + this.severityInfo = severityInfo; + } } diff --git a/apps/dashboard/src/main/resources/struts.xml b/apps/dashboard/src/main/resources/struts.xml index e9536f35af..26c53faa7f 100644 --- a/apps/dashboard/src/main/resources/struts.xml +++ b/apps/dashboard/src/main/resources/struts.xml @@ -4026,6 +4026,27 @@ + + + + + ISSUES + READ + + + + 403 + false + ^actionErrors.* + + + + 422 + false + ^actionErrors.* + + + diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/dashboard/HomeDashboard.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/dashboard/HomeDashboard.jsx index 8b35ca2168..d0611dec34 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/dashboard/HomeDashboard.jsx +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/dashboard/HomeDashboard.jsx @@ -134,7 +134,8 @@ function HomeDashboard() { observeApi.getUserEndpoints(), api.findTotalIssues(startTimestamp, endTimestamp), api.fetchApiStats(startTimestamp, endTimestamp), - api.fetchEndpointsCount(startTimestamp, endTimestamp) + api.fetchEndpointsCount(startTimestamp, endTimestamp), + testingApi.fetchSeverityInfoForIssues({ startEpoch: startTimestamp }, [], endTimestamp) ]; let results = await Promise.allSettled(apiPromises); @@ -143,6 +144,7 @@ function HomeDashboard() { let findTotalIssuesResp = results[1].status === 'fulfilled' ? results[1].value : {} let apisStatsResp = results[2].status === 'fulfilled' ? results[2].value : {} let fetchEndpointsCountResp = results[3].status === 'fulfilled' ? results[3].value : {} + let issueSeverityMap = results[4].status === 'fulfilled' ? results[4].value : {} setShowBannerComponent(!userEndpoints) @@ -153,7 +155,7 @@ function HomeDashboard() { buildAuthTypesData(apisStatsResp.apiStatsEnd) buildSetRiskScoreData(apisStatsResp.apiStatsEnd) //todo getCollectionsWithCoverage() - buildSeverityMap(apisStatsResp.apiStatsEnd) + buildSeverityMap(issueSeverityMap.severityInfo) buildIssuesSummary(findTotalIssuesResp) const fetchHistoricalDataResp = { "finalHistoricalData": finalHistoricalData, "initialHistoricalData": initialHistoricalData } @@ -447,8 +449,17 @@ function HomeDashboard() { /> ) : null - function buildSeverityMap(apiStats) { - const countMap = apiStats ? apiStats.criticalMap : {}; + function buildSeverityMap(severityInfo) { + const countMap = { HIGH: 0, MEDIUM: 0, LOW: 0 } + + if (severityInfo && severityInfo != undefined && severityInfo != null && severityInfo instanceof Object) { + for (const apiCollectionId in severityInfo) { + let temp = severityInfo[apiCollectionId] + for (const key in temp) { + countMap[key] += temp[key] + } + } + } const result = { "High": { @@ -508,11 +519,11 @@ function HomeDashboard() { />
} - title="Vulnerable APIs by Severity" - titleToolTip="Breakdown of vulnerable APIs categorized by severity level (High, Medium, Low). Click to see details for each category." + title="Issues by Severity" + titleToolTip="Breakdown of issues categorized by severity level (High, Medium, Low). Click to see details for each category." linkText="Fix critical issues" linkUrl="/dashboard/issues" - /> : No vulnerable APIs found: runTestEmptyCardComponent}/> + /> : No issues found for this time-frame: runTestEmptyCardComponent}/> const criticalUnsecuredAPIsOverTime = diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/api.js b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/api.js index abbb7a61ff..a3d530a321 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/api.js +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/api.js @@ -487,5 +487,12 @@ export default { method: 'post', data: { generatedReportId } }) + }, + fetchSeverityInfoForIssues(filters, issueIds, endTimeStamp) { + return request({ + url: '/api/fetchSeverityInfoForIssues', + method: 'post', + data: {...filters, issueIds, endTimeStamp} + }) } } \ No newline at end of file diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/vulnerability_report/ReportSummary.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/vulnerability_report/ReportSummary.jsx index dbbba1b9c3..f8c85fa164 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/vulnerability_report/ReportSummary.jsx +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/vulnerability_report/ReportSummary.jsx @@ -18,7 +18,7 @@ const ReportSummary = ({ reportSummaryItems, severityMap, graphData, organizatio - Vulnerable APIs by Severity + Issues by Severity { }) issuesFilters.filterCollectionsId = issuesFilters.filterCollectionsId.filter((x) => x !== undefined) + setCurrentDate(func.getFormattedDate(Date.now()/1000)) + await issuesApi.fetchVulnerableTestingRunResultsFromIssues(issuesFilters, issueIdsFromFilter, resultsCount).then(resp => { vulnerableTestingRunResults = [...vulnerableTestingRunResults, ...resp.testingRunResults] testingRunCountsFromDB = resp.testingRunResults.length @@ -207,7 +209,7 @@ const VulnerabilityReport = () => { setAktoRecommendations(vulMap.aktoRecommendations) setGraphData(vulMap.graphData) - const severityMapRes = reportTransform.createVulnerableAPIsSeverity(vulnerableTestingRunResults, categoryMap) + const severityMapRes = reportTransform.createVulnerableAPIsSeverity(vulnerableTestingRunResults) setSeverityMap(severityMapRes) } diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/vulnerability_report/transform.js b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/vulnerability_report/transform.js index 787b77e1f1..5ca146ee18 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/vulnerability_report/transform.js +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/vulnerability_report/transform.js @@ -2,33 +2,18 @@ import { Badge, Box, Link } from "@shopify/polaris" import func from "@/util/func" const reportTransform = { - createVulnerableAPIsSeverity: (vulnerableTestingRunResults, categoryMap) => { - - const severityOrder = func.getAktoSeverities().reverse() + createVulnerableAPIsSeverity: (vulnerableTestingRunResults) => { const countMap = { HIGH: 0, MEDIUM: 0, LOW: 0, } - const endpointMap = {} - vulnerableTestingRunResults.forEach(item => { - const endpointKey = `${item.apiInfoKey.apiCollectionId}-${item.apiInfoKey.method}-${item.apiInfoKey.url}` - const highestConfidence = item.testResults.reduce((max, result) => { - return severityOrder.indexOf(result.confidence) > severityOrder.indexOf(max) - ? result.confidence - : max - }, 'LOW') - if (!endpointMap[endpointKey] || severityOrder.indexOf(highestConfidence) > severityOrder.indexOf(endpointMap[endpointKey])) { - endpointMap[endpointKey] = highestConfidence - } - }) - - Object.values(endpointMap).forEach(severity => { - if(countMap[severity] !== undefined) { - countMap[severity]++ - } + const confidence = item.testResults.filter((result) => { + return result.vulnerable + }).map((result) => result.confidence)[0] + countMap[confidence]++ }) const result = { @@ -48,7 +33,6 @@ const reportTransform = { "filterKey": "Low" } } - return result }, diff --git a/libs/dao/src/main/java/com/akto/dao/testing_run_findings/TestingRunIssuesDao.java b/libs/dao/src/main/java/com/akto/dao/testing_run_findings/TestingRunIssuesDao.java index b3d507cd27..e43fef603e 100644 --- a/libs/dao/src/main/java/com/akto/dao/testing_run_findings/TestingRunIssuesDao.java +++ b/libs/dao/src/main/java/com/akto/dao/testing_run_findings/TestingRunIssuesDao.java @@ -64,10 +64,18 @@ public void createIndicesIfAbsent() { } public Map> getSeveritiesMapForCollections(){ + return getSeveritiesMapForCollections(null, true); + } + + public Map> getSeveritiesMapForCollections(Bson filter, boolean expandApiGroups){ Map> resultMap = new HashMap<>() ; List pipeline = new ArrayList<>(); pipeline.add(Aggregates.match(Filters.eq(TestingRunIssues.TEST_RUN_ISSUES_STATUS, "OPEN"))); + if(filter!=null){ + pipeline.add(Aggregates.match(filter)); + } + try { List collectionIds = UsersCollectionsList.getCollectionsIdForUser(Context.userId.get(), Context.accountId.get()); if(collectionIds != null) { @@ -76,12 +84,16 @@ public Map> getSeveritiesMapForCollections(){ } catch(Exception e){ } - UnwindOptions unwindOptions = new UnwindOptions(); - unwindOptions.preserveNullAndEmptyArrays(false); - pipeline.add(Aggregates.unwind("$collectionIds", unwindOptions)); + BasicDBObject groupedId = new BasicDBObject("apiCollectionId", "$_id.apiInfoKey.apiCollectionId") + .append("severity", "$severity"); - BasicDBObject groupedId = new BasicDBObject("apiCollectionId", "$collectionIds") - .append("severity", "$severity") ; + if (expandApiGroups) { + UnwindOptions unwindOptions = new UnwindOptions(); + unwindOptions.preserveNullAndEmptyArrays(false); + pipeline.add(Aggregates.unwind("$collectionIds", unwindOptions)); + groupedId = new BasicDBObject("apiCollectionId", "$collectionIds") + .append("severity", "$severity"); + } pipeline.add(Aggregates.group(groupedId, Accumulators.sum("count", 1))); From 2cc072e4518459f961d5523db804bd64615a0c10 Mon Sep 17 00:00:00 2001 From: Umesh Kumar <166806589+TangoBeeAkto@users.noreply.github.com> Date: Mon, 30 Dec 2024 23:03:57 +0530 Subject: [PATCH 16/40] showing title even if search value is empty --- .../dashboard/components/shared/DropdownSearch.jsx | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/shared/DropdownSearch.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/shared/DropdownSearch.jsx index adf632e989..1f5809c392 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/shared/DropdownSearch.jsx +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/shared/DropdownSearch.jsx @@ -65,9 +65,19 @@ function DropdownSearch(props) { setLoading(true); } + const defaultSliceValue = sliceMaxVal || 20 + setTimeout(() => { if (value === '' && selectedOptions.length === 0) { - setOptions(deselectedOptions); + const options = deselectedOptions.slice(0, defaultSliceValue); + const title = options.length >= defaultSliceValue + ? `Showing ${options.length} result${func.addPlurality(options.length)} only. (type more to refine results)` + : "Showing all results"; + const nestedOptions = [{ + title: title, + options: options + }] + setOptions(nestedOptions); setLoading(false); return; } @@ -86,7 +96,6 @@ function DropdownSearch(props) { }); }); }else{ - const defaultSliceValue = sliceMaxVal || 20 resultOptions = deselectedOptions.filter((option) => option[searchKey].match(filterRegex) ).slice(0, defaultSliceValue); From 1795d1c0dc3c4e82f298c85da7510f1dcf20c63c Mon Sep 17 00:00:00 2001 From: Ark2307 Date: Mon, 30 Dec 2024 23:52:39 +0530 Subject: [PATCH 17/40] Handled for issue status --- .../src/main/java/com/akto/testing/Utils.java | 27 +++++++++++++------ 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/libs/utils/src/main/java/com/akto/testing/Utils.java b/libs/utils/src/main/java/com/akto/testing/Utils.java index 3999cce874..77913061df 100644 --- a/libs/utils/src/main/java/com/akto/testing/Utils.java +++ b/libs/utils/src/main/java/com/akto/testing/Utils.java @@ -14,6 +14,7 @@ import org.bson.conversions.Bson; import org.bson.types.ObjectId; import com.akto.dao.testing.TestingRunResultDao; +import com.akto.dao.testing_run_findings.TestingRunIssuesDao; import com.akto.dto.ApiInfo.ApiInfoKey; import com.akto.dto.CollectionConditions.ConditionsType; import com.akto.dto.OriginalHttpRequest; @@ -21,6 +22,8 @@ import com.akto.dto.test_editor.DataOperandsFilterResponse; import com.akto.dto.test_editor.FilterNode; import com.akto.dto.test_editor.Util; +import com.akto.dto.test_run_findings.TestingIssuesId; +import com.akto.dto.test_run_findings.TestingRunIssues; import com.akto.dto.testing.TestingRunResult; import com.akto.dto.testing.WorkflowUpdatedSampleData; import com.akto.dto.type.RequestTemplate; @@ -28,7 +31,10 @@ import com.akto.log.LoggerMaker.LogDb; import com.akto.test_editor.filter.Filter; import com.akto.test_editor.filter.data_operands_impl.ValidationResult; +import com.akto.testing_utils.TestingUtils; +import com.akto.util.Constants; import com.akto.util.JSONUtils; +import com.akto.util.enums.GlobalEnums; import com.akto.util.enums.GlobalEnums.Severity; import com.mongodb.BasicDBList; import com.mongodb.BasicDBObject; @@ -430,23 +436,28 @@ public static Map finalCountIssuesMap(ObjectId testingRunResult countIssuesMap.put(Severity.MEDIUM.toString(), 0); countIssuesMap.put(Severity.LOW.toString(), 0); - Bson projection = Projections.computed("confidence", Projections.computed("$first", "$testResults.confidence")); + Bson projection = Projections.include(TestingRunResult.API_INFO_KEY, TestingRunResult.TEST_SUB_TYPE); Bson filterQ = Filters.and( Filters.eq(TestingRunResult.TEST_RUN_RESULT_SUMMARY_ID, testingRunResultSummaryId), Filters.eq(TestingRunResult.VULNERABLE, true) ); - BasicDBObject groupId = new BasicDBObject("_id", "$confidence"); + List allVulResults = TestingRunResultDao.instance.findAll(filterQ, projection); + + Map testingIssuesIdsMap = TestingUtils. + listOfIssuesIdsFromTestingRunResults(allVulResults, true, false); + + Bson inQuery = Filters.and(Filters.in(Constants.ID, testingIssuesIdsMap.keySet().toArray()), Filters.eq(TestingRunIssues.TEST_RUN_ISSUES_STATUS, GlobalEnums.TestRunIssueStatus.OPEN)); List pipeline = new ArrayList<>(); + pipeline.add(Aggregates.match(inQuery)); + pipeline.add(Aggregates.group( + "$" + TestingRunIssues.KEY_SEVERITY, Accumulators.sum("count",1) + )); - pipeline.add(Aggregates.match(filterQ)); - pipeline.add(Aggregates.project(projection)); - pipeline.add(Aggregates.group(groupId, Accumulators.sum("count", 1))); - MongoCursor cursor = TestingRunResultDao.instance.getMCollection().aggregate(pipeline, BasicDBObject.class).cursor(); + MongoCursor cursor = TestingRunIssuesDao.instance.getMCollection().aggregate(pipeline, BasicDBObject.class).cursor(); while(cursor.hasNext()){ BasicDBObject dbObject = cursor.next(); - BasicDBObject objectId = (BasicDBObject) dbObject.get("_id"); - String id = objectId.getString("_id"); + String id = dbObject.getString("_id"); int val = dbObject.getInt("count"); countIssuesMap.put(id, val); From ff35d73547bb4f5b396c4bc92bd58a08e4da60b9 Mon Sep 17 00:00:00 2001 From: Umesh Kumar <166806589+TangoBeeAkto@users.noreply.github.com> Date: Tue, 31 Dec 2024 09:28:24 +0530 Subject: [PATCH 18/40] clean code --- .../web/src/apps/dashboard/components/shared/DropdownSearch.jsx | 2 +- .../dashboard/pages/observe/api_collections/ApiGroupModal.jsx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/shared/DropdownSearch.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/shared/DropdownSearch.jsx index 1f5809c392..37bbc3c538 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/shared/DropdownSearch.jsx +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/shared/DropdownSearch.jsx @@ -70,7 +70,7 @@ function DropdownSearch(props) { setTimeout(() => { if (value === '' && selectedOptions.length === 0) { const options = deselectedOptions.slice(0, defaultSliceValue); - const title = options.length >= defaultSliceValue + const title = options.length > defaultSliceValue ? `Showing ${options.length} result${func.addPlurality(options.length)} only. (type more to refine results)` : "Showing all results"; const nestedOptions = [{ diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/ApiGroupModal.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/ApiGroupModal.jsx index 3cf756e12e..d809345cb4 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/ApiGroupModal.jsx +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/ApiGroupModal.jsx @@ -67,7 +67,7 @@ function ApiGroupModal(props){ label="Select API group" placeholder="Select API group" optionsList={ - allCollections.filter((x) => { return (x.type === 'API_GROUP' && x.deactivated === false) }).map((x) => { + activatedGroupCollections.map((x) => { return { label: x.displayName, value: x.displayName From 3d27baa844d983a732c00470a5ccead861778206 Mon Sep 17 00:00:00 2001 From: notshivansh Date: Tue, 31 Dec 2024 11:43:04 +0530 Subject: [PATCH 19/40] revert functionality --- .../pages/testing/vulnerability_report/VulnerabilityReport.jsx | 1 - 1 file changed, 1 deletion(-) diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/vulnerability_report/VulnerabilityReport.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/vulnerability_report/VulnerabilityReport.jsx index 19bdcdde4c..c145b6be07 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/vulnerability_report/VulnerabilityReport.jsx +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/vulnerability_report/VulnerabilityReport.jsx @@ -140,7 +140,6 @@ const VulnerabilityReport = () => { }) issuesFilters.filterCollectionsId = issuesFilters.filterCollectionsId.filter((x) => x !== undefined) - setCurrentDate(func.getFormattedDate(Date.now()/1000)) await issuesApi.fetchVulnerableTestingRunResultsFromIssues(issuesFilters, issueIdsFromFilter, resultsCount).then(resp => { vulnerableTestingRunResults = [...vulnerableTestingRunResults, ...resp.testingRunResults] From bc842467d00a4cc4a139c9f9efd4f3656a19d544 Mon Sep 17 00:00:00 2001 From: Ark2307 Date: Tue, 31 Dec 2024 12:31:32 +0530 Subject: [PATCH 20/40] Fixing count of apis --- .../com/akto/action/ApiCollectionsAction.java | 2 +- .../java/com/akto/dao/SingleTypeInfoDao.java | 7 ++++ .../java/com/akto/dto/ApiCollectionUsers.java | 38 +++++++++++++++++++ 3 files changed, 46 insertions(+), 1 deletion(-) diff --git a/apps/dashboard/src/main/java/com/akto/action/ApiCollectionsAction.java b/apps/dashboard/src/main/java/com/akto/action/ApiCollectionsAction.java index efb90f2876..7d9da3402d 100644 --- a/apps/dashboard/src/main/java/com/akto/action/ApiCollectionsAction.java +++ b/apps/dashboard/src/main/java/com/akto/action/ApiCollectionsAction.java @@ -527,7 +527,7 @@ public String getEndpointsListFromConditions() { InventoryAction inventoryAction = new InventoryAction(); inventoryAction.attachAPIInfoListInResponse(list,-1); this.setResponse(inventoryAction.getResponse()); - response.put("apiCount", ApiCollectionUsers.getApisCountFromConditions(conditions, new ArrayList<>(deactivatedCollections))); + response.put("apiCount", ApiCollectionUsers.getApisCountFromConditionsWithStis(conditions, new ArrayList<>(deactivatedCollections))); return SUCCESS.toUpperCase(); } public String getEndpointsFromConditions(){ diff --git a/libs/dao/src/main/java/com/akto/dao/SingleTypeInfoDao.java b/libs/dao/src/main/java/com/akto/dao/SingleTypeInfoDao.java index d848b96565..83392514cf 100644 --- a/libs/dao/src/main/java/com/akto/dao/SingleTypeInfoDao.java +++ b/libs/dao/src/main/java/com/akto/dao/SingleTypeInfoDao.java @@ -828,4 +828,11 @@ public List fetchRecentEndpoints(int startTimestamp, int endTimes return endpoints; } + public static BasicDBObject getApiInfoGroupedId() { + BasicDBObject groupedId = + new BasicDBObject("apiCollectionId", "$apiCollectionId") + .append("url", "$url") + .append("method", "$method"); + return groupedId; + } } diff --git a/libs/dao/src/main/java/com/akto/dto/ApiCollectionUsers.java b/libs/dao/src/main/java/com/akto/dto/ApiCollectionUsers.java index 86b6aefbb2..3750ad7a28 100644 --- a/libs/dao/src/main/java/com/akto/dto/ApiCollectionUsers.java +++ b/libs/dao/src/main/java/com/akto/dto/ApiCollectionUsers.java @@ -27,6 +27,7 @@ import com.akto.dao.context.Context; import com.akto.dao.demo.VulnerableRequestForTemplateDao; import com.akto.dao.testing_run_findings.TestingRunIssuesDao; +import com.akto.dto.rbac.UsersCollectionsList; import com.akto.dto.testing.CustomTestingEndpoints; import com.akto.dto.testing.SensitiveDataEndpoints; import com.akto.dto.testing.TestingEndpoints; @@ -35,6 +36,7 @@ import com.akto.util.Constants; import com.mongodb.BasicDBObject; import com.mongodb.client.MongoCursor; +import com.mongodb.client.model.Accumulators; import com.mongodb.client.model.Aggregates; import com.mongodb.client.model.Filters; import com.mongodb.client.model.Projections; @@ -94,6 +96,42 @@ public static int getApisCountFromConditions(List conditions, return (int) ApiInfoDao.instance.count(apiInfoFilters); } + public static int getApisCountFromConditionsWithStis(List conditions, List deactivatedCollections){ + if(conditions == null || conditions.isEmpty()){ + return 0; + } + + Bson stiFiltes = getFilters(conditions, CollectionType.ApiCollectionId); + List pipeLine = new ArrayList<>(); + pipeLine.add(Aggregates.match(stiFiltes)); + + try { + List collectionIds = UsersCollectionsList.getCollectionsIdForUser(Context.userId.get(), Context.accountId.get()); + if(collectionIds != null) { + pipeLine.add(Aggregates.match(Filters.in(SingleTypeInfo._COLLECTION_IDS, collectionIds))); + } + } catch(Exception e){ + } + + pipeLine.add(Aggregates.match(Filters.and( + Filters.eq(SingleTypeInfo._RESPONSE_CODE, -1), + Filters.eq(SingleTypeInfo._IS_HEADER, true) + ))); + BasicDBObject groupedId = SingleTypeInfoDao.getApiInfoGroupedId(); + pipeLine.add(Aggregates.group(groupedId, Accumulators.sum("count", 1))); + pipeLine.add(Aggregates.count("finalCount")); + + int ansCount = 0; + + MongoCursor countCursor = SingleTypeInfoDao.instance.getMCollection().aggregate(pipeLine, BasicDBObject.class).cursor(); + while(countCursor.hasNext()){ + BasicDBObject dbObject = countCursor.next(); + ansCount = dbObject.getInt("finalCount"); + } + + return ansCount; + } + public static void updateApiCollection(List conditions, int id) { ApiCollectionsDao.instance.updateOne( From 7f86f6310bb7dce29b3b114e0d6792d0c9cd4cf1 Mon Sep 17 00:00:00 2001 From: notshivansh Date: Tue, 31 Dec 2024 12:39:01 +0530 Subject: [PATCH 21/40] add more logs for api collection count --- .../main/java/com/akto/action/ApiCollectionsAction.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/apps/dashboard/src/main/java/com/akto/action/ApiCollectionsAction.java b/apps/dashboard/src/main/java/com/akto/action/ApiCollectionsAction.java index efb90f2876..1cf5879057 100644 --- a/apps/dashboard/src/main/java/com/akto/action/ApiCollectionsAction.java +++ b/apps/dashboard/src/main/java/com/akto/action/ApiCollectionsAction.java @@ -93,16 +93,17 @@ public List fillApiCollectionsUrlCount(List apiCol if(apiCollectionWithCond.getConditions() != null && apiCollectionWithCond.getConditions().get(0) != null){ if(apiCollectionWithCond.getConditions().get(0).getType().equals(TestingEndpoints.Type.CUSTOM)){ CustomTestingEndpoints testingEndpoints = (CustomTestingEndpoints) apiCollectionWithCond.getConditions().get(0); - if(testingEndpoints.getApisList() != null && !testingEndpoints.getApisList().isEmpty()){ + if (testingEndpoints.getApisList() != null && !testingEndpoints.getApisList().isEmpty()) { conditionsCount = testingEndpoints.getApisList().size(); + loggerMaker.infoAndAddToDb("fillApiCollectionsUrlCount collection: " + apiCollectionWithCond.getDisplayName() + " count: " + conditionsCount); } } } } - if(conditionsCount != 0){ - count = conditionsCount; - }else if (count == null) { + if (conditionsCount != 0) { + count = conditionsCount; + } else if (count == null) { count = fallbackCount; } apiCollection.setUrlsCount(count); From d2aca4494bd02c5fb710af442bc1054fb22d972e Mon Sep 17 00:00:00 2001 From: notshivansh Date: Tue, 31 Dec 2024 13:29:13 +0530 Subject: [PATCH 22/40] show all issue data --- .../web/src/apps/dashboard/pages/dashboard/HomeDashboard.jsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/dashboard/HomeDashboard.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/dashboard/HomeDashboard.jsx index d0611dec34..b284f6c9a6 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/dashboard/HomeDashboard.jsx +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/dashboard/HomeDashboard.jsx @@ -135,7 +135,7 @@ function HomeDashboard() { api.findTotalIssues(startTimestamp, endTimestamp), api.fetchApiStats(startTimestamp, endTimestamp), api.fetchEndpointsCount(startTimestamp, endTimestamp), - testingApi.fetchSeverityInfoForIssues({ startEpoch: startTimestamp }, [], endTimestamp) + testingApi.fetchSeverityInfoForIssues({}, [], 0) ]; let results = await Promise.allSettled(apiPromises); From e16c5421d0ea68a33f08df2ae53c293e11c95578 Mon Sep 17 00:00:00 2001 From: notshivansh Date: Tue, 31 Dec 2024 14:13:18 +0530 Subject: [PATCH 23/40] add inactive field to akto data types --- .../com/akto/action/CustomDataTypeAction.java | 5 +++-- .../pages/observe/data_types/DataTypes.jsx | 10 ++++------ .../pages/observe/data_types/transform.js | 2 +- .../java/com/akto/dao/SingleTypeInfoDao.java | 18 ++++++++++++++++- .../main/java/com/akto/dto/AktoDataType.java | 20 +++++++++++++++++++ 5 files changed, 45 insertions(+), 10 deletions(-) diff --git a/apps/dashboard/src/main/java/com/akto/action/CustomDataTypeAction.java b/apps/dashboard/src/main/java/com/akto/action/CustomDataTypeAction.java index 1e7f055036..a54ec464a8 100644 --- a/apps/dashboard/src/main/java/com/akto/action/CustomDataTypeAction.java +++ b/apps/dashboard/src/main/java/com/akto/action/CustomDataTypeAction.java @@ -248,7 +248,7 @@ public String execute() { private AktoDataType aktoDataType; - public String saveAktoDataType(){ + public String saveAktoDataType() { aktoDataType = AktoDataTypeDao.instance.findOne("name",name); if(aktoDataType==null){ @@ -305,7 +305,8 @@ public String saveAktoDataType(){ Updates.set(AktoDataType.KEY_CONDITIONS, keyConditions), Updates.set(AktoDataType.VALUE_CONDITIONS, valueConditions), Updates.set(AktoDataType.OPERATOR, mainOperator), - Updates.set(AktoDataType.DATA_TYPE_PRIORITY, dataTypePriority) + Updates.set(AktoDataType.DATA_TYPE_PRIORITY, dataTypePriority), + Updates.set(AktoDataType._INACTIVE, !active) ), options ); diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/data_types/DataTypes.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/data_types/DataTypes.jsx index 3fa380adc2..500234cafc 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/data_types/DataTypes.jsx +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/data_types/DataTypes.jsx @@ -175,7 +175,7 @@ function DataTypes() { valueOperator: currState.valueConditions.operator, dataTypePriority: currState?.priority ? currState.priority.toUpperCase() : "", ...transform.convertToSensitiveData(currState.sensitiveState), - + active: JSON.parse(currState.active) } api.saveAktoDataType(obj).then((response) => { func.setToast(true, false, "Data type updated successfully"); @@ -250,11 +250,9 @@ function DataTypes() { requiredIndicator={true} {...errorMessage.length > 0 ? {error: errorMessage} : {}} /> - {currState.dataType === 'Custom' ? - { handleChange({ active: val }) }} - initial={currState.active} label="Active" /> - : null} + { handleChange({ active: val }) }} + initial={currState.active} label="Active" /> sensitiveSubTypeNames() { // AKTO sensitive for (SingleTypeInfo.SubType subType: SingleTypeInfo.subTypeMap.values()) { if (subType.isSensitiveAlways()) { + AktoDataType dt = SingleTypeInfo.getAktoDataTypeMap(Context.accountId.get()).get(subType.getName()); + if (dt != null && !dt.getActive()) { + continue; + } sensitiveSubTypes.add(subType.getName()); } } // Custom data type sensitive for (CustomDataType customDataType: SingleTypeInfo.getCustomDataTypeMap(Context.accountId.get()).values()) { - if (customDataType.isSensitiveAlways()) { + if (customDataType.isSensitiveAlways() && customDataType.isActive()){ sensitiveSubTypes.add(customDataType.getName()); } } @@ -229,6 +234,11 @@ public List sensitiveSubTypeInRequestNames() { if (subType.isSensitiveAlways() || subType.getSensitivePosition().contains(SingleTypeInfo.Position.REQUEST_HEADER) || subType.getSensitivePosition().contains(SingleTypeInfo.Position.REQUEST_PAYLOAD)) { + + AktoDataType dt = SingleTypeInfo.getAktoDataTypeMap(Context.accountId.get()).get(subType.getName()); + if (dt != null && !dt.getActive()) { + continue; + } sensitiveInRequest.add(subType.getName()); } } @@ -252,6 +262,12 @@ public List sensitiveSubTypeInResponseNames() { if (subType.isSensitiveAlways() || subType.getSensitivePosition().contains(SingleTypeInfo.Position.RESPONSE_HEADER) || subType.getSensitivePosition().contains(SingleTypeInfo.Position.RESPONSE_PAYLOAD)) { + + AktoDataType dt = SingleTypeInfo.getAktoDataTypeMap(Context.accountId.get()).get(subType.getName()); + if (dt != null && !dt.getActive()) { + continue; + } + sensitiveInResponse.add(subType.getName()); } } diff --git a/libs/dao/src/main/java/com/akto/dto/AktoDataType.java b/libs/dao/src/main/java/com/akto/dto/AktoDataType.java index 5d3774487f..f5c4c7d2ce 100644 --- a/libs/dao/src/main/java/com/akto/dto/AktoDataType.java +++ b/libs/dao/src/main/java/com/akto/dto/AktoDataType.java @@ -32,6 +32,9 @@ public class AktoDataType { public static final String OPERATOR = "operator"; Conditions.Operator operator; + public static final String _INACTIVE = "inactive"; + private boolean inactive; + public AktoDataType() { } public AktoDataType(String name, boolean sensitiveAlways, List sensitivePosition,int timestamp, IgnoreData ignoreData, boolean redacted, boolean sampleDataFixed) { @@ -152,4 +155,21 @@ public boolean validate(Object value, Object key) { public boolean validateRaw(Object value, Object key) throws Exception { return CustomDataType.validateRawUtility(value, key, this.keyConditions, this.valueConditions, this.operator); } + + /* + * Default is inactive:false + */ + public boolean getInactive() { + return inactive; + } + + public void setInactive(boolean inactive) { + this.inactive = inactive; + } + + // To send a field in frontend. + public boolean getActive() { + return !inactive; + } + } From b8f5006571b0b7188006ccf7969a8ae554369902 Mon Sep 17 00:00:00 2001 From: Ankush Jain Date: Tue, 31 Dec 2024 03:37:20 -0800 Subject: [PATCH 24/40] clean up resources --- .../src/main/java/com/akto/testing/Main.java | 3 +- .../java/com/akto/testing/TestExecutor.java | 146 +++++++++++++++++- .../akto/dto/testing/GenericTestResult.java | 5 +- .../akto/dto/testing/MultiExecTestResult.java | 24 +++ .../java/com/akto/dto/testing/TestResult.java | 6 + .../akto/dto/testing/TestingRunConfig.java | 22 ++- .../java/com/akto/dto/traffic/SampleData.java | 4 + .../java/com/akto/testing/ApiExecutor.java | 38 ++++- .../src/main/java/com/akto/testing/Utils.java | 18 ++- 9 files changed, 245 insertions(+), 21 deletions(-) diff --git a/apps/testing/src/main/java/com/akto/testing/Main.java b/apps/testing/src/main/java/com/akto/testing/Main.java index 01b1b4a0d6..d53a1ff096 100644 --- a/apps/testing/src/main/java/com/akto/testing/Main.java +++ b/apps/testing/src/main/java/com/akto/testing/Main.java @@ -220,7 +220,8 @@ private static void setTestingRunConfig(TestingRun testingRun, TestingRunResultS } public static void main(String[] args) throws InterruptedException { - String mongoURI = System.getenv("AKTO_MONGO_CONN"); + String mongoURI = "mongodb://localhost:27017/admini"; + System.out.println("here/......."); ReadPreference readPreference = ReadPreference.secondary(); WriteConcern writeConcern = WriteConcern.W1; DaoInit.init(new ConnectionString(mongoURI), readPreference, writeConcern); diff --git a/apps/testing/src/main/java/com/akto/testing/TestExecutor.java b/apps/testing/src/main/java/com/akto/testing/TestExecutor.java index a47f88d518..cc9d2e1c8a 100644 --- a/apps/testing/src/main/java/com/akto/testing/TestExecutor.java +++ b/apps/testing/src/main/java/com/akto/testing/TestExecutor.java @@ -5,6 +5,7 @@ import com.akto.dao.ActivitiesDao; import com.akto.dao.ApiInfoDao; import com.akto.dao.CustomAuthTypeDao; +import com.akto.dao.DependencyNodeDao; import com.akto.dao.context.Context; import com.akto.dao.test_editor.YamlTemplateDao; import com.akto.dao.testing.TestingRunResultDao; @@ -14,8 +15,13 @@ import com.akto.dto.ApiInfo; import com.akto.dto.ApiInfo.ApiInfoKey; import com.akto.dto.billing.SyncLimit; +import com.akto.dto.dependency_flow.KVPair; +import com.akto.dto.dependency_flow.ReplaceDetail; import com.akto.dto.CustomAuthType; +import com.akto.dto.DependencyNode; +import com.akto.dto.DependencyNode.ParamInfo; import com.akto.dto.OriginalHttpRequest; +import com.akto.dto.OriginalHttpResponse; import com.akto.dto.RawApi; import com.akto.dto.api_workflow.Graph; import com.akto.dto.test_editor.*; @@ -26,12 +32,14 @@ import com.akto.dto.type.RequestTemplate; import com.akto.dto.type.SingleTypeInfo; import com.akto.dto.type.URLMethods; +import com.akto.dto.type.URLMethods.Method; import com.akto.github.GithubUtils; import com.akto.log.LoggerMaker; import com.akto.log.LoggerMaker.LogDb; import com.akto.store.AuthMechanismStore; import com.akto.store.SampleMessageStore; import com.akto.store.TestingUtil; +import com.akto.test_editor.execution.Build; import com.akto.test_editor.execution.Executor; import com.akto.test_editor.execution.VariableResolver; import com.akto.test_editor.filter.data_operands_impl.ValidationResult; @@ -48,12 +56,15 @@ import com.mongodb.WriteConcern; import com.mongodb.client.model.*; +import org.bson.conversions.Bson; import org.bson.types.ObjectId; import org.json.JSONObject; import org.mortbay.util.ajax.JSON; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static com.akto.test_editor.execution.Build.modifyRequest; + import java.net.URI; import java.net.URISyntaxException; import java.util.*; @@ -212,7 +223,7 @@ public void apiWiseInit(TestingRun testingRun, ObjectId summaryId, boolean debug try { currentTime = Context.now(); loggerMaker.infoAndAddToDb("Starting StatusCodeAnalyser at: " + currentTime, LogDb.TESTING); - StatusCodeAnalyser.run(sampleDataMapForStatusCodeAnalyser, sampleMessageStore , authMechanismStore, testingRun.getTestingRunConfig(), hosts); + // StatusCodeAnalyser.run(sampleDataMapForStatusCodeAnalyser, sampleMessageStore , authMechanismStore, testingRun.getTestingRunConfig(), hosts); loggerMaker.infoAndAddToDb("Completing StatusCodeAnalyser in: " + (Context.now() - currentTime) + " at: " + Context.now(), LogDb.TESTING); } catch (Exception e) { loggerMaker.errorAndAddToDb("Error while running status code analyser " + e.getMessage(), LogDb.TESTING); @@ -649,6 +660,7 @@ public void startTestNew(ApiInfo.ApiInfoKey apiInfoKey, ObjectId testRunId, } insertResultsAndMakeIssues(testingRunResults, testRunResultSummaryId); + }else{ if(GetRunningTestsStatus.getRunningTests().getCurrentState(testRunId) != null && GetRunningTestsStatus.getRunningTests().getCurrentState(testRunId).equals(TestingRun.State.STOPPED)){ logger.info("Test stopped for id: " + testRunId.toString()); @@ -662,6 +674,102 @@ public void startTestNew(ApiInfo.ApiInfoKey apiInfoKey, ObjectId testRunId, } + private Map> cleanUpTestArtifacts(List testingRunResults, ApiInfoKey apiInfoKey, TestingUtil testingUtil) { + + Map> cleanedUpRequests = new HashMap<>(); + + for (TestingRunResult trr: testingRunResults) { + + for(GenericTestResult gtr: trr.getTestResults()) { + for(String message: gtr.getResponses()) { + if (message != null) { + RawApi rawApiToBeReplayed = RawApi.buildFromMessage(message); + if (rawApiToBeReplayed.getResponse().getStatusCode() >= 300) { + continue; + } + switch (apiInfoKey.getMethod()) { + case POST: + Bson filterQ = DependencyNodeDao.generateChildrenFilter(apiInfoKey.getApiCollectionId(), apiInfoKey.getUrl(), apiInfoKey.getMethod()); + Bson delFilterQ = Filters.and(filterQ, Filters.eq(DependencyNode.METHOD_REQ, Method.DELETE.name())); + List children = DependencyNodeDao.instance.findAll(filterQ); + + if (!children.isEmpty()) { + for(DependencyNode node: children) { + OriginalHttpRequest copiedReq = rawApiToBeReplayed.copy().getRequest(); + copiedReq.setUrl(node.getUrlReq()); + copiedReq.setMethod(node.getMethodReq()); + + Map> valuesMap = Build.getValuesMap(rawApiToBeReplayed.getResponse()); + + List samples = testingUtil.getSampleMessages().get(apiInfoKey); + if (samples == null || samples.isEmpty()) { + continue; + } else { + RawApi nextApi = RawApi.buildFromMessage(samples.get(0)); + nextApi.getRequest().setHeaders(copiedReq.getHeaders()); + + List kvPairs = new ArrayList<>(); + boolean fullReplace = true; + for(ParamInfo paramInfo: node.getParamInfos()) { + if (paramInfo.isHeader()) continue; + Object valueFromResponse = valuesMap.get(paramInfo.getResponseParam()); + + if (valueFromResponse == null) { + fullReplace = false; + + break; + } + KVPair kvPair = new KVPair(); + kvPairs.add(kvPair); + } + + if (!fullReplace) { + continue; + } + + ReplaceDetail replaceDetail = new ReplaceDetail(apiInfoKey.getApiCollectionId(), apiInfoKey.getUrl(), apiInfoKey.getMethod().name(), kvPairs); + modifyRequest(nextApi.getRequest(), replaceDetail); + System.out.println("====REQUEST===="); + System.out.println(nextApi.getRequest().getMethod() + " " + nextApi.getRequest().getUrl() + "?" + nextApi.getRequest().getQueryParams()); + System.out.println(nextApi.getRequest().getHeaders()); + System.out.println(nextApi.getRequest().getBody()); + System.out.println("====RESPONSE===="); + try { + OriginalHttpResponse nextResponse = ApiExecutor.sendRequest(nextApi.getRequest(), true, null, false, new ArrayList<>()); + System.out.println(nextApi.getResponse().getHeaders()); + System.out.println(nextResponse.getBody()); + + } catch (Exception e) { + e.printStackTrace(); + System.out.println("exception in sending api request for cleanup" + e.getMessage()); + } + } + } + } + + break; + case PUT: + + break; + case PATCH: + + break; + case DELETE: + + break; + + case GET: + default: + break; + } + } + } + } + } + + return cleanedUpRequests; + } + public boolean applyRunOnceCheck(ApiInfoKey apiInfoKey, TestConfig testConfig, ConcurrentHashMap subCategoryEndpointMap, Map apiInfoKeyToHostMap, String testSubCategory) { if (testConfig.getStrategy() == null || testConfig.getStrategy().getRunOnce() == null) { @@ -743,6 +851,26 @@ public TestingRunResult runTestNew(ApiInfo.ApiInfoKey apiInfoKey, ObjectId testR List customAuthTypes = testingUtil.getCustomAuthTypes(); // TestingUtil -> authMechanism // TestingConfig -> auth + + System.out.println("reached here Test Executor:855..."); + + if(testingRunConfig != null && testingRunConfig.getConfigsAdvancedSettings() != null && !testingRunConfig.getConfigsAdvancedSettings().isEmpty()){ + ApiExecutor.calculateFinalRequestFromAdvancedSettings(rawApi.getRequest(), testingRunConfig.getConfigsAdvancedSettings()); + // try { + // OriginalHttpResponse response = ApiExecutor.sendRequest(rawApi.getRequest(), false, testingRunConfig, false, new ArrayList<>()); + // if (response.getStatusCode() < 300) { + // rawApi = new RawApi(rawApi.getRequest(), response, ""); + // rawApi.fillOriginalMessage(0, 0, "", ""); + // } + // } catch (Exception e) { + // System.out.println("exception while making initial req: " + e.getMessage()); + // e.printStackTrace(); + // } + } + + + + com.akto.test_editor.execution.Executor executor = new Executor(); executor.overrideTestUrl(rawApi, testingRunConfig); YamlTestTemplate yamlTestTemplate = new YamlTestTemplate(apiInfoKey,filterNode, validatorNode, executorNode, @@ -775,11 +903,17 @@ public TestingRunResult runTestNew(ApiInfo.ApiInfoKey apiInfoKey, ObjectId testR int confidencePercentage = 100; - return new TestingRunResult( - testRunId, apiInfoKey, testSuperType, testSubType ,testResults.getTestResults(), - vulnerable,singleTypeInfos,confidencePercentage,startTime, - endTime, testRunResultSummaryId, testResults.getWorkflowTest(), testLogs - ); + + TestingRunResult ret = new TestingRunResult( + testRunId, apiInfoKey, testSuperType, testSubType ,testResults.getTestResults(), + vulnerable,singleTypeInfos,confidencePercentage,startTime, + endTime, testRunResultSummaryId, testResults.getWorkflowTest(), testLogs); + + // if (testingRunConfig.getCleanUp()) { + cleanUpTestArtifacts(Collections.singletonList(ret), apiInfoKey, testingUtil); + // } + + return ret; } public Confidence getConfidenceForTests(TestConfig testConfig, YamlTestTemplate template) { diff --git a/libs/dao/src/main/java/com/akto/dto/testing/GenericTestResult.java b/libs/dao/src/main/java/com/akto/dto/testing/GenericTestResult.java index 7e97f1be21..4443b36d2f 100644 --- a/libs/dao/src/main/java/com/akto/dto/testing/GenericTestResult.java +++ b/libs/dao/src/main/java/com/akto/dto/testing/GenericTestResult.java @@ -1,5 +1,7 @@ package com.akto.dto.testing; +import java.util.List; + import com.akto.dto.testing.TestResult.Confidence; public abstract class GenericTestResult { @@ -43,5 +45,6 @@ public String toString() { ", confidence='" + getConfidence() + "'" + "}"; } - + + public abstract List getResponses(); } diff --git a/libs/dao/src/main/java/com/akto/dto/testing/MultiExecTestResult.java b/libs/dao/src/main/java/com/akto/dto/testing/MultiExecTestResult.java index b6fc941b40..387854cce5 100644 --- a/libs/dao/src/main/java/com/akto/dto/testing/MultiExecTestResult.java +++ b/libs/dao/src/main/java/com/akto/dto/testing/MultiExecTestResult.java @@ -91,4 +91,28 @@ public List convertToExistingTestResult(TestingRunResult test return runResults; } + @Override + public List getResponses() { + List ret = new ArrayList<>(); + + Map nodeResultMap = this.getNodeResultMap(); + for (int i=0; i < this.executionOrder.size(); i++) { + String k = this.executionOrder.get(i); + NodeResult nodeRes = nodeResultMap.get(k); + List messageList = Arrays.asList(nodeRes.getMessage().split("\"request\": ")); + + for (int j = 1; j getResponses() { + return Collections.singletonList(message); + } } diff --git a/libs/dao/src/main/java/com/akto/dto/testing/TestingRunConfig.java b/libs/dao/src/main/java/com/akto/dto/testing/TestingRunConfig.java index 637b473760..49033ce3aa 100644 --- a/libs/dao/src/main/java/com/akto/dto/testing/TestingRunConfig.java +++ b/libs/dao/src/main/java/com/akto/dto/testing/TestingRunConfig.java @@ -22,17 +22,25 @@ public class TestingRunConfig { private String overriddenTestAppUrl; private List configsAdvancedSettings; + private boolean cleanUp; public TestingRunConfig() {} + + public TestingRunConfig(int id, Map> collectionWiseApiInfoKey, + List testSubCategoryList, + ObjectId authMechanismId, String overriddenTestAppUrl, String testRoleId) { + this(id, collectionWiseApiInfoKey, testSubCategoryList, authMechanismId, overriddenTestAppUrl, testRoleId, false); + } public TestingRunConfig(int id, Map> collectionWiseApiInfoKey, List testSubCategoryList, - ObjectId authMechanismId, String overriddenTestAppUrl, String testRoleId) { + ObjectId authMechanismId, String overriddenTestAppUrl, String testRoleId, boolean cleanUp) { this.id = id; this.collectionWiseApiInfoKey = collectionWiseApiInfoKey; this.testSubCategoryList = testSubCategoryList; this.authMechanismId = authMechanismId; this.overriddenTestAppUrl = overriddenTestAppUrl; this.testRoleId = testRoleId; + this.cleanUp = cleanUp; } public List getTestSubCategoryList() { @@ -99,7 +107,10 @@ public void rebaseOn(TestingRunConfig that) { if(this.testRoleId == null) { this.testRoleId = that.testRoleId; } + + this.cleanUp = that.cleanUp; } + public String getTestRoleId() { return testRoleId; @@ -115,4 +126,13 @@ public List getConfigsAdvancedSettings() { public void setConfigsAdvancedSettings(List configsAdvancedSettings) { this.configsAdvancedSettings = configsAdvancedSettings; } + + public boolean getCleanUp() { + return this.cleanUp; + } + + public void setCleanUp(boolean cleanUp) { + this.cleanUp = cleanUp; + } + } diff --git a/libs/dao/src/main/java/com/akto/dto/traffic/SampleData.java b/libs/dao/src/main/java/com/akto/dto/traffic/SampleData.java index 6fa212bf60..1d86e9f4f4 100644 --- a/libs/dao/src/main/java/com/akto/dto/traffic/SampleData.java +++ b/libs/dao/src/main/java/com/akto/dto/traffic/SampleData.java @@ -2,9 +2,13 @@ import java.util.Arrays; import java.util.List; + +import org.bson.codecs.pojo.annotations.BsonId; + import com.akto.util.Util; public class SampleData { + @BsonId Key id; public static final String SAMPLES = "samples"; diff --git a/libs/utils/src/main/java/com/akto/testing/ApiExecutor.java b/libs/utils/src/main/java/com/akto/testing/ApiExecutor.java index abe28cc3b9..02e15d7212 100644 --- a/libs/utils/src/main/java/com/akto/testing/ApiExecutor.java +++ b/libs/utils/src/main/java/com/akto/testing/ApiExecutor.java @@ -37,11 +37,14 @@ public class ApiExecutor { private static Map testScriptMap = new HashMap<>(); private static OriginalHttpResponse common(Request request, boolean followRedirects, boolean debug, List testLogs, boolean skipSSRFCheck, String requestProtocol) throws Exception { - + debug = true; Integer accountId = Context.accountId.get(); if (accountId != null) { int i = 0; boolean rateLimitHit = true; + + + while (RateLimitHandler.getInstance(accountId).shouldWait(request)) { if(rateLimitHit){ if (!(request.url().toString().contains("insertRuntimeLog") || request.url().toString().contains("insertTestingLog") || request.url().toString().contains("insertProtectionLog"))) { @@ -79,6 +82,9 @@ private static OriginalHttpResponse common(Request request, boolean followRedire } Call call = client.newCall(request); + + System.out.println("------ACTUAL REQ-------"); + System.out.println(request.headers()); Response response = null; String body; byte[] grpcBody = null; @@ -289,7 +295,6 @@ private static void addHeaders(OriginalHttpRequest request, Request.Builder buil public static OriginalHttpResponse sendRequest(OriginalHttpRequest request, boolean followRedirects, TestingRunConfig testingRunConfig, boolean debug, List testLogs, boolean skipSSRFCheck) throws Exception { // don't lowercase url because query params will change and will result in incorrect request - String url = prepareUrl(request, testingRunConfig); if (!(url.contains("insertRuntimeLog") || url.contains("insertTestingLog") || url.contains("insertProtectionLog"))) { @@ -308,9 +313,6 @@ public static OriginalHttpResponse sendRequest(OriginalHttpRequest request, bool boolean executeScript = testingRunConfig != null; //calculateHashAndAddAuth(request, executeScript); - if(testingRunConfig != null && testingRunConfig.getConfigsAdvancedSettings() != null && !testingRunConfig.getConfigsAdvancedSettings().isEmpty()){ - calculateFinalRequestFromAdvancedSettings(request, testingRunConfig.getConfigsAdvancedSettings()); - } OriginalHttpResponse response = null; HostValidator.validate(url); @@ -335,7 +337,9 @@ public static OriginalHttpResponse sendRequest(OriginalHttpRequest request, bool if (!(url.contains("insertRuntimeLog") || url.contains("insertTestingLog") || url.contains("insertProtectionLog"))) { loggerMaker.infoAndAddToDb("Received response from: " + url, LogDb.TESTING); } - + System.out.println("====RESPONSE===="); + System.out.println(response.getHeaders()); + System.out.println(response.getBody()); return response; } public static OriginalHttpResponse sendRequest(OriginalHttpRequest request, boolean followRedirects, TestingRunConfig testingRunConfig, boolean debug, List testLogs) throws Exception { @@ -388,7 +392,8 @@ public void writeTo(BufferedSink sink) throws IOException { } - private static void calculateFinalRequestFromAdvancedSettings(OriginalHttpRequest originalHttpRequest, List advancedSettings){ + public static void calculateFinalRequestFromAdvancedSettings(OriginalHttpRequest originalHttpRequest, List advancedSettings){ + System.out.println("in calculateFinalRequestFromAdvancedSettings..."); Map> headerConditions = new HashMap<>(); Map> payloadConditions = new HashMap<>(); @@ -407,14 +412,18 @@ private static void calculateFinalRequestFromAdvancedSettings(OriginalHttpReques headerConditions.getOrDefault(TestEditorEnums.TerminalExecutorDataOperands.DELETE_HEADER.name(), emptyList) ); + System.out.println("modifyBodyOperations calculateFinalRequestFromAdvancedSettings..."); Utils.modifyBodyOperations(originalHttpRequest, payloadConditions.getOrDefault(TestEditorEnums.NonTerminalExecutorDataOperands.MODIFY_BODY_PARAM.name(), emptyList), payloadConditions.getOrDefault(TestEditorEnums.NonTerminalExecutorDataOperands.ADD_BODY_PARAM.name(), emptyList), payloadConditions.getOrDefault(TestEditorEnums.TerminalExecutorDataOperands.DELETE_BODY_PARAM.name(), emptyList) ); + System.out.println("modifyBodyOperations completed calculateFinalRequestFromAdvancedSettings..."); + } private static OriginalHttpResponse sendWithRequestBody(OriginalHttpRequest request, Request.Builder builder, boolean followRedirects, boolean debug, List testLogs, boolean skipSSRFCheck, String requestProtocol) throws Exception { + Map> headers = request.getHeaders(); if (headers == null) { headers = new HashMap<>(); @@ -465,6 +474,15 @@ private static OriginalHttpResponse sendWithRequestBody(OriginalHttpRequest requ } if (payload == null) payload = ""; + + try { + System.out.println("payloadStr: " + payload); + payload = Utils.replaceVariables(payload, new HashMap<>(), false, false); + System.out.println("payloadStrFinal: " + payload); + + } catch (Exception e) { + System.out.println("failed to replace vars in payload: " + e.getMessage()); + } if (body == null) {// body not created by GRPC block yet if (request.getHeaders().containsKey("charset")) { body = RequestBody.create(payload, null); @@ -473,6 +491,12 @@ private static OriginalHttpResponse sendWithRequestBody(OriginalHttpRequest requ body = RequestBody.create(payload, MediaType.parse(contentType)); } } + + System.out.println("====REQUEST===="); + System.out.println(request.getMethod() + " " + request.getUrl() + "?" + request.getQueryParams()); + System.out.println(request.getHeaders()); + System.out.println(payload); + builder = builder.method(request.getMethod(), body); Request okHttpRequest = builder.build(); return common(okHttpRequest, followRedirects, debug, testLogs, skipSSRFCheck, requestProtocol); diff --git a/libs/utils/src/main/java/com/akto/testing/Utils.java b/libs/utils/src/main/java/com/akto/testing/Utils.java index 3a2caaa710..8bb2cffcfa 100644 --- a/libs/utils/src/main/java/com/akto/testing/Utils.java +++ b/libs/utils/src/main/java/com/akto/testing/Utils.java @@ -7,6 +7,7 @@ import java.util.Map; import java.util.Objects; import java.util.Set; +import java.util.UUID; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -199,11 +200,16 @@ public static String replaceVariables(String payload, Map values if (key == null) continue; Object obj = valuesMap.get(key); if (obj == null) { - loggerMaker.errorAndAddToDb("couldn't find: " + key, LogDb.TESTING); - if(shouldThrowException){ - throw new Exception("Couldn't find " + key); - }else{ - continue; + if (key.toLowerCase().startsWith("x0.unique_")) { + String suffix = key.substring(key.toLowerCase().indexOf("_")+1); + obj = suffix+"_"+System.nanoTime(); + } else { + loggerMaker.errorAndAddToDb("couldn't find: " + key, LogDb.TESTING); + if(shouldThrowException){ + throw new Exception("Couldn't find " + key); + }else{ + continue; + } } } String val = obj.toString(); @@ -336,6 +342,7 @@ private static ValidationResult validate(FilterNode node, RawApi rawApi, RawApi } public static void modifyBodyOperations(OriginalHttpRequest httpRequest, List modifyOperations, List addOperations, List deleteOperations){ + System.out.println("inside modifyBodyOperations"); String oldReqBody = httpRequest.getBody(); if(oldReqBody == null || oldReqBody.isEmpty()){ return ; @@ -378,6 +385,7 @@ public static void modifyBodyOperations(OriginalHttpRequest httpRequest, List Date: Wed, 1 Jan 2025 02:57:15 -0800 Subject: [PATCH 25/40] ask for user info --- apps/dashboard/web/pages/login.jsp | 24 +++++- .../components/WelcomeBackDetailsModal.jsx | 81 ++++++++++++++++--- .../src/apps/dashboard/pages/Dashboard.jsx | 4 +- 3 files changed, 97 insertions(+), 12 deletions(-) diff --git a/apps/dashboard/web/pages/login.jsp b/apps/dashboard/web/pages/login.jsp index 829bd34315..d46db74e1a 100644 --- a/apps/dashboard/web/pages/login.jsp +++ b/apps/dashboard/web/pages/login.jsp @@ -109,7 +109,19 @@ mixpanel.init('c403d0b00353cc31d7e33d68dc778806', { debug: false, ignore_dnt: true }); let distinct_id = window.USER_NAME + '_' + (window.IS_SAAS === 'true' ? "SAAS" : window.DASHBOARD_MODE); mixpanel.identify(distinct_id); - mixpanel.people.set({ "$email": window.USER_NAME, "$account Name": window.ACCOUNT_NAME }); + let mixpanelUserProps = { "$email": window.USER_NAME, "$account Name": window.ACCOUNT_NAME } + + if (window.USER_FULL_NAME) { + mixpanelUserProps["name"] = window.USER_FULL_NAME + mixpanelUserProps["$name"] = window.USER_FULL_NAME + } + + if (window.ORGANIZATION_NAME) { + mixpanelUserProps["company"] = window.ORGANIZATION_NAME + mixpanelUserProps["$company"] = window.ORGANIZATION_NAME + } + + mixpanel.people.set(mixpanelUserProps); mixpanel.register({ 'email': window.USER_NAME, @@ -128,6 +140,16 @@ data_ingestion_paused: window.USAGE_PAUSED?.dataIngestion === 'true', test_runs_paused: window.USAGE_PAUSED?.testRuns === 'true' }; + + if (window.USER_FULL_NAME) { + window.intercomSettings["name"] = window.USER_FULL_NAME + } + + if (window.ORGANIZATION_NAME && window.USER_NAME.indexOf("@")> 0) { + let company_id = window.USER_NAME.split("@")[1]; + window.intercomSettings["company"] = {name: window.ORGANIZATION_NAME, company_id} + } + } else if (window.location.href.includes('check-inbox') || window.location.href.includes('business-email')) { (function () { var w = window; var ic = w.Intercom; if (typeof ic === "function") { ic('reattach_activator'); ic('update', w.intercomSettings); } else { var d = document; var i = function () { i.c(arguments); }; i.q = []; i.c = function (args) { i.q.push(args); }; w.Intercom = i; var l = function () { var s = d.createElement('script'); s.type = 'text/javascript'; s.async = true; s.src = 'https://widget.intercom.io/widget/e9w9wkdk'; var x = d.getElementsByTagName('script')[0]; x.parentNode.insertBefore(s, x); }; if (document.readyState === 'complete') { l(); } else if (w.attachEvent) { w.attachEvent('onload', l); } else { w.addEventListener('load', l, false); } } })(); window.intercomSettings = { diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/WelcomeBackDetailsModal.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/WelcomeBackDetailsModal.jsx index eb8106eba1..933ec3b680 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/WelcomeBackDetailsModal.jsx +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/WelcomeBackDetailsModal.jsx @@ -4,10 +4,52 @@ import func from '@/util/func' import homeRequests from "../pages/home/api" const WelcomeBackDetailsModal = ({ isAdmin }) => { - const [modalToggle, setModalToggle] = useState(true) - const [username, setUsername] = useState(window.USER_FULL_NAME || "") - const [organization, setOrganization] = useState(window.ORGANIZATION_NAME || "") + const extractEmailDetails = (email) => { + // Define the regex pattern + const pattern = /^(.*?)@([\w.-]+)\.[a-z]{2,}$/; + + // Match the regex pattern + const match = email.match(pattern); + + if (match) { + let rawUsername = match[1]; // Extract username + let mailserver = match[2]; // Extract mailserver (including subdomains) + + let username = rawUsername + .split(/[^a-zA-Z]+/) // Split by any non-alphabet character + .filter(Boolean) // Remove empty segments + .map(segment => segment.charAt(0).toUpperCase() + segment.slice(1)) // Capitalize each segment + .join(' '); // Join segments with a space + + mailserver = mailserver.charAt(0).toUpperCase() + mailserver.slice(1); + + return { username, mailserver }; + } else { + return { error: "Invalid email format" }; + } + }; + + function suggestOrgName() { + if (window.ORGANIZATION_NAME?.length > 0) { + return window.ORGANIZATION_NAME + } else { + return extractEmailDetails(window.USER_NAME)?.mailserver || "" + } + } + + function suggestFullName() { + if (window.USER_FULL_NAME?.length > 0) { + return window.USER_FULL_NAME + } else { + return extractEmailDetails(window.USER_NAME)?.username || "" + } + } + + const [modalToggle, setModalToggle] = useState(!window.localStorage.getItem("username")) + + const [username, setUsername] = useState(suggestFullName()) + const [organization, setOrganization] = useState(suggestOrgName()) const handleWelcomeBackDetails = async () => { @@ -18,6 +60,27 @@ const WelcomeBackDetailsModal = ({ isAdmin }) => { return } + if (window.Intercom) { + let updateObj = {name: username} + if (window.USER_NAME.indexOf("@")> 0) { + updateObj["company"] = {name: organization, company_id: window.USER_NAME.split("@")[1]} + } + window.Intercom("update", updateObj) + } + + if (window.mixpanel) { + mixpanelUserProps = { + "name": username, + "company": organization, + "$name": username, + "$company": organization + } + window.mixpanel.people.set(mixpanelUserProps); + } + + window.localStorage.setItem("username", username) + window.localStorage.setItem("organization", organization) + homeRequests.updateUsernameAndOrganization(username, organization).then((resp) => { try { setModalToggle(false) @@ -40,25 +103,25 @@ const WelcomeBackDetailsModal = ({ isAdmin }) => { - Tell us more about yourself. + Please tell us more... {username.length}/24 + (username.length > 20) && {username.length}/40 )} /> { isAdmin && { autoComplete="off" maxLength={"24"} suffix={( - {organization.length}/24 + (organization.length) > 20 && {organization.length}/40 )} /> } diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/Dashboard.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/Dashboard.jsx index 58e5824026..094bbf84eb 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/Dashboard.jsx +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/Dashboard.jsx @@ -172,13 +172,13 @@ function Dashboard() { },[]) - // const shouldShowWelcomeBackModal = !func.checkLocal() && window?.USER_NAME?.length > 0 && (window?.USER_FULL_NAME?.length === 0 || (window?.USER_ROLE === 'ADMIN' && window?.ORGANIZATION_NAME?.length === 0)) + const shouldShowWelcomeBackModal = window.IS_SAAS === "true" && window?.USER_NAME?.length > 0 && (window?.USER_FULL_NAME?.length === 0 || (window?.USER_ROLE === 'ADMIN' && window?.ORGANIZATION_NAME?.length === 0)) return (
- {/* {shouldShowWelcomeBackModal && } */} + {shouldShowWelcomeBackModal && } {toastMarkup} {ConfirmationModalMarkup} {displayItems.length > 0 ?
From f7c0f87f67082583eff7fbc53e20742182277011 Mon Sep 17 00:00:00 2001 From: notshivansh Date: Wed, 1 Jan 2025 16:31:00 +0530 Subject: [PATCH 26/40] backend for clean up testing resources --- .../akto/test_editor/execution/Executor.java | 2 +- .../src/main/java/com/akto/testing/Main.java | 3 +- .../java/com/akto/testing/TestExecutor.java | 91 ++++++++++++++----- .../com/akto/dto/OriginalHttpResponse.java | 6 +- .../java/com/akto/runtime/utils/Utils.java | 39 ++++++++ 5 files changed, 113 insertions(+), 28 deletions(-) diff --git a/apps/testing/src/main/java/com/akto/test_editor/execution/Executor.java b/apps/testing/src/main/java/com/akto/test_editor/execution/Executor.java index 9dd96167cb..43d8bcfde7 100644 --- a/apps/testing/src/main/java/com/akto/test_editor/execution/Executor.java +++ b/apps/testing/src/main/java/com/akto/test_editor/execution/Executor.java @@ -485,7 +485,7 @@ private static boolean removeCustomAuth(RawApi rawApi, List cust return removed; } - private ExecutorSingleOperationResp modifyAuthTokenInRawApi(TestRoles testRole, RawApi rawApi) { + public static ExecutorSingleOperationResp modifyAuthTokenInRawApi(TestRoles testRole, RawApi rawApi) { Map> rawHeaders = rawApi.fetchReqHeaders(); for(AuthWithCond authWithCond: testRole.getAuthWithCondList()) { diff --git a/apps/testing/src/main/java/com/akto/testing/Main.java b/apps/testing/src/main/java/com/akto/testing/Main.java index acd9d69be7..b0fc059786 100644 --- a/apps/testing/src/main/java/com/akto/testing/Main.java +++ b/apps/testing/src/main/java/com/akto/testing/Main.java @@ -276,8 +276,7 @@ public void run() { private static final ScheduledExecutorService executorService = Executors.newSingleThreadScheduledExecutor(); public static void main(String[] args) throws InterruptedException { - String mongoURI = "mongodb://localhost:27017/admini"; - System.out.println("here/......."); + String mongoURI = System.getenv("AKTO_MONGO_CONN"); ReadPreference readPreference = ReadPreference.secondary(); if(DashboardMode.isOnPremDeployment()){ readPreference = ReadPreference.primary(); diff --git a/apps/testing/src/main/java/com/akto/testing/TestExecutor.java b/apps/testing/src/main/java/com/akto/testing/TestExecutor.java index 17bd4197a6..ebe011d24e 100644 --- a/apps/testing/src/main/java/com/akto/testing/TestExecutor.java +++ b/apps/testing/src/main/java/com/akto/testing/TestExecutor.java @@ -8,6 +8,7 @@ import com.akto.dao.DependencyNodeDao; import com.akto.dao.context.Context; import com.akto.dao.test_editor.YamlTemplateDao; +import com.akto.dao.testing.TestRolesDao; import com.akto.dao.testing.TestingRunResultDao; import com.akto.dao.testing.TestingRunResultSummariesDao; import com.akto.dao.testing.WorkflowTestResultsDao; @@ -56,6 +57,7 @@ import com.mongodb.WriteConcern; import com.mongodb.client.model.*; +import org.apache.commons.lang3.StringUtils; import org.bson.conversions.Bson; import org.bson.types.ObjectId; import org.json.JSONObject; @@ -676,52 +678,59 @@ public void startTestNew(ApiInfo.ApiInfoKey apiInfoKey, ObjectId testRunId, } - private Map> cleanUpTestArtifacts(List testingRunResults, ApiInfoKey apiInfoKey, TestingUtil testingUtil) { + private Map> cleanUpTestArtifacts(List testingRunResults, ApiInfoKey apiInfoKey, TestingUtil testingUtil, TestingRunConfig testingRunConfig) { - Map> cleanedUpRequests = new HashMap<>(); + Map> cleanedUpRequests = new HashMap<>(); for (TestingRunResult trr: testingRunResults) { for(GenericTestResult gtr: trr.getTestResults()) { for(String message: gtr.getResponses()) { if (message != null) { - RawApi rawApiToBeReplayed = RawApi.buildFromMessage(message); + String formattedMessage = null; + try { + formattedMessage = com.akto.runtime.utils.Utils.convertToSampleMessage(message); + } catch (Exception e) { + loggerMaker.errorAndAddToDb("Error while formatting message: " + e.getMessage(), LogDb.TESTING); + } + if (formattedMessage == null) { + continue; + } + RawApi rawApiToBeReplayed = RawApi.buildFromMessage(formattedMessage); if (rawApiToBeReplayed.getResponse().getStatusCode() >= 300) { continue; } switch (apiInfoKey.getMethod()) { case POST: Bson filterQ = DependencyNodeDao.generateChildrenFilter(apiInfoKey.getApiCollectionId(), apiInfoKey.getUrl(), apiInfoKey.getMethod()); - Bson delFilterQ = Filters.and(filterQ, Filters.eq(DependencyNode.METHOD_REQ, Method.DELETE.name())); + // Bson delFilterQ = Filters.and(filterQ, Filters.eq(DependencyNode.METHOD_REQ, Method.DELETE.name())); List children = DependencyNodeDao.instance.findAll(filterQ); if (!children.isEmpty()) { for(DependencyNode node: children) { - OriginalHttpRequest copiedReq = rawApiToBeReplayed.copy().getRequest(); - copiedReq.setUrl(node.getUrlReq()); - copiedReq.setMethod(node.getMethodReq()); - Map> valuesMap = Build.getValuesMap(rawApiToBeReplayed.getResponse()); - List samples = testingUtil.getSampleMessages().get(apiInfoKey); + ApiInfoKey cleanUpApiInfoKey = new ApiInfoKey(Integer.valueOf(node.getApiCollectionIdReq()), node.getUrlReq(), Method.valueOf(node.getMethodReq())); + List samples = testingUtil.getSampleMessages().get(cleanUpApiInfoKey); if (samples == null || samples.isEmpty()) { continue; } else { RawApi nextApi = RawApi.buildFromMessage(samples.get(0)); - nextApi.getRequest().setHeaders(copiedReq.getHeaders()); List kvPairs = new ArrayList<>(); boolean fullReplace = true; for(ParamInfo paramInfo: node.getParamInfos()) { if (paramInfo.isHeader()) continue; - Object valueFromResponse = valuesMap.get(paramInfo.getResponseParam()); + Set valuesFromResponse = valuesMap.get(paramInfo.getResponseParam()); - if (valueFromResponse == null) { + if (valuesFromResponse == null || valuesFromResponse.isEmpty()) { fullReplace = false; - break; } - KVPair kvPair = new KVPair(); + Object valueFromResponse = valuesFromResponse.iterator().next(); + + KVPair.KVType type = valueFromResponse instanceof Integer ? KVPair.KVType.INTEGER : KVPair.KVType.STRING; + KVPair kvPair = new KVPair(paramInfo.getRequestParam(), valueFromResponse.toString(), false, false, type); kvPairs.add(kvPair); } @@ -729,18 +738,47 @@ private Map> cleanUpTestArtifacts(List()); - System.out.println(nextApi.getResponse().getHeaders()); - System.out.println(nextResponse.getBody()); - + OriginalHttpResponse nextResponse = ApiExecutor.sendRequest(nextApi.getRequest(), true, testingRunConfig, false, new ArrayList<>()); + loggerMaker.infoAndAddToDb("cleanUpTestArtifacts: RESPONSE headers: " + nextApi.getResponse().getHeaders()); + loggerMaker.infoAndAddToDb("cleanUpTestArtifacts: RESPONSE body: " + nextResponse.getBody()); + loggerMaker.infoAndAddToDb("cleanUpTestArtifacts: RESPONSE status code: " + nextResponse.getStatusCode()); + + if(nextResponse.getStatusCode() < 300) { + if(cleanedUpRequests.get(apiInfoKey) != null) { + cleanedUpRequests.get(apiInfoKey).add(cleanUpApiInfoKey); + } else { + cleanedUpRequests.put(apiInfoKey, Arrays.asList(cleanUpApiInfoKey)); + } + } + } catch (Exception e) { e.printStackTrace(); System.out.println("exception in sending api request for cleanup" + e.getMessage()); @@ -750,6 +788,7 @@ private Map> cleanUpTestArtifacts(List parseCookie(List cookieList){ Map cookieMap = new HashMap<>(); if(cookieList==null)return cookieMap; From 994f48abba2acddbf6fcd24ae944aff9e4ceb608 Mon Sep 17 00:00:00 2001 From: Ankush Jain Date: Wed, 1 Jan 2025 03:01:29 -0800 Subject: [PATCH 27/40] black list test runs status event --- apps/dashboard/web/polaris_web/web/src/util/request.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/dashboard/web/polaris_web/web/src/util/request.js b/apps/dashboard/web/polaris_web/web/src/util/request.js index 8c70972c2a..11139d1d5c 100644 --- a/apps/dashboard/web/polaris_web/web/src/util/request.js +++ b/apps/dashboard/web/polaris_web/web/src/util/request.js @@ -129,7 +129,7 @@ service.interceptors.response.use((response) => { return response.data }, err) -const black_list_apis = ['dashboard/accessToken', 'api/fetchBurpPluginInfo', 'api/fetchActiveLoaders', 'api/fetchAllSubCategories', 'api/fetchVulnerableRequests'] +const black_list_apis = ['dashboard/accessToken', 'api/fetchBurpPluginInfo', 'api/fetchActiveLoaders', 'api/fetchAllSubCategories', 'api/fetchVulnerableRequests', 'api/fetchActiveTestRunsStatus'] async function raiseMixpanelEvent(api) { if (window?.Intercom) { if (api?.startsWith("/api/ingestPostman")) { From 56ca54515f0b9d37d6d5a0827ce40362bf141e44 Mon Sep 17 00:00:00 2001 From: notshivansh Date: Wed, 1 Jan 2025 16:40:43 +0530 Subject: [PATCH 28/40] frontend for cleanup testing resources --- .../java/com/akto/action/testing/StartTestAction.java | 11 +++++++++++ .../pages/observe/api_collections/RunTest.jsx | 8 +++++++- .../src/main/java/com/akto/testing/TestExecutor.java | 4 ++-- 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/apps/dashboard/src/main/java/com/akto/action/testing/StartTestAction.java b/apps/dashboard/src/main/java/com/akto/action/testing/StartTestAction.java index 7ab4be3d8d..35c8428b28 100644 --- a/apps/dashboard/src/main/java/com/akto/action/testing/StartTestAction.java +++ b/apps/dashboard/src/main/java/com/akto/action/testing/StartTestAction.java @@ -74,6 +74,8 @@ public class StartTestAction extends UserAction { private Map issuesSummaryInfoMap = new HashMap<>(); private String testRoleId; + private boolean cleanUpTestingResources; + private static final Gson gson = new Gson(); private static List getTestingRunListFromSummary(Bson filters){ @@ -152,6 +154,7 @@ private TestingRun createTestingRun(int scheduleTimestamp, int periodInSeconds) if(this.testConfigsAdvancedSettings != null && !this.testConfigsAdvancedSettings.isEmpty()){ testingRunConfig.setConfigsAdvancedSettings(this.testConfigsAdvancedSettings); } + testingRunConfig.setCleanUp(this.cleanUpTestingResources); this.testIdConfig = testingRunConfig.getId(); TestingRunConfigDao.instance.insertOne(testingRunConfig); } @@ -1401,4 +1404,12 @@ public Map getTestCountMap() { public void setReportFilterList(Map> reportFilterList) { this.reportFilterList = reportFilterList; } + + public boolean getCleanUpTestingResources() { + return cleanUpTestingResources; + } + + public void setCleanUpTestingResources(boolean cleanUpTestingResources) { + this.cleanUpTestingResources = cleanUpTestingResources; + } } diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/RunTest.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/RunTest.jsx index 7365689c4f..18ee9bf3f1 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/RunTest.jsx +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/RunTest.jsx @@ -35,7 +35,8 @@ function RunTest({ endpoints, filtered, apiCollectionId, disabled, runTestFromOu authMechanismPresent: false, testRoleLabel: "No test role selected", testRoleId: "", - sendSlackAlert: false + sendSlackAlert: false, + cleanUpTestingResources: false } const navigate = useNavigate() @@ -767,6 +768,11 @@ function RunTest({ endpoints, filtered, apiCollectionId, disabled, runTestFromOu } + setTestRun(prev => ({ ...prev, cleanUpTestingResources: !prev.cleanUpTestingResources}))} + /> diff --git a/apps/testing/src/main/java/com/akto/testing/TestExecutor.java b/apps/testing/src/main/java/com/akto/testing/TestExecutor.java index ebe011d24e..3ed4a0a2fa 100644 --- a/apps/testing/src/main/java/com/akto/testing/TestExecutor.java +++ b/apps/testing/src/main/java/com/akto/testing/TestExecutor.java @@ -949,13 +949,13 @@ public TestingRunResult runTestNew(ApiInfo.ApiInfoKey apiInfoKey, ObjectId testR vulnerable,singleTypeInfos,confidencePercentage,startTime, endTime, testRunResultSummaryId, testResults.getWorkflowTest(), testLogs); - // if (testingRunConfig.getCleanUp()) { + if (testingRunConfig!=null && testingRunConfig.getCleanUp()) { try { cleanUpTestArtifacts(Collections.singletonList(ret), apiInfoKey, testingUtil, testingRunConfig); } catch(Exception e){ loggerMaker.errorAndAddToDb("Error while cleaning up test artifacts: " + e.getMessage(), LogDb.TESTING); } - // } + } return ret; } From 8c55b2f2e88e8775053f318b632543f9b2a445a5 Mon Sep 17 00:00:00 2001 From: notshivansh Date: Wed, 1 Jan 2025 16:48:04 +0530 Subject: [PATCH 29/40] clean up --- .../akto/action/testing/StartTestAction.java | 3 +- .../java/com/akto/dto/traffic/SampleData.java | 3 -- .../java/com/akto/testing/ApiExecutor.java | 28 +------------------ .../src/main/java/com/akto/testing/Utils.java | 17 ++++------- 4 files changed, 7 insertions(+), 44 deletions(-) diff --git a/apps/dashboard/src/main/java/com/akto/action/testing/StartTestAction.java b/apps/dashboard/src/main/java/com/akto/action/testing/StartTestAction.java index 35c8428b28..deb67bd947 100644 --- a/apps/dashboard/src/main/java/com/akto/action/testing/StartTestAction.java +++ b/apps/dashboard/src/main/java/com/akto/action/testing/StartTestAction.java @@ -149,12 +149,11 @@ private TestingRun createTestingRun(int scheduleTimestamp, int periodInSeconds) } if (this.selectedTests != null) { int id = UUID.randomUUID().hashCode() & 0xfffffff; - TestingRunConfig testingRunConfig = new TestingRunConfig(id, null, this.selectedTests, authMechanism.getId(), this.overriddenTestAppUrl, this.testRoleId); + TestingRunConfig testingRunConfig = new TestingRunConfig(id, null, this.selectedTests, authMechanism.getId(), this.overriddenTestAppUrl, this.testRoleId, this.cleanUpTestingResources); // add advanced setting here if(this.testConfigsAdvancedSettings != null && !this.testConfigsAdvancedSettings.isEmpty()){ testingRunConfig.setConfigsAdvancedSettings(this.testConfigsAdvancedSettings); } - testingRunConfig.setCleanUp(this.cleanUpTestingResources); this.testIdConfig = testingRunConfig.getId(); TestingRunConfigDao.instance.insertOne(testingRunConfig); } diff --git a/libs/dao/src/main/java/com/akto/dto/traffic/SampleData.java b/libs/dao/src/main/java/com/akto/dto/traffic/SampleData.java index 1d86e9f4f4..dc1f170d23 100644 --- a/libs/dao/src/main/java/com/akto/dto/traffic/SampleData.java +++ b/libs/dao/src/main/java/com/akto/dto/traffic/SampleData.java @@ -3,12 +3,9 @@ import java.util.Arrays; import java.util.List; -import org.bson.codecs.pojo.annotations.BsonId; - import com.akto.util.Util; public class SampleData { - @BsonId Key id; public static final String SAMPLES = "samples"; diff --git a/libs/utils/src/main/java/com/akto/testing/ApiExecutor.java b/libs/utils/src/main/java/com/akto/testing/ApiExecutor.java index 53f34c8636..2443385e47 100644 --- a/libs/utils/src/main/java/com/akto/testing/ApiExecutor.java +++ b/libs/utils/src/main/java/com/akto/testing/ApiExecutor.java @@ -37,14 +37,10 @@ public class ApiExecutor { private static Map testScriptMap = new HashMap<>(); private static OriginalHttpResponse common(Request request, boolean followRedirects, boolean debug, List testLogs, boolean skipSSRFCheck, String requestProtocol) throws Exception { - debug = true; Integer accountId = Context.accountId.get(); if (accountId != null) { int i = 0; boolean rateLimitHit = true; - - - while (RateLimitHandler.getInstance(accountId).shouldWait(request)) { if(rateLimitHit){ if (!(request.url().toString().contains("insertRuntimeLog") || request.url().toString().contains("insertTestingLog") || request.url().toString().contains("insertProtectionLog"))) { @@ -82,9 +78,6 @@ private static OriginalHttpResponse common(Request request, boolean followRedire } Call call = client.newCall(request); - - System.out.println("------ACTUAL REQ-------"); - System.out.println(request.headers()); Response response = null; String body; byte[] grpcBody = null; @@ -341,9 +334,7 @@ public static OriginalHttpResponse sendRequest(OriginalHttpRequest request, bool if (!(url.contains("insertRuntimeLog") || url.contains("insertTestingLog") || url.contains("insertProtectionLog"))) { loggerMaker.infoAndAddToDb("Received response from: " + url, LogDb.TESTING); } - System.out.println("====RESPONSE===="); - System.out.println(response.getHeaders()); - System.out.println(response.getBody()); + return response; } public static OriginalHttpResponse sendRequest(OriginalHttpRequest request, boolean followRedirects, TestingRunConfig testingRunConfig, boolean debug, List testLogs) throws Exception { @@ -397,7 +388,6 @@ public void writeTo(BufferedSink sink) throws IOException { } public static void calculateFinalRequestFromAdvancedSettings(OriginalHttpRequest originalHttpRequest, List advancedSettings){ - System.out.println("in calculateFinalRequestFromAdvancedSettings..."); Map> headerConditions = new HashMap<>(); Map> payloadConditions = new HashMap<>(); @@ -416,14 +406,11 @@ public static void calculateFinalRequestFromAdvancedSettings(OriginalHttpRequest headerConditions.getOrDefault(TestEditorEnums.TerminalExecutorDataOperands.DELETE_HEADER.name(), emptyList) ); - System.out.println("modifyBodyOperations calculateFinalRequestFromAdvancedSettings..."); Utils.modifyBodyOperations(originalHttpRequest, payloadConditions.getOrDefault(TestEditorEnums.NonTerminalExecutorDataOperands.MODIFY_BODY_PARAM.name(), emptyList), payloadConditions.getOrDefault(TestEditorEnums.NonTerminalExecutorDataOperands.ADD_BODY_PARAM.name(), emptyList), payloadConditions.getOrDefault(TestEditorEnums.TerminalExecutorDataOperands.DELETE_BODY_PARAM.name(), emptyList) ); - System.out.println("modifyBodyOperations completed calculateFinalRequestFromAdvancedSettings..."); - } private static OriginalHttpResponse sendWithRequestBody(OriginalHttpRequest request, Request.Builder builder, boolean followRedirects, boolean debug, List testLogs, boolean skipSSRFCheck, String requestProtocol) throws Exception { @@ -479,14 +466,6 @@ private static OriginalHttpResponse sendWithRequestBody(OriginalHttpRequest requ if (payload == null) payload = ""; - try { - System.out.println("payloadStr: " + payload); - payload = Utils.replaceVariables(payload, new HashMap<>(), false, false); - System.out.println("payloadStrFinal: " + payload); - - } catch (Exception e) { - System.out.println("failed to replace vars in payload: " + e.getMessage()); - } if (body == null) {// body not created by GRPC block yet if (request.getHeaders().containsKey("charset")) { body = RequestBody.create(payload, null); @@ -496,11 +475,6 @@ private static OriginalHttpResponse sendWithRequestBody(OriginalHttpRequest requ } } - System.out.println("====REQUEST===="); - System.out.println(request.getMethod() + " " + request.getUrl() + "?" + request.getQueryParams()); - System.out.println(request.getHeaders()); - System.out.println(payload); - builder = builder.method(request.getMethod(), body); Request okHttpRequest = builder.build(); return common(okHttpRequest, followRedirects, debug, testLogs, skipSSRFCheck, requestProtocol); diff --git a/libs/utils/src/main/java/com/akto/testing/Utils.java b/libs/utils/src/main/java/com/akto/testing/Utils.java index 73ea006de2..a228dfa437 100644 --- a/libs/utils/src/main/java/com/akto/testing/Utils.java +++ b/libs/utils/src/main/java/com/akto/testing/Utils.java @@ -8,7 +8,6 @@ import java.util.Map; import java.util.Objects; import java.util.Set; -import java.util.UUID; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -217,16 +216,11 @@ public static String replaceVariables(String payload, Map values if (key == null) continue; Object obj = valuesMap.get(key); if (obj == null) { - if (key.toLowerCase().startsWith("x0.unique_")) { - String suffix = key.substring(key.toLowerCase().indexOf("_")+1); - obj = suffix+"_"+System.nanoTime(); - } else { - loggerMaker.errorAndAddToDb("couldn't find: " + key, LogDb.TESTING); - if(shouldThrowException){ - throw new Exception("Couldn't find " + key); - }else{ - continue; - } + loggerMaker.errorAndAddToDb("couldn't find: " + key, LogDb.TESTING); + if(shouldThrowException){ + throw new Exception("Couldn't find " + key); + }else{ + continue; } } String val = obj.toString(); @@ -359,7 +353,6 @@ private static ValidationResult validate(FilterNode node, RawApi rawApi, RawApi } public static void modifyBodyOperations(OriginalHttpRequest httpRequest, List modifyOperations, List addOperations, List deleteOperations){ - System.out.println("inside modifyBodyOperations"); String oldReqBody = httpRequest.getBody(); if(oldReqBody == null || oldReqBody.isEmpty()){ return ; From b3fb15fc54b30d46807a7b70b1636d9368b0e1a2 Mon Sep 17 00:00:00 2001 From: notshivansh Date: Wed, 1 Jan 2025 16:53:35 +0530 Subject: [PATCH 30/40] clean up --- .../java/com/akto/testing/TestExecutor.java | 22 +------------------ .../java/com/akto/dto/traffic/SampleData.java | 1 - .../java/com/akto/testing/ApiExecutor.java | 6 ++--- .../src/main/java/com/akto/testing/Utils.java | 1 - 4 files changed, 3 insertions(+), 27 deletions(-) diff --git a/apps/testing/src/main/java/com/akto/testing/TestExecutor.java b/apps/testing/src/main/java/com/akto/testing/TestExecutor.java index 3ed4a0a2fa..8477cf6b95 100644 --- a/apps/testing/src/main/java/com/akto/testing/TestExecutor.java +++ b/apps/testing/src/main/java/com/akto/testing/TestExecutor.java @@ -225,7 +225,7 @@ public void apiWiseInit(TestingRun testingRun, ObjectId summaryId, boolean debug try { currentTime = Context.now(); loggerMaker.infoAndAddToDb("Starting StatusCodeAnalyser at: " + currentTime, LogDb.TESTING); - // StatusCodeAnalyser.run(sampleDataMapForStatusCodeAnalyser, sampleMessageStore , authMechanismStore, testingRun.getTestingRunConfig(), hosts); + StatusCodeAnalyser.run(sampleDataMapForStatusCodeAnalyser, sampleMessageStore , authMechanismStore, testingRun.getTestingRunConfig(), hosts); loggerMaker.infoAndAddToDb("Completing StatusCodeAnalyser in: " + (Context.now() - currentTime) + " at: " + Context.now(), LogDb.TESTING); } catch (Exception e) { loggerMaker.errorAndAddToDb("Error while running status code analyser " + e.getMessage(), LogDb.TESTING); @@ -891,26 +891,6 @@ public TestingRunResult runTestNew(ApiInfo.ApiInfoKey apiInfoKey, ObjectId testR List customAuthTypes = testingUtil.getCustomAuthTypes(); // TestingUtil -> authMechanism // TestingConfig -> auth - - System.out.println("reached here Test Executor:855..."); - - if(testingRunConfig != null && testingRunConfig.getConfigsAdvancedSettings() != null && !testingRunConfig.getConfigsAdvancedSettings().isEmpty()){ - ApiExecutor.calculateFinalRequestFromAdvancedSettings(rawApi.getRequest(), testingRunConfig.getConfigsAdvancedSettings()); - // try { - // OriginalHttpResponse response = ApiExecutor.sendRequest(rawApi.getRequest(), false, testingRunConfig, false, new ArrayList<>()); - // if (response.getStatusCode() < 300) { - // rawApi = new RawApi(rawApi.getRequest(), response, ""); - // rawApi.fillOriginalMessage(0, 0, "", ""); - // } - // } catch (Exception e) { - // System.out.println("exception while making initial req: " + e.getMessage()); - // e.printStackTrace(); - // } - } - - - - com.akto.test_editor.execution.Executor executor = new Executor(); executor.overrideTestUrl(rawApi, testingRunConfig); YamlTestTemplate yamlTestTemplate = new YamlTestTemplate(apiInfoKey,filterNode, validatorNode, executorNode, diff --git a/libs/dao/src/main/java/com/akto/dto/traffic/SampleData.java b/libs/dao/src/main/java/com/akto/dto/traffic/SampleData.java index dc1f170d23..6fa212bf60 100644 --- a/libs/dao/src/main/java/com/akto/dto/traffic/SampleData.java +++ b/libs/dao/src/main/java/com/akto/dto/traffic/SampleData.java @@ -2,7 +2,6 @@ import java.util.Arrays; import java.util.List; - import com.akto.util.Util; public class SampleData { diff --git a/libs/utils/src/main/java/com/akto/testing/ApiExecutor.java b/libs/utils/src/main/java/com/akto/testing/ApiExecutor.java index 2443385e47..fd9a6bedde 100644 --- a/libs/utils/src/main/java/com/akto/testing/ApiExecutor.java +++ b/libs/utils/src/main/java/com/akto/testing/ApiExecutor.java @@ -37,6 +37,7 @@ public class ApiExecutor { private static Map testScriptMap = new HashMap<>(); private static OriginalHttpResponse common(Request request, boolean followRedirects, boolean debug, List testLogs, boolean skipSSRFCheck, String requestProtocol) throws Exception { + Integer accountId = Context.accountId.get(); if (accountId != null) { int i = 0; @@ -387,7 +388,7 @@ public void writeTo(BufferedSink sink) throws IOException { } - public static void calculateFinalRequestFromAdvancedSettings(OriginalHttpRequest originalHttpRequest, List advancedSettings){ + private static void calculateFinalRequestFromAdvancedSettings(OriginalHttpRequest originalHttpRequest, List advancedSettings){ Map> headerConditions = new HashMap<>(); Map> payloadConditions = new HashMap<>(); @@ -414,7 +415,6 @@ public static void calculateFinalRequestFromAdvancedSettings(OriginalHttpRequest } private static OriginalHttpResponse sendWithRequestBody(OriginalHttpRequest request, Request.Builder builder, boolean followRedirects, boolean debug, List testLogs, boolean skipSSRFCheck, String requestProtocol) throws Exception { - Map> headers = request.getHeaders(); if (headers == null) { headers = new HashMap<>(); @@ -465,7 +465,6 @@ private static OriginalHttpResponse sendWithRequestBody(OriginalHttpRequest requ } if (payload == null) payload = ""; - if (body == null) {// body not created by GRPC block yet if (request.getHeaders().containsKey("charset")) { body = RequestBody.create(payload, null); @@ -474,7 +473,6 @@ private static OriginalHttpResponse sendWithRequestBody(OriginalHttpRequest requ body = RequestBody.create(payload, MediaType.parse(contentType)); } } - builder = builder.method(request.getMethod(), body); Request okHttpRequest = builder.build(); return common(okHttpRequest, followRedirects, debug, testLogs, skipSSRFCheck, requestProtocol); diff --git a/libs/utils/src/main/java/com/akto/testing/Utils.java b/libs/utils/src/main/java/com/akto/testing/Utils.java index a228dfa437..77913061df 100644 --- a/libs/utils/src/main/java/com/akto/testing/Utils.java +++ b/libs/utils/src/main/java/com/akto/testing/Utils.java @@ -395,7 +395,6 @@ public static void modifyBodyOperations(OriginalHttpRequest httpRequest, List Date: Wed, 1 Jan 2025 17:08:36 +0530 Subject: [PATCH 31/40] fixes --- .../web/src/apps/dashboard/pages/observe/api.js | 8 ++++---- .../dashboard/pages/observe/api_collections/RunTest.jsx | 6 +++--- .../src/main/java/com/akto/testing/TestExecutor.java | 5 +++-- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api.js b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api.js index 1a425949a9..f17426ffb7 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api.js +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api.js @@ -573,20 +573,20 @@ export default { data: {} }) }, - scheduleTestForCollection(apiCollectionId, startTimestamp, recurringDaily, selectedTests, testName, testRunTime, maxConcurrentRequests, overriddenTestAppUrl, testRoleId, continuousTesting, sendSlackAlert, testConfigsAdvancedSettings) { + scheduleTestForCollection(apiCollectionId, startTimestamp, recurringDaily, selectedTests, testName, testRunTime, maxConcurrentRequests, overriddenTestAppUrl, testRoleId, continuousTesting, sendSlackAlert, testConfigsAdvancedSettings, cleanUpTestingResources) { return request({ url: '/api/startTest', method: 'post', - data: { apiCollectionId, type: "COLLECTION_WISE", startTimestamp, recurringDaily, selectedTests, testName, testRunTime, maxConcurrentRequests, overriddenTestAppUrl, testRoleId, continuousTesting, sendSlackAlert, testConfigsAdvancedSettings} + data: { apiCollectionId, type: "COLLECTION_WISE", startTimestamp, recurringDaily, selectedTests, testName, testRunTime, maxConcurrentRequests, overriddenTestAppUrl, testRoleId, continuousTesting, sendSlackAlert, testConfigsAdvancedSettings, cleanUpTestingResources} }).then((resp) => { return resp }) }, - scheduleTestForCustomEndpoints(apiInfoKeyList, startTimestamp, recurringDaily, selectedTests, testName, testRunTime, maxConcurrentRequests, overriddenTestAppUrl, source, testRoleId, continuousTesting, sendSlackAlert, testConfigsAdvancedSettings) { + scheduleTestForCustomEndpoints(apiInfoKeyList, startTimestamp, recurringDaily, selectedTests, testName, testRunTime, maxConcurrentRequests, overriddenTestAppUrl, source, testRoleId, continuousTesting, sendSlackAlert, testConfigsAdvancedSettings, cleanUpTestingResources) { return request({ url: '/api/startTest', method: 'post', - data: {apiInfoKeyList, type: "CUSTOM", startTimestamp, recurringDaily, selectedTests, testName, testRunTime, maxConcurrentRequests, overriddenTestAppUrl, source, testRoleId, continuousTesting, sendSlackAlert, testConfigsAdvancedSettings} + data: {apiInfoKeyList, type: "CUSTOM", startTimestamp, recurringDaily, selectedTests, testName, testRunTime, maxConcurrentRequests, overriddenTestAppUrl, source, testRoleId, continuousTesting, sendSlackAlert, testConfigsAdvancedSettings, cleanUpTestingResources} }).then((resp) => { return resp }) diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/RunTest.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/RunTest.jsx index 18ee9bf3f1..a89d1f59cc 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/RunTest.jsx +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/RunTest.jsx @@ -414,7 +414,7 @@ function RunTest({ endpoints, filtered, apiCollectionId, disabled, runTestFromOu } async function handleRun() { - const { startTimestamp, recurringDaily, testName, testRunTime, maxConcurrentRequests, overriddenTestAppUrl, testRoleId, continuousTesting, sendSlackAlert } = testRun + const { startTimestamp, recurringDaily, testName, testRunTime, maxConcurrentRequests, overriddenTestAppUrl, testRoleId, continuousTesting, sendSlackAlert, cleanUpTestingResources } = testRun const collectionId = parseInt(apiCollectionId) const tests = testRun.tests @@ -455,9 +455,9 @@ function RunTest({ endpoints, filtered, apiCollectionId, disabled, runTestFromOu } if (filtered || selectedResourcesForPrimaryAction.length > 0) { - await observeApi.scheduleTestForCustomEndpoints(apiInfoKeyList, startTimestamp, recurringDaily, selectedTests, testName, testRunTime, maxConcurrentRequests, overriddenTestAppUrl, "TESTING_UI", testRoleId, continuousTesting, sendSlackAlert, finalAdvancedConditions) + await observeApi.scheduleTestForCustomEndpoints(apiInfoKeyList, startTimestamp, recurringDaily, selectedTests, testName, testRunTime, maxConcurrentRequests, overriddenTestAppUrl, "TESTING_UI", testRoleId, continuousTesting, sendSlackAlert, finalAdvancedConditions, cleanUpTestingResources) } else { - await observeApi.scheduleTestForCollection(collectionId, startTimestamp, recurringDaily, selectedTests, testName, testRunTime, maxConcurrentRequests, overriddenTestAppUrl, testRoleId, continuousTesting, sendSlackAlert, finalAdvancedConditions) + await observeApi.scheduleTestForCollection(collectionId, startTimestamp, recurringDaily, selectedTests, testName, testRunTime, maxConcurrentRequests, overriddenTestAppUrl, testRoleId, continuousTesting, sendSlackAlert, finalAdvancedConditions, cleanUpTestingResources) } setActive(false) diff --git a/apps/testing/src/main/java/com/akto/testing/TestExecutor.java b/apps/testing/src/main/java/com/akto/testing/TestExecutor.java index 8477cf6b95..447f85399d 100644 --- a/apps/testing/src/main/java/com/akto/testing/TestExecutor.java +++ b/apps/testing/src/main/java/com/akto/testing/TestExecutor.java @@ -703,8 +703,9 @@ private Map> cleanUpTestArtifacts(List children = DependencyNodeDao.instance.findAll(filterQ); + // TODO: Handle cases where the delete API does not have the delete method + Bson delFilterQ = Filters.and(filterQ, Filters.eq(DependencyNode.METHOD_REQ, Method.DELETE.name())); + List children = DependencyNodeDao.instance.findAll(delFilterQ); if (!children.isEmpty()) { for(DependencyNode node: children) { From 84a6cd8e2db0724ff82ad1abdb922bb6933d67b5 Mon Sep 17 00:00:00 2001 From: notshivansh Date: Wed, 1 Jan 2025 18:26:59 +0530 Subject: [PATCH 32/40] add active account check --- .../pages/observe/api_collections/RunTest.jsx | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/RunTest.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/RunTest.jsx index a89d1f59cc..e6a714b94c 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/RunTest.jsx +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/RunTest.jsx @@ -768,11 +768,13 @@ function RunTest({ endpoints, filtered, apiCollectionId, disabled, runTestFromOu } - setTestRun(prev => ({ ...prev, cleanUpTestingResources: !prev.cleanUpTestingResources}))} - /> + {window.ACTIVE_ACCOUNT === 1723492815 && + setTestRun(prev => ({ ...prev, cleanUpTestingResources: !prev.cleanUpTestingResources }))} + /> + } From 811034cc4f13c8b8f25b8cbd33d842f61a5992cf Mon Sep 17 00:00:00 2001 From: Ankush Jain <91221068+ankush-jain-akto@users.noreply.github.com> Date: Wed, 1 Jan 2025 05:06:54 -0800 Subject: [PATCH 33/40] only for internal users --- .../polaris_web/web/src/apps/dashboard/pages/Dashboard.jsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/Dashboard.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/Dashboard.jsx index 094bbf84eb..37d5049167 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/Dashboard.jsx +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/Dashboard.jsx @@ -172,7 +172,7 @@ function Dashboard() { },[]) - const shouldShowWelcomeBackModal = window.IS_SAAS === "true" && window?.USER_NAME?.length > 0 && (window?.USER_FULL_NAME?.length === 0 || (window?.USER_ROLE === 'ADMIN' && window?.ORGANIZATION_NAME?.length === 0)) + const shouldShowWelcomeBackModal = window.IS_SAAS === "true" && window?.USER_NAME?.length > 0 && (window.USER_NAME.indexOf("@akto.io")>0) && (window?.USER_FULL_NAME?.length === 0 || (window?.USER_ROLE === 'ADMIN' && window?.ORGANIZATION_NAME?.length === 0)) return (
@@ -209,4 +209,4 @@ function Dashboard() { ) } -export default Dashboard \ No newline at end of file +export default Dashboard From f16f22d2e0df867092b0a097aa6e78c645ffccd1 Mon Sep 17 00:00:00 2001 From: Ankush Jain <91221068+ankush-jain-akto@users.noreply.github.com> Date: Wed, 1 Jan 2025 05:11:27 -0800 Subject: [PATCH 34/40] check for length --- apps/dashboard/web/pages/login.jsp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/apps/dashboard/web/pages/login.jsp b/apps/dashboard/web/pages/login.jsp index d46db74e1a..6a221a096b 100644 --- a/apps/dashboard/web/pages/login.jsp +++ b/apps/dashboard/web/pages/login.jsp @@ -111,12 +111,12 @@ mixpanel.identify(distinct_id); let mixpanelUserProps = { "$email": window.USER_NAME, "$account Name": window.ACCOUNT_NAME } - if (window.USER_FULL_NAME) { + if (window.USER_FULL_NAME?.length > 0) { mixpanelUserProps["name"] = window.USER_FULL_NAME mixpanelUserProps["$name"] = window.USER_FULL_NAME } - if (window.ORGANIZATION_NAME) { + if (window.ORGANIZATION_NAME?.length > 0) { mixpanelUserProps["company"] = window.ORGANIZATION_NAME mixpanelUserProps["$company"] = window.ORGANIZATION_NAME } @@ -141,11 +141,11 @@ test_runs_paused: window.USAGE_PAUSED?.testRuns === 'true' }; - if (window.USER_FULL_NAME) { + if (window.USER_FULL_NAME?.length > 0) { window.intercomSettings["name"] = window.USER_FULL_NAME } - if (window.ORGANIZATION_NAME && window.USER_NAME.indexOf("@")> 0) { + if (window.ORGANIZATION_NAME?.length > 0 && window.USER_NAME.indexOf("@")> 0) { let company_id = window.USER_NAME.split("@")[1]; window.intercomSettings["company"] = {name: window.ORGANIZATION_NAME, company_id} } @@ -187,4 +187,4 @@ - \ No newline at end of file + From 93cccde2c06f3a9007a59366ed36eebddc72df3a Mon Sep 17 00:00:00 2001 From: Umesh Kumar <166806589+TangoBeeAkto@users.noreply.github.com> Date: Wed, 1 Jan 2025 19:26:17 +0530 Subject: [PATCH 35/40] feat: okta oidc sso --- .../main/java/com/akto/action/HomeAction.java | 2 +- .../java/com/akto/action/SignupAction.java | 76 ++++++++++++------- .../com/akto/action/user/OktaSsoAction.java | 37 ++++----- .../main/java/com/akto/utils/OktaLogin.java | 17 ++++- .../settings/integrations/OktaIntegration.jsx | 2 +- .../src/main/java/com/akto/dto/Config.java | 48 +++++++++++- 6 files changed, 133 insertions(+), 49 deletions(-) diff --git a/apps/dashboard/src/main/java/com/akto/action/HomeAction.java b/apps/dashboard/src/main/java/com/akto/action/HomeAction.java index c1bbb000da..5d84c442f7 100644 --- a/apps/dashboard/src/main/java/com/akto/action/HomeAction.java +++ b/apps/dashboard/src/main/java/com/akto/action/HomeAction.java @@ -56,7 +56,7 @@ public String execute() { if (GithubLogin.getGithubUrl() != null) { servletRequest.setAttribute("githubUrl", GithubLogin.getGithubUrl()); } - if(OktaLogin.getAuthorisationUrl() != null){ + if(DashboardMode.isOnPremDeployment() && OktaLogin.getAuthorisationUrl() != null){ servletRequest.setAttribute("oktaAuthUrl", new String(Base64.getEncoder().encode(OktaLogin.getAuthorisationUrl().getBytes()))); } if (InitializerListener.aktoVersion != null && InitializerListener.aktoVersion.contains("akto-release-version")) { diff --git a/apps/dashboard/src/main/java/com/akto/action/SignupAction.java b/apps/dashboard/src/main/java/com/akto/action/SignupAction.java index 587a297ea3..1e15177698 100644 --- a/apps/dashboard/src/main/java/com/akto/action/SignupAction.java +++ b/apps/dashboard/src/main/java/com/akto/action/SignupAction.java @@ -509,32 +509,38 @@ public String registerViaGithub() { } public String registerViaOkta() throws IOException{ - if (!DashboardMode.isOnPremDeployment()) return Action.ERROR.toUpperCase(); - OktaLogin oktaLoginInstance = OktaLogin.getInstance(); - if(oktaLoginInstance == null){ - servletResponse.sendRedirect("/login"); - return ERROR.toUpperCase(); - } + try { + Config.OktaConfig oktaConfig; + if(DashboardMode.isOnPremDeployment()) { + OktaLogin oktaLoginInstance = OktaLogin.getInstance(); + if(oktaLoginInstance == null){ + servletResponse.sendRedirect("/login"); + return ERROR.toUpperCase(); + } - Config.OktaConfig oktaConfig = OktaLogin.getInstance().getOktaConfig(); - if (oktaConfig == null) { - servletResponse.sendRedirect("/login"); - return ERROR.toUpperCase(); - } + setAccountId(1000000); + oktaConfig = OktaLogin.getInstance().getOktaConfig(); + } else { + setAccountId(Integer.parseInt(state)); + oktaConfig = Config.getOktaConfig(accountId); + } + if(oktaConfig == null) { + servletResponse.sendRedirect("/login"); + return ERROR.toUpperCase(); + } - String domainUrl = "https://" + oktaConfig.getOktaDomainUrl() + "/oauth2/" + oktaConfig.getAuthorisationServerId() + "/v1"; - String clientId = oktaConfig.getClientId(); - String clientSecret = oktaConfig.getClientSecret(); - String redirectUri = oktaConfig.getRedirectUri(); + String domainUrl = "https://" + oktaConfig.getOktaDomainUrl() + "/oauth2/" + oktaConfig.getAuthorisationServerId() + "/v1"; + String clientId = oktaConfig.getClientId(); + String clientSecret = oktaConfig.getClientSecret(); + String redirectUri = oktaConfig.getRedirectUri(); - BasicDBObject params = new BasicDBObject(); - params.put("grant_type", "authorization_code"); - params.put("code", this.code); - params.put("client_id", clientId); - params.put("client_secret", clientSecret); - params.put("redirect_uri", redirectUri); + BasicDBObject params = new BasicDBObject(); + params.put("grant_type", "authorization_code"); + params.put("code", this.code); + params.put("client_id", clientId); + params.put("client_secret", clientSecret); + params.put("redirect_uri", redirectUri); - try { Map tokenData = CustomHttpRequest.postRequestEncodedType(domainUrl +"/token",params); String accessToken = tokenData.get("access_token").toString(); Map userInfo = CustomHttpRequest.getRequest( domainUrl + "/userinfo","Bearer " + accessToken); @@ -544,7 +550,7 @@ public String registerViaOkta() throws IOException{ SignupInfo.OktaSignupInfo oktaSignupInfo= new SignupInfo.OktaSignupInfo(accessToken, username); shouldLogin = "true"; - createUserAndRedirect(email, username, oktaSignupInfo, 1000000, Config.ConfigType.OKTA.toString()); + createUserAndRedirect(email, username, oktaSignupInfo, accountId, Config.ConfigType.OKTA.toString(), RBAC.Role.MEMBER); code = ""; } catch (Exception e) { loggerMaker.errorAndAddToDb("Error while signing in via okta sso \n" + e.getMessage(), LogDb.DASHBOARD); @@ -560,7 +566,7 @@ public String registerViaOkta() throws IOException{ public String sendRequestToSamlIdP() throws IOException{ String queryString = servletRequest.getQueryString(); String emailId = Util.getValueFromQueryString(queryString, "email"); - if(emailId.length() == 0){ + if(emailId.isEmpty()){ code = "Error, user email cannot be empty"; logger.error(code); servletResponse.sendRedirect("/login"); @@ -569,11 +575,10 @@ public String sendRequestToSamlIdP() throws IOException{ logger.info("Trying to sign in for: " + emailId); setUserEmail(emailId); SAMLConfig samlConfig = SSOConfigsDao.instance.getSSOConfig(userEmail); - if(samlConfig == null){ - code = "Error, cannot login via SSO, redirecting to login"; + if(samlConfig == null) { + code = "Error, cannot login via SSO, trying to login with okta sso"; logger.error(code); - servletResponse.sendRedirect("/login"); - return ERROR.toUpperCase(); + return oktaAuthUrlCreator(emailId); } int tempAccountId = Integer.parseInt(samlConfig.getId()); logger.info("Account id: " + tempAccountId + " found for " + emailId); @@ -599,6 +604,21 @@ public String sendRequestToSamlIdP() throws IOException{ return SUCCESS.toUpperCase(); } + public String oktaAuthUrlCreator(String emailId) throws IOException { + logger.info("Trying to create auth url for okta sso for: " + emailId); + Config.OktaConfig oktaConfig = Config.getOktaConfig(emailId); + if(oktaConfig == null) { + code= "Error, cannot find okta sso for this organization, redirecting to login"; + logger.error(code); + servletResponse.sendRedirect("/login"); + return ERROR.toUpperCase(); + } + + String authorisationUrl = OktaLogin.getAuthorisationUrl(emailId); + servletResponse.sendRedirect(authorisationUrl); + return SUCCESS.toUpperCase(); + } + public String registerViaAzure() throws Exception{ Auth auth; try { diff --git a/apps/dashboard/src/main/java/com/akto/action/user/OktaSsoAction.java b/apps/dashboard/src/main/java/com/akto/action/user/OktaSsoAction.java index 23dec2273f..c971a50ef3 100644 --- a/apps/dashboard/src/main/java/com/akto/action/user/OktaSsoAction.java +++ b/apps/dashboard/src/main/java/com/akto/action/user/OktaSsoAction.java @@ -24,12 +24,6 @@ public class OktaSsoAction extends UserAction { private String redirectUri; public String addOktaSso() { - - if(!DashboardMode.isOnPremDeployment()){ - addActionError("This feature is only available in on-prem deployment"); - return ERROR.toUpperCase(); - } - if (SsoUtils.isAnySsoActive()) { addActionError("A SSO Integration already exists."); return ERROR.toUpperCase(); @@ -41,6 +35,10 @@ public String addOktaSso() { oktaConfig.setAuthorisationServerId(authorisationServerId); oktaConfig.setOktaDomainUrl(oktaDomain); oktaConfig.setRedirectUri(redirectUri); + oktaConfig.setAccountId(Context.accountId.get()); + String userLogin = getSUser().getLogin(); + String domain = userLogin.split("@")[1]; + oktaConfig.setOrganizationDomain(domain); ConfigsDao.instance.insertOne(oktaConfig); @@ -48,13 +46,18 @@ public String addOktaSso() { } public String deleteOktaSso() { - if(!DashboardMode.isOnPremDeployment()){ - addActionError("This feature is only available in on-prem deployment"); - return ERROR.toUpperCase(); + DeleteResult result; + if(DashboardMode.isOnPremDeployment()) { + result = ConfigsDao.instance.deleteAll(Filters.eq("_id", "OKTA-ankush")); + } else { + result = ConfigsDao.instance.deleteAll( + Filters.and( + Filters.eq("_id", "OKTA-ankush"), + Filters.eq(Config.OktaConfig.ACCOUNT_ID, Context.accountId.get()) + ) + ); } - DeleteResult result = ConfigsDao.instance.deleteAll(Filters.eq("_id", "OKTA-ankush")); - if (result.getDeletedCount() > 0) { for (Object obj : UsersDao.instance.getAllUsersInfoForTheAccount(Context.accountId.get())) { BasicDBObject detailsObj = (BasicDBObject) obj; @@ -68,13 +71,13 @@ public String deleteOktaSso() { @Override public String execute() throws Exception { - - if(!DashboardMode.isOnPremDeployment()){ - addActionError("This feature is only available in on-prem deployment"); - return ERROR.toUpperCase(); + Config.OktaConfig oktaConfig; + if(DashboardMode.isOnPremDeployment()) { + oktaConfig = (Config.OktaConfig) ConfigsDao.instance.findOne("_id", "OKTA-ankush"); + } else { + String email = getSUser().getLogin(); + oktaConfig = Config.getOktaConfig(email); } - - Config.OktaConfig oktaConfig = (Config.OktaConfig) ConfigsDao.instance.findOne("_id", "OKTA-ankush"); if (SsoUtils.isAnySsoActive() && oktaConfig == null) { addActionError("A different SSO Integration already exists."); return ERROR.toUpperCase(); diff --git a/apps/dashboard/src/main/java/com/akto/utils/OktaLogin.java b/apps/dashboard/src/main/java/com/akto/utils/OktaLogin.java index aec09a1a79..317690ae59 100644 --- a/apps/dashboard/src/main/java/com/akto/utils/OktaLogin.java +++ b/apps/dashboard/src/main/java/com/akto/utils/OktaLogin.java @@ -2,7 +2,6 @@ import java.util.HashMap; import java.util.Map; -import java.util.stream.Collectors; import com.akto.dao.ConfigsDao; @@ -55,6 +54,22 @@ public static String getAuthorisationUrl() { return authUrl; } + public static String getAuthorisationUrl(String email) { + OktaConfig oktaConfig = Config.getOktaConfig(email); + + Map paramMap = new HashMap<>(); + paramMap.put("client_id", oktaConfig.getClientId()); + paramMap.put("redirect_uri",oktaConfig.getRedirectUri()); + paramMap.put("response_type", "code"); + paramMap.put("scope", "openid%20email%20profile"); + paramMap.put("state", String.valueOf(oktaConfig.getAccountId())); + + String queryString = SsoUtils.getQueryString(paramMap); + + String authUrl = "https://" + oktaConfig.getOktaDomainUrl() + "/oauth2/" + oktaConfig.getAuthorisationServerId() + "/v1/authorize?" + queryString; + return authUrl; + } + private OktaLogin() { } diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/settings/integrations/OktaIntegration.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/settings/integrations/OktaIntegration.jsx index 8375b7ff12..be8d1e2833 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/settings/integrations/OktaIntegration.jsx +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/settings/integrations/OktaIntegration.jsx @@ -22,7 +22,7 @@ function OktaIntegration() { const [oktaDomain, setOktaDomain] = useState('') const [authorizationServerId, setAuthorizationServerId] = useState('') const [showDeleteModal, setShowDeleteModal] = useState(false); - const [nextButtonActive,setNextButtonActive] = useState(window.DASHBOARD_MODE === "ON_PREM") + const [nextButtonActive,setNextButtonActive] = useState(true) const redirectUri = hostname + "/authorization-code/callback" diff --git a/libs/dao/src/main/java/com/akto/dto/Config.java b/libs/dao/src/main/java/com/akto/dto/Config.java index bb652dea71..570346bed7 100644 --- a/libs/dao/src/main/java/com/akto/dto/Config.java +++ b/libs/dao/src/main/java/com/akto/dto/Config.java @@ -4,6 +4,8 @@ import java.util.HashSet; import java.util.Set; +import com.akto.dao.ConfigsDao; +import com.mongodb.client.model.Filters; import org.bson.codecs.pojo.annotations.BsonDiscriminator; @BsonDiscriminator @@ -356,7 +358,11 @@ public static class OktaConfig extends Config { private String oktaDomainUrl; private String authorisationServerId; private String redirectUri; - + public static final String ORGANIZATION_DOMAIN = "organizationDomain"; + private String organizationDomain; + public static final String ACCOUNT_ID = "accountId"; + private int accountId; + public static final String CONFIG_ID = ConfigType.OKTA.name() + CONFIG_SALT; public OktaConfig() { @@ -399,6 +405,20 @@ public String getRedirectUri() { public void setRedirectUri(String redirectUri) { this.redirectUri = redirectUri; } + + public String getOrganizationDomain() { + return organizationDomain; + } + public void setOrganizationDomain(String organizationDomain) { + this.organizationDomain = organizationDomain; + } + + public int getAccountId() { + return accountId; + } + public void setAccountId(int accountId) { + this.accountId = accountId; + } } @BsonDiscriminator @@ -664,4 +684,30 @@ public static boolean isConfigSSOType(ConfigType configType){ } return ssoConfigTypes.contains(configType); } + + public static OktaConfig getOktaConfig(int accountId) { + OktaConfig config = (OktaConfig) ConfigsDao.instance.findOne( + Filters.and( + Filters.eq("_id", "OKTA-ankush"), + Filters.eq(OktaConfig.ACCOUNT_ID, accountId) + ) + ); + return config; + } + + public static OktaConfig getOktaConfig(String userEmail){ + if (userEmail == null || userEmail.trim().isEmpty()) { + return null; + } + String[] companyKeyArr = userEmail.split("@"); + if(companyKeyArr == null || companyKeyArr.length < 2){ + return null; + } + + String domain = companyKeyArr[1]; + OktaConfig config = (OktaConfig) ConfigsDao.instance.findOne( + Filters.eq(OktaConfig.ORGANIZATION_DOMAIN, domain) + ); + return config; + } } From 9ebaf0039e8f00f7db50c93413a47d196aa2b9bd Mon Sep 17 00:00:00 2001 From: Ark2307 Date: Wed, 1 Jan 2025 22:52:44 +0530 Subject: [PATCH 36/40] Adding info in UI --- .../com/akto/action/user/OktaSsoAction.java | 11 ++++++----- .../settings/integrations/OktaIntegration.jsx | 17 +++++++++++++++-- 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/apps/dashboard/src/main/java/com/akto/action/user/OktaSsoAction.java b/apps/dashboard/src/main/java/com/akto/action/user/OktaSsoAction.java index c971a50ef3..719c9a01d9 100644 --- a/apps/dashboard/src/main/java/com/akto/action/user/OktaSsoAction.java +++ b/apps/dashboard/src/main/java/com/akto/action/user/OktaSsoAction.java @@ -35,11 +35,12 @@ public String addOktaSso() { oktaConfig.setAuthorisationServerId(authorisationServerId); oktaConfig.setOktaDomainUrl(oktaDomain); oktaConfig.setRedirectUri(redirectUri); - oktaConfig.setAccountId(Context.accountId.get()); - String userLogin = getSUser().getLogin(); - String domain = userLogin.split("@")[1]; - oktaConfig.setOrganizationDomain(domain); - + if(!DashboardMode.isOnPremDeployment()){ + oktaConfig.setAccountId(Context.accountId.get()); + String userLogin = getSUser().getLogin(); + String domain = userLogin.split("@")[1]; + oktaConfig.setOrganizationDomain(domain); + } ConfigsDao.instance.insertOne(oktaConfig); return SUCCESS.toUpperCase(); diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/settings/integrations/OktaIntegration.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/settings/integrations/OktaIntegration.jsx index be8d1e2833..21fa86632c 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/settings/integrations/OktaIntegration.jsx +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/settings/integrations/OktaIntegration.jsx @@ -1,7 +1,7 @@ import React, { useEffect, useState } from 'react' import CopyCommand from '../../../components/shared/CopyCommand'; import IntegrationsLayout from './IntegrationsLayout'; -import { Button, Form, FormLayout, HorizontalStack, LegacyCard, Text, TextField } from '@shopify/polaris'; +import { Button, Form, FormLayout, HorizontalStack, LegacyCard, Link, Text, TextField, VerticalStack } from '@shopify/polaris'; import func from "@/util/func" import settingRequests from '../api'; import SpinnerCentered from "../../../components/progress/SpinnerCentered" @@ -135,6 +135,19 @@ function OktaIntegration() { },[]) const cardContent = "Enable login via Okta SSO in your dashboard." + + const useCardContent = ( + + {cardContent} + + Use + https://app.akto.io/sso-login + for signing into AKTO dashboard via SSO. + + + ) + + const oktaSSOComponent = ( loading ? : @@ -145,7 +158,7 @@ function OktaIntegration() { return ( <> - + ) From e658126577b89d8fc888c43c0b2837f8186824c2 Mon Sep 17 00:00:00 2001 From: Umesh Kumar <166806589+TangoBeeAkto@users.noreply.github.com> Date: Thu, 2 Jan 2025 00:17:16 +0530 Subject: [PATCH 37/40] fixed a bug --- .../src/apps/dashboard/components/WelcomeBackDetailsModal.jsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/WelcomeBackDetailsModal.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/WelcomeBackDetailsModal.jsx index 933ec3b680..e4ed036418 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/WelcomeBackDetailsModal.jsx +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/WelcomeBackDetailsModal.jsx @@ -69,7 +69,7 @@ const WelcomeBackDetailsModal = ({ isAdmin }) => { } if (window.mixpanel) { - mixpanelUserProps = { + const mixpanelUserProps = { "name": username, "company": organization, "$name": username, From b1f1e168a5f8ece0ab8e058e77af90bfe8838593 Mon Sep 17 00:00:00 2001 From: Ankush Jain <91221068+ankush-jain-akto@users.noreply.github.com> Date: Wed, 1 Jan 2025 20:01:16 -0800 Subject: [PATCH 38/40] Enable modal for all users --- .../web/polaris_web/web/src/apps/dashboard/pages/Dashboard.jsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/Dashboard.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/Dashboard.jsx index 37d5049167..ea0ece7f05 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/Dashboard.jsx +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/Dashboard.jsx @@ -172,7 +172,7 @@ function Dashboard() { },[]) - const shouldShowWelcomeBackModal = window.IS_SAAS === "true" && window?.USER_NAME?.length > 0 && (window.USER_NAME.indexOf("@akto.io")>0) && (window?.USER_FULL_NAME?.length === 0 || (window?.USER_ROLE === 'ADMIN' && window?.ORGANIZATION_NAME?.length === 0)) + const shouldShowWelcomeBackModal = window.IS_SAAS === "true" && window?.USER_NAME?.length > 0 && (window?.USER_FULL_NAME?.length === 0 || (window?.USER_ROLE === 'ADMIN' && window?.ORGANIZATION_NAME?.length === 0)) return (
From 14e3e7b7d2fe466185d171d5a3c6b4354d1a1e38 Mon Sep 17 00:00:00 2001 From: Ark2307 Date: Thu, 2 Jan 2025 11:45:31 +0530 Subject: [PATCH 39/40] Adding empty list check for conditions --- .../src/main/java/com/akto/action/ApiCollectionsAction.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/dashboard/src/main/java/com/akto/action/ApiCollectionsAction.java b/apps/dashboard/src/main/java/com/akto/action/ApiCollectionsAction.java index fa537a6c41..1f5f071649 100644 --- a/apps/dashboard/src/main/java/com/akto/action/ApiCollectionsAction.java +++ b/apps/dashboard/src/main/java/com/akto/action/ApiCollectionsAction.java @@ -90,7 +90,7 @@ public List fillApiCollectionsUrlCount(List apiCol int conditionsCount = 0; if(!apiCollection.getAutomated()){ ApiCollection apiCollectionWithCond = ApiCollectionsDao.instance.findOne(Filters.eq(Constants.ID, apiCollection.getId()), Projections.include("conditions")); - if(apiCollectionWithCond.getConditions() != null && apiCollectionWithCond.getConditions().get(0) != null){ + if(apiCollectionWithCond.getConditions() != null && !apiCollectionWithCond.getConditions().isEmpty() && apiCollectionWithCond.getConditions().get(0) != null){ if(apiCollectionWithCond.getConditions().get(0).getType().equals(TestingEndpoints.Type.CUSTOM)){ CustomTestingEndpoints testingEndpoints = (CustomTestingEndpoints) apiCollectionWithCond.getConditions().get(0); if (testingEndpoints.getApisList() != null && !testingEndpoints.getApisList().isEmpty()) { From d3a4a74f5b0a7fe8463dc0400e2c534d0bf4eb8e Mon Sep 17 00:00:00 2001 From: Ark2307 Date: Thu, 2 Jan 2025 12:13:19 +0530 Subject: [PATCH 40/40] Adding null check for payload --- .../src/apps/dashboard/components/shared/customDiffEditor.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/shared/customDiffEditor.js b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/shared/customDiffEditor.js index 8d405033e5..dc5e61ca4c 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/shared/customDiffEditor.js +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/shared/customDiffEditor.js @@ -256,7 +256,7 @@ const transform = { }) } finalData = finalData.split("\n").sort().join("\n"); - const isPayloadEmpty = Object.keys(payLoad).length === 0 + const isPayloadEmpty = payLoad === null || Object.keys(payLoad).length === 0 const isMultiformData = data?.json?.requestHeaders?.['content-type']?.includes('multipart/form-data') return (localFirstLine + "\n" + finalData + (finalData.trim().length === 0 || isPayloadEmpty ? "\n" : "\n\n") + (!isPayloadEmpty ? (isMultiformData ? payLoad : this.formatJson(payLoad)) : '')) }