diff --git a/apps/api-runtime/pom.xml b/apps/api-runtime/pom.xml index e2e12789b0..70a30f57cc 100644 --- a/apps/api-runtime/pom.xml +++ b/apps/api-runtime/pom.xml @@ -47,12 +47,6 @@ 5.4.2 test - - com.fasterxml.jackson.core - jackson-databind - 2.12.7.1 - compile - org.apache.kafka kafka-clients diff --git a/apps/api-runtime/src/main/java/com/akto/parsers/HttpCallParser.java b/apps/api-runtime/src/main/java/com/akto/parsers/HttpCallParser.java index bb567c9a58..799a0aedc9 100644 --- a/apps/api-runtime/src/main/java/com/akto/parsers/HttpCallParser.java +++ b/apps/api-runtime/src/main/java/com/akto/parsers/HttpCallParser.java @@ -197,7 +197,7 @@ public void syncFunction(List responseParams, boolean syncIm filteredResponseParams = filterDefaultPayloads(filteredResponseParams, accountSettings.getDefaultPayloads()); } filteredResponseParams = filterHttpResponseParams(filteredResponseParams); - boolean isHarOrPcap = aggregate(filteredResponseParams); + boolean isHarOrPcap = aggregate(filteredResponseParams, aggregatorMap); for (int apiCollectionId: aggregatorMap.keySet()) { URLAggregator aggregator = aggregatorMap.get(apiCollectionId); @@ -445,7 +445,7 @@ public Map getAggregatorMap(){ return this.aggregatorMap; } - public boolean aggregate(List responses) { + public static boolean aggregate(List responses, Map aggregatorMap) { int count = 0; boolean ret = false; Set urlSet= new HashSet<>(); diff --git a/apps/api-runtime/src/main/java/com/akto/runtime/APICatalogSync.java b/apps/api-runtime/src/main/java/com/akto/runtime/APICatalogSync.java index 7810178e76..7250518d01 100644 --- a/apps/api-runtime/src/main/java/com/akto/runtime/APICatalogSync.java +++ b/apps/api-runtime/src/main/java/com/akto/runtime/APICatalogSync.java @@ -58,13 +58,20 @@ public class APICatalogSync { public static boolean mergeAsyncOutside = true; public APICatalogSync(String userIdentifier,int thresh, boolean fetchAllSTI) { + this(userIdentifier, thresh, fetchAllSTI, true); + } + + // New overloaded constructor + public APICatalogSync(String userIdentifier, int thresh, boolean fetchAllSTI, boolean buildFromDb) { this.thresh = thresh; this.userIdentifier = userIdentifier; this.dbState = new HashMap<>(); this.delta = new HashMap<>(); this.sensitiveParamInfoBooleanMap = new HashMap<>(); this.aktoPolicyNew = new AktoPolicyNew(); - buildFromDB(false, fetchAllSTI); + if (buildFromDb) { + buildFromDB(false, fetchAllSTI); + } } public static final int STRING_MERGING_THRESHOLD = 10; diff --git a/apps/api-runtime/src/main/java/com/akto/utils/SampleDataToSTI.java b/apps/api-runtime/src/main/java/com/akto/utils/SampleDataToSTI.java index 95b81403e1..e07fbd10c2 100644 --- a/apps/api-runtime/src/main/java/com/akto/utils/SampleDataToSTI.java +++ b/apps/api-runtime/src/main/java/com/akto/utils/SampleDataToSTI.java @@ -33,14 +33,13 @@ public SampleDataToSTI(){ public void setSampleDataToSTI(List allData) { - HttpCallParser parse = new HttpCallParser("", 0, 0, 0, true); for (SampleData sampleData : allData) { Method method = sampleData.getId().getMethod(); String url = sampleData.getId().getUrl(); List singleTypeInfoPerURL = new ArrayList<>(); for (String dataString : sampleData.getSamples()) { - singleTypeInfoPerURL.addAll(getSampleDataToSTIUtil(dataString, url,parse)); + singleTypeInfoPerURL.addAll(getSampleDataToSTIUtil(dataString, url)); } Map> responseCodeToSTI = new HashMap<>(); for(SingleTypeInfo singleTypeInfo:singleTypeInfoPerURL){ @@ -74,7 +73,7 @@ public void setSensitiveSampleDataToSTI(List allData){ String url = sensitiveSampleData.getId().getUrl(); List singleTypeInfoPerURL = new ArrayList<>(); for (String dataString : sensitiveSampleData.getSampleData()) { - singleTypeInfoPerURL.addAll(getSampleDataToSTIUtil(dataString, url,parse)); + singleTypeInfoPerURL.addAll(getSampleDataToSTIUtil(dataString, url)); } Map> responseCodeToSTI = new HashMap<>(); for(SingleTypeInfo singleTypeInfo:singleTypeInfoPerURL){ @@ -107,7 +106,7 @@ public List getSingleTypeList(){ return this.singleTypeInfos; } - private List getSampleDataToSTIUtil(String dataString, String url,HttpCallParser parse) { + private List getSampleDataToSTIUtil(String dataString, String url) { List singleTypeInfos = new ArrayList<>(); @@ -127,18 +126,15 @@ private List getSampleDataToSTIUtil(String dataString, String ur List responseParams = new ArrayList<>(); responseParams.add(httpResponseParams); - List filteredResponseParams = parse.filterHttpResponseParams(responseParams); Map aggregatorMap = new HashMap<>(); - parse.setAggregatorMap(aggregatorMap); - parse.aggregate(filteredResponseParams); - aggregatorMap = parse.getAggregatorMap(); - parse.apiCatalogSync = new APICatalogSync("0",0, true); + HttpCallParser.aggregate(responseParams, aggregatorMap); + APICatalogSync apiCatalogSync = new APICatalogSync("0",0, true,false); for (int apiCollectionId : aggregatorMap.keySet()) { URLAggregator aggregator = aggregatorMap.get(apiCollectionId); - parse.apiCatalogSync.computeDelta(aggregator, false, apiCollectionId); - for (Integer key : parse.apiCatalogSync.delta.keySet()) { - APICatalog apiCatlog = parse.apiCatalogSync.delta.get(key); - singleTypeInfos.addAll(apiCatlog.getAllTypeInfo()); + apiCatalogSync.computeDelta(aggregator, false, apiCollectionId); + for (Integer key : apiCatalogSync.delta.keySet()) { + APICatalog apiCatalog = apiCatalogSync.delta.get(key); + singleTypeInfos.addAll(apiCatalog.getAllTypeInfo()); } } diff --git a/apps/dashboard/pom.xml b/apps/dashboard/pom.xml index f6862777ab..fe60d387a3 100644 --- a/apps/dashboard/pom.xml +++ b/apps/dashboard/pom.xml @@ -161,22 +161,6 @@ twilio 8.8.0 - - - com.fasterxml.jackson.core - jackson-databind - 2.12.7.1 - - - com.fasterxml.jackson.core - jackson-core - 2.13.0 - - - com.fasterxml.jackson.core - jackson-annotations - 2.13.0 - com.fasterxml.jackson.dataformat jackson-dataformat-yaml diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/shared/DonutChart.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/shared/DonutChart.jsx index 3cba3baae8..54d61a6943 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/shared/DonutChart.jsx +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/shared/DonutChart.jsx @@ -2,9 +2,13 @@ import React from 'react' import HighchartsReact from "highcharts-react-official" import Highcharts from "highcharts" import { useRef } from "react"; +import { useNavigate } from "react-router-dom" -function DonutChart({data, title, size}) { + +function DonutChart({data, title, size,type,navurl}) { const chartComponentRef = useRef(null) + const navigate = useNavigate() + let seriesData = [] if(data && Object.keys(data).length > 0){ @@ -40,6 +44,8 @@ function DonutChart({data, title, size}) { [1, '#E0E0E0'] ] }, + headerFormat: '', + pointFormat: '{point.name} {point.y}', borderWidth: 1, borderColor: '#AAA' }, @@ -50,6 +56,20 @@ function DonutChart({data, title, size}) { dataLabels: { enabled: false } + }, + series: { + point: { + events: { + + click: (event) => { + const { point } = event; + if(navurl && navurl !=''){ + navigate(`${navurl}${point.name}?filter=${type.toLowerCase()}`); + } + } + } + } + } }, series:[{ diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/dashboard/HomeDashboard.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/dashboard/HomeDashboard.jsx index e8b210aad2..97db1421eb 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/dashboard/HomeDashboard.jsx +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/dashboard/HomeDashboard.jsx @@ -168,8 +168,8 @@ function HomeDashboard() { Sensitive Data - - + + diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/TestRunsPage/ChartypeComponent.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/TestRunsPage/ChartypeComponent.jsx index 456401f418..aa3d05d207 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/TestRunsPage/ChartypeComponent.jsx +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/TestRunsPage/ChartypeComponent.jsx @@ -3,7 +3,7 @@ import React from 'react' import DonutChart from '../../../components/shared/DonutChart' import ConcentricCirclesChart from '../../../components/shared/ConcentricCirclesChart' -function ChartypeComponent({data, title,charTitle, chartSubtitle, reverse, isNormal, boxHeight}) { +function ChartypeComponent({data, title,charTitle, chartSubtitle, reverse, isNormal, boxHeight, navurl}) { const maxBoxHeight = boxHeight || '200px' let tableRows = [] if(data && Object.keys(data).length > 0) @@ -30,7 +30,8 @@ function ChartypeComponent({data, title,charTitle, chartSubtitle, reverse, isNor }, {}) : data const chartComponent = ( - isNormal ? : + + isNormal ? : ) return ( diff --git a/apps/testing/src/main/java/com/akto/testing/Main.java b/apps/testing/src/main/java/com/akto/testing/Main.java index ecb6903f94..f11501f3b7 100644 --- a/apps/testing/src/main/java/com/akto/testing/Main.java +++ b/apps/testing/src/main/java/com/akto/testing/Main.java @@ -251,6 +251,19 @@ public void run() { return; } + if (testingRun.getState().equals(State.STOPPED)) { + loggerMaker.infoAndAddToDb("Testing run stopped"); + if (trrs != null) { + loggerMaker.infoAndAddToDb("Stopping TRRS: " + trrs.getId()); + TestingRunResultSummariesDao.instance.updateOneNoUpsert( + Filters.eq(Constants.ID, trrs.getId()), + Updates.set(TestingRunResultSummary.STATE, State.STOPPED) + ); + loggerMaker.infoAndAddToDb("Stopped TRRS: " + trrs.getId()); + } + return; + } + loggerMaker.infoAndAddToDb("Starting test for accountID: " + accountId); boolean isTestingRunRunning = testingRun.getState().equals(State.RUNNING); @@ -278,8 +291,14 @@ public void run() { if (isSummaryRunning || isTestingRunRunning) { loggerMaker.infoAndAddToDb("TRRS or TR is in running state, checking if it should run it or not"); - Map objectIdTestingRunResultSummaryMap = TestingRunResultSummariesDao.instance.fetchLatestTestingRunResultSummaries(Collections.singletonList(testingRun.getId())); - TestingRunResultSummary testingRunResultSummary = objectIdTestingRunResultSummaryMap.get(testingRun.getId()); + TestingRunResultSummary testingRunResultSummary; + if (trrs != null) { + testingRunResultSummary = trrs; + } else { + Map objectIdTestingRunResultSummaryMap = TestingRunResultSummariesDao.instance.fetchLatestTestingRunResultSummaries(Collections.singletonList(testingRun.getId())); + testingRunResultSummary = objectIdTestingRunResultSummaryMap.get(testingRun.getId()); + } + if (testingRunResultSummary != null) { List testingRunResults = TestingRunResultDao.instance.fetchLatestTestingRunResult(Filters.eq(TestingRunResult.TEST_RUN_RESULT_SUMMARY_ID, testingRunResultSummary.getId()), 1); if (testingRunResults != null && !testingRunResults.isEmpty()) { diff --git a/libs/dao/pom.xml b/libs/dao/pom.xml index 0fa50c7f81..acd28287c6 100644 --- a/libs/dao/pom.xml +++ b/libs/dao/pom.xml @@ -61,7 +61,7 @@ io.swagger.parser.v3 swagger-parser - 2.0.27 + 2.1.20 org.slf4j @@ -76,7 +76,7 @@ io.swagger swagger-inflector - 2.0.0 + 2.0.10 org.slf4j diff --git a/libs/dao/src/main/java/com/akto/DaoInit.java b/libs/dao/src/main/java/com/akto/DaoInit.java index 8d74c6b0df..aca7c4362d 100644 --- a/libs/dao/src/main/java/com/akto/DaoInit.java +++ b/libs/dao/src/main/java/com/akto/DaoInit.java @@ -342,6 +342,8 @@ public static void createIndices() { SingleTypeInfoDao.instance.createIndicesIfAbsent(); TrafficMetricsDao.instance.createIndicesIfAbsent(); TestRolesDao.instance.createIndicesIfAbsent(); + UsersDao.instance.createIndicesIfAbsent(); + AccountsDao.instance.createIndexIfAbsent(); ApiInfoDao.instance.createIndicesIfAbsent(); RuntimeLogsDao.instance.createIndicesIfAbsent(); diff --git a/libs/dao/src/main/java/com/akto/dao/AccountsDao.java b/libs/dao/src/main/java/com/akto/dao/AccountsDao.java index 55bba2668c..c3e14e324f 100644 --- a/libs/dao/src/main/java/com/akto/dao/AccountsDao.java +++ b/libs/dao/src/main/java/com/akto/dao/AccountsDao.java @@ -9,6 +9,13 @@ public class AccountsDao extends CommonContextDao { public static final AccountsDao instance = new AccountsDao(); + public void createIndexIfAbsent() { + + String[] fieldNames = { Account.INACTIVE_STR }; + MCollection.createIndexIfAbsent(getDBName(), getCollName(), fieldNames, true); + + } + @Override public String getCollName() { return "accounts"; diff --git a/libs/dao/src/main/java/com/akto/dao/SingleTypeInfoDao.java b/libs/dao/src/main/java/com/akto/dao/SingleTypeInfoDao.java index 742f214b36..c9edf7b756 100644 --- a/libs/dao/src/main/java/com/akto/dao/SingleTypeInfoDao.java +++ b/libs/dao/src/main/java/com/akto/dao/SingleTypeInfoDao.java @@ -52,8 +52,8 @@ public void createIndicesIfAbsent() { String[] fieldNames = {SingleTypeInfo._URL, SingleTypeInfo._METHOD, SingleTypeInfo._RESPONSE_CODE, SingleTypeInfo._IS_HEADER, SingleTypeInfo._PARAM, SingleTypeInfo.SUB_TYPE, SingleTypeInfo._API_COLLECTION_ID}; MCollection.createIndexIfAbsent(getDBName(), getCollName(), fieldNames, true); - fieldNames = new String[]{SingleTypeInfo._API_COLLECTION_ID}; - MCollection.createIndexIfAbsent(getDBName(), getCollName(), fieldNames, true); + fieldNames = new String[] { SingleTypeInfo._API_COLLECTION_ID, SingleTypeInfo._TIMESTAMP }; + MCollection.createIndexIfAbsent(getDBName(), getCollName(), fieldNames, false); fieldNames = new String[]{SingleTypeInfo._PARAM, SingleTypeInfo._API_COLLECTION_ID}; MCollection.createIndexIfAbsent(getDBName(), getCollName(), fieldNames, true); diff --git a/libs/dao/src/main/java/com/akto/dao/UsersDao.java b/libs/dao/src/main/java/com/akto/dao/UsersDao.java index 78470b1466..d982fdcd87 100644 --- a/libs/dao/src/main/java/com/akto/dao/UsersDao.java +++ b/libs/dao/src/main/java/com/akto/dao/UsersDao.java @@ -23,6 +23,13 @@ public class UsersDao extends CommonContextDao { + public void createIndicesIfAbsent() { + + String[] fieldNames = { User.LOGIN }; + MCollection.createIndexIfAbsent(getDBName(), getCollName(), fieldNames, true); + + } + public static User addUser(String login, String name, String password, boolean emailValidated) { // Checking if the user with same login exists or not if (UsersDao.instance.getMCollection().find(eq(User.LOGIN,login)).first() != null) { diff --git a/libs/dao/src/main/java/com/akto/util/enums/GlobalEnums.java b/libs/dao/src/main/java/com/akto/util/enums/GlobalEnums.java index a788e0a71d..5812b9468c 100644 --- a/libs/dao/src/main/java/com/akto/util/enums/GlobalEnums.java +++ b/libs/dao/src/main/java/com/akto/util/enums/GlobalEnums.java @@ -43,6 +43,7 @@ public enum TestCategory { SSTI("SSTI", Severity.HIGH, "Server Side Template Injection (SSTI)", "Server Side Template Injection"), LFI("LFI", Severity.HIGH, "Local File Inclusion (LFI)", "Local File Inclusion"), XSS("XSS", Severity.HIGH, "Cross-site scripting (XSS)", "Cross-site scripting"), + IIM("IIM", Severity.HIGH, "Improper Inventory Management (IIM)", "Improper Inventory Management"), LLM("LLM",Severity.HIGH,"LLM (Large Language Models) Top 10","LLM"); private final String name; diff --git a/libs/utils/pom.xml b/libs/utils/pom.xml index 9c190d52f5..ef81986018 100644 --- a/libs/utils/pom.xml +++ b/libs/utils/pom.xml @@ -83,33 +83,17 @@ 4.5.13 compile - - com.fasterxml.jackson.core - jackson-databind - 2.12.7.1 - compile - de.sstoehr har-reader 2.2.0 - - com.fasterxml.jackson.core - jackson-core - 2.12.2 - org.junit.jupiter junit-jupiter-api 5.4.2 test - - com.fasterxml.jackson.core - jackson-annotations - 2.12.2 - org.apache.kafka kafka-clients diff --git a/libs/utils/src/main/java/com/akto/open_api/parser/parameter_parser/PathParamParser.java b/libs/utils/src/main/java/com/akto/open_api/parser/parameter_parser/PathParamParser.java index 48dc0227ba..952a80075f 100644 --- a/libs/utils/src/main/java/com/akto/open_api/parser/parameter_parser/PathParamParser.java +++ b/libs/utils/src/main/java/com/akto/open_api/parser/parameter_parser/PathParamParser.java @@ -35,7 +35,11 @@ private static String replacePathParameterUtil(String path, Parameter parameter) String type = schema.getType(); String format = schema.getFormat(); - if ("integer".equalsIgnoreCase(type)) { + String example = schema.getExample() != null ? String.valueOf(schema.getExample()) : null; + if(example != null) { + replacement = example; + } + else if ("integer".equalsIgnoreCase(type)) { replacement = "INTEGER"; } else if ("string".equalsIgnoreCase(type)) { if ("uuid".equalsIgnoreCase(format)) { diff --git a/pom.xml b/pom.xml index 7a7b52da8e..a823527581 100644 --- a/pom.xml +++ b/pom.xml @@ -108,6 +108,22 @@ 4.13.1 test + + com.fasterxml.jackson.core + jackson-databind + 2.16.1 + compile + + + com.fasterxml.jackson.core + jackson-core + 2.16.1 + + + com.fasterxml.jackson.core + jackson-annotations + 2.16.1 +