Merge branch 'master' of https://github.com/akto-api-security/akto

akto-api-security · Apr 8, 2024 · 418d070 · 418d070
2 parents d18b624 + 5f6eae1
commit 418d070
Show file tree

Hide file tree

Showing 18 changed files with 108 additions and 66 deletions.
diff --git a/apps/api-runtime/pom.xml b/apps/api-runtime/pom.xml
@@ -47,12 +47,6 @@
             <version>5.4.2</version>
             <scope>test</scope>
         </dependency>
-        <dependency>
-            <groupId>com.fasterxml.jackson.core</groupId>
-            <artifactId>jackson-databind</artifactId>
-            <version>2.12.7.1</version>
-            <scope>compile</scope>
-        </dependency>
         <dependency>
             <groupId>org.apache.kafka</groupId>
             <artifactId>kafka-clients</artifactId>

diff --git a/apps/api-runtime/src/main/java/com/akto/parsers/HttpCallParser.java b/apps/api-runtime/src/main/java/com/akto/parsers/HttpCallParser.java
@@ -197,7 +197,7 @@ public void syncFunction(List<HttpResponseParams> responseParams, boolean syncIm
             filteredResponseParams = filterDefaultPayloads(filteredResponseParams, accountSettings.getDefaultPayloads());
         }
         filteredResponseParams = filterHttpResponseParams(filteredResponseParams);
-        boolean isHarOrPcap = aggregate(filteredResponseParams);
+        boolean isHarOrPcap = aggregate(filteredResponseParams, aggregatorMap);
 
         for (int apiCollectionId: aggregatorMap.keySet()) {
             URLAggregator aggregator = aggregatorMap.get(apiCollectionId);
@@ -445,7 +445,7 @@ public Map<Integer, URLAggregator> getAggregatorMap(){
         return this.aggregatorMap;
     }
 
-    public boolean aggregate(List<HttpResponseParams> responses) {
+    public static boolean aggregate(List<HttpResponseParams> responses, Map<Integer, URLAggregator> aggregatorMap) {
         int count = 0;
         boolean ret = false;
         Set<String> urlSet= new HashSet<>();

diff --git a/apps/api-runtime/src/main/java/com/akto/runtime/APICatalogSync.java b/apps/api-runtime/src/main/java/com/akto/runtime/APICatalogSync.java
@@ -58,13 +58,20 @@ public class APICatalogSync {
     public static boolean mergeAsyncOutside = true;
 
     public APICatalogSync(String userIdentifier,int thresh, boolean fetchAllSTI) {
+        this(userIdentifier, thresh, fetchAllSTI, true);
+    }
+
+    // New overloaded constructor
+    public APICatalogSync(String userIdentifier, int thresh, boolean fetchAllSTI, boolean buildFromDb) {
         this.thresh = thresh;
         this.userIdentifier = userIdentifier;
         this.dbState = new HashMap<>();
         this.delta = new HashMap<>();
         this.sensitiveParamInfoBooleanMap = new HashMap<>();
         this.aktoPolicyNew = new AktoPolicyNew();
-        buildFromDB(false, fetchAllSTI);
+        if (buildFromDb) {
+            buildFromDB(false, fetchAllSTI);
+        }
     }
 
     public static final int STRING_MERGING_THRESHOLD = 10;

diff --git a/apps/api-runtime/src/main/java/com/akto/utils/SampleDataToSTI.java b/apps/api-runtime/src/main/java/com/akto/utils/SampleDataToSTI.java
@@ -33,14 +33,13 @@ public SampleDataToSTI(){
 
     public void setSampleDataToSTI(List<SampleData> allData) {
 
-        HttpCallParser parse = new HttpCallParser("", 0, 0, 0, true);
         for (SampleData sampleData : allData) {
 
             Method method = sampleData.getId().getMethod();
             String url = sampleData.getId().getUrl();
             List<SingleTypeInfo> singleTypeInfoPerURL = new ArrayList<>();
             for (String dataString : sampleData.getSamples()) {
-                singleTypeInfoPerURL.addAll(getSampleDataToSTIUtil(dataString, url,parse));
+                singleTypeInfoPerURL.addAll(getSampleDataToSTIUtil(dataString, url));
             }
             Map<Integer, List<SingleTypeInfo>> responseCodeToSTI = new HashMap<>();
             for(SingleTypeInfo singleTypeInfo:singleTypeInfoPerURL){
@@ -74,7 +73,7 @@ public void setSensitiveSampleDataToSTI(List<SensitiveSampleData> allData){
             String url = sensitiveSampleData.getId().getUrl();
             List<SingleTypeInfo> singleTypeInfoPerURL = new ArrayList<>();
             for (String dataString : sensitiveSampleData.getSampleData()) {
-                singleTypeInfoPerURL.addAll(getSampleDataToSTIUtil(dataString, url,parse));
+                singleTypeInfoPerURL.addAll(getSampleDataToSTIUtil(dataString, url));
             }
             Map<Integer, List<SingleTypeInfo>> responseCodeToSTI = new HashMap<>();
             for(SingleTypeInfo singleTypeInfo:singleTypeInfoPerURL){
@@ -107,7 +106,7 @@ public List<SingleTypeInfo> getSingleTypeList(){
         return this.singleTypeInfos;
     }
 
-    private List<SingleTypeInfo> getSampleDataToSTIUtil(String dataString, String url,HttpCallParser parse) {
+    private List<SingleTypeInfo> getSampleDataToSTIUtil(String dataString, String url) {
 
         List<SingleTypeInfo> singleTypeInfos = new ArrayList<>();
 
@@ -127,18 +126,15 @@ private List<SingleTypeInfo> getSampleDataToSTIUtil(String dataString, String ur
 
         List<HttpResponseParams> responseParams = new ArrayList<>();
         responseParams.add(httpResponseParams);
-        List<HttpResponseParams> filteredResponseParams = parse.filterHttpResponseParams(responseParams);
         Map<Integer, URLAggregator> aggregatorMap = new HashMap<>();
-        parse.setAggregatorMap(aggregatorMap);
-        parse.aggregate(filteredResponseParams);
-        aggregatorMap = parse.getAggregatorMap();
-        parse.apiCatalogSync = new APICatalogSync("0",0, true);
+        HttpCallParser.aggregate(responseParams, aggregatorMap);
+        APICatalogSync apiCatalogSync = new APICatalogSync("0",0, true,false);
         for (int apiCollectionId : aggregatorMap.keySet()) {
             URLAggregator aggregator = aggregatorMap.get(apiCollectionId);
-            parse.apiCatalogSync.computeDelta(aggregator, false, apiCollectionId);
-            for (Integer key : parse.apiCatalogSync.delta.keySet()) {
-                APICatalog apiCatlog = parse.apiCatalogSync.delta.get(key);
-                singleTypeInfos.addAll(apiCatlog.getAllTypeInfo());
+            apiCatalogSync.computeDelta(aggregator, false, apiCollectionId);
+            for (Integer key : apiCatalogSync.delta.keySet()) {
+                APICatalog apiCatalog = apiCatalogSync.delta.get(key);
+                singleTypeInfos.addAll(apiCatalog.getAllTypeInfo());
             }
         }
 

diff --git a/apps/dashboard/pom.xml b/apps/dashboard/pom.xml
@@ -161,22 +161,6 @@
             <artifactId>twilio</artifactId>
             <version>8.8.0</version>
         </dependency>
-        <!--faster xml required by twilio-->
-        <dependency>
-            <groupId>com.fasterxml.jackson.core</groupId>
-            <artifactId>jackson-databind</artifactId>
-            <version>2.12.7.1</version>
-        </dependency>
-        <dependency>
-            <groupId>com.fasterxml.jackson.core</groupId>
-            <artifactId>jackson-core</artifactId>
-            <version>2.13.0</version>
-        </dependency>
-        <dependency>
-            <groupId>com.fasterxml.jackson.core</groupId>
-            <artifactId>jackson-annotations</artifactId>
-            <version>2.13.0</version>
-        </dependency>
         <dependency>
             <groupId>com.fasterxml.jackson.dataformat</groupId>
             <artifactId>jackson-dataformat-yaml</artifactId>

diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/shared/DonutChart.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/shared/DonutChart.jsx
@@ -2,9 +2,13 @@ import React from 'react'
 import HighchartsReact from "highcharts-react-official"
 import Highcharts from "highcharts"
 import { useRef } from "react";
+import { useNavigate } from "react-router-dom"
 
-function DonutChart({data, title, size}) {
+
+function DonutChart({data, title, size,type,navurl}) {
     const chartComponentRef = useRef(null)
+    const navigate = useNavigate()
+
 
     let seriesData = []
     if(data && Object.keys(data).length > 0){
@@ -40,6 +44,8 @@ function DonutChart({data, title, size}) {
                     [1, '#E0E0E0']
                 ]
             },
+            headerFormat: '',
+            pointFormat: '<b>{point.name} </b> {point.y}',
             borderWidth: 1,
             borderColor: '#AAA'
         },
@@ -50,6 +56,20 @@ function DonutChart({data, title, size}) {
               dataLabels: {
                 enabled: false
               }
+            },
+            series: {
+                point: {
+                    events: {
+
+                        click: (event) => {
+                            const { point } = event;
+                            if(navurl && navurl !=''){
+                                navigate(`${navurl}${point.name}?filter=${type.toLowerCase()}`);
+                            }
+                        }
+                    }
+                }
+
             }
         },
         series:[{

diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/dashboard/HomeDashboard.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/dashboard/HomeDashboard.jsx
@@ -168,8 +168,8 @@ function HomeDashboard() {
             <VerticalStack gap={5}>
                 <Text variant="bodyLg" fontWeight="semibold">Sensitive Data</Text>
                 <HorizontalGrid gap={5} columns={2}>
-                    <ChartypeComponent data={sensitiveData.request} title={"Request"} isNormal={true} boxHeight={'100px'}/>
-                    <ChartypeComponent data={sensitiveData.response} title={"Response"} isNormal={true} boxHeight={'100px'}/>
+                    <ChartypeComponent navurl={"/dashboard/observe/sensitive/"} data={sensitiveData.request} title={"Request"} isNormal={true} boxHeight={'100px'}/>
+                    <ChartypeComponent navurl={"/dashboard/observe/sensitive/"} data={sensitiveData.response} title={"Response"} isNormal={true} boxHeight={'100px'}/>
                 </HorizontalGrid>
             </VerticalStack>
         </Card>

diff --git a/...d/web/polaris_web/web/src/apps/dashboard/pages/testing/TestRunsPage/ChartypeComponent.jsx b/...d/web/polaris_web/web/src/apps/dashboard/pages/testing/TestRunsPage/ChartypeComponent.jsx
@@ -3,7 +3,7 @@ import React from 'react'
 import DonutChart from '../../../components/shared/DonutChart'
 import ConcentricCirclesChart from '../../../components/shared/ConcentricCirclesChart'
 
-function ChartypeComponent({data, title,charTitle, chartSubtitle, reverse, isNormal, boxHeight}) {
+function ChartypeComponent({data, title,charTitle, chartSubtitle, reverse, isNormal, boxHeight, navurl}) {
     const maxBoxHeight = boxHeight || '200px'
     let tableRows = []
     if(data && Object.keys(data).length > 0)
@@ -30,7 +30,8 @@ function ChartypeComponent({data, title,charTitle, chartSubtitle, reverse, isNor
       }, {}) : data
 
     const chartComponent = (
-        isNormal ? <DonutChart data={chartData} title="" size={210}/> : <ConcentricCirclesChart data={chartData} title={charTitle} size={210} subtitle={chartSubtitle} />
+
+        isNormal ? <DonutChart navurl={navurl} data={chartData}  title=""  type={title} size={210}/> : <ConcentricCirclesChart data={chartData} title={charTitle} size={210} subtitle={chartSubtitle} />
     )
 
     return (

diff --git a/apps/testing/src/main/java/com/akto/testing/Main.java b/apps/testing/src/main/java/com/akto/testing/Main.java
@@ -251,6 +251,19 @@ public void run() {
                     return;
                 }
 
+                if (testingRun.getState().equals(State.STOPPED)) {
+                    loggerMaker.infoAndAddToDb("Testing run stopped");
+                    if (trrs != null) {
+                        loggerMaker.infoAndAddToDb("Stopping TRRS: " + trrs.getId());
+                        TestingRunResultSummariesDao.instance.updateOneNoUpsert(
+                                Filters.eq(Constants.ID, trrs.getId()),
+                                Updates.set(TestingRunResultSummary.STATE, State.STOPPED)
+                        );
+                        loggerMaker.infoAndAddToDb("Stopped TRRS: " + trrs.getId());
+                    }
+                    return;
+                }
+
                 loggerMaker.infoAndAddToDb("Starting test for accountID: " + accountId);
 
                 boolean isTestingRunRunning = testingRun.getState().equals(State.RUNNING);
@@ -278,8 +291,14 @@ public void run() {
 
                     if (isSummaryRunning || isTestingRunRunning) {
                         loggerMaker.infoAndAddToDb("TRRS or TR is in running state, checking if it should run it or not");
-                        Map<ObjectId, TestingRunResultSummary> objectIdTestingRunResultSummaryMap = TestingRunResultSummariesDao.instance.fetchLatestTestingRunResultSummaries(Collections.singletonList(testingRun.getId()));
-                        TestingRunResultSummary testingRunResultSummary = objectIdTestingRunResultSummaryMap.get(testingRun.getId());
+                        TestingRunResultSummary testingRunResultSummary;
+                        if (trrs != null) {
+                            testingRunResultSummary = trrs;
+                        } else {
+                            Map<ObjectId, TestingRunResultSummary> objectIdTestingRunResultSummaryMap = TestingRunResultSummariesDao.instance.fetchLatestTestingRunResultSummaries(Collections.singletonList(testingRun.getId()));
+                            testingRunResultSummary = objectIdTestingRunResultSummaryMap.get(testingRun.getId());
+                        }
+
                         if (testingRunResultSummary != null) {
                             List<TestingRunResult> testingRunResults = TestingRunResultDao.instance.fetchLatestTestingRunResult(Filters.eq(TestingRunResult.TEST_RUN_RESULT_SUMMARY_ID, testingRunResultSummary.getId()), 1);
                             if (testingRunResults != null && !testingRunResults.isEmpty()) {

diff --git a/libs/dao/pom.xml b/libs/dao/pom.xml
@@ -61,7 +61,7 @@
         <dependency>
             <groupId>io.swagger.parser.v3</groupId>
             <artifactId>swagger-parser</artifactId>
-            <version>2.0.27</version>
+            <version>2.1.20</version>
             <exclusions>
                 <exclusion>
                     <groupId>org.slf4j</groupId>
@@ -76,7 +76,7 @@
         <dependency>
             <groupId>io.swagger</groupId>
             <artifactId>swagger-inflector</artifactId>
-            <version>2.0.0</version>
+            <version>2.0.10</version>
             <exclusions>
                 <exclusion>
                     <groupId>org.slf4j</groupId>

diff --git a/libs/dao/src/main/java/com/akto/DaoInit.java b/libs/dao/src/main/java/com/akto/DaoInit.java
@@ -342,6 +342,8 @@ public static void createIndices() {
         SingleTypeInfoDao.instance.createIndicesIfAbsent();
         TrafficMetricsDao.instance.createIndicesIfAbsent();
         TestRolesDao.instance.createIndicesIfAbsent();
+        UsersDao.instance.createIndicesIfAbsent();
+        AccountsDao.instance.createIndexIfAbsent();
 
         ApiInfoDao.instance.createIndicesIfAbsent();
         RuntimeLogsDao.instance.createIndicesIfAbsent();

diff --git a/libs/dao/src/main/java/com/akto/dao/AccountsDao.java b/libs/dao/src/main/java/com/akto/dao/AccountsDao.java
@@ -9,6 +9,13 @@ public class AccountsDao extends CommonContextDao<Account> {
 
     public static final AccountsDao instance = new AccountsDao();
 
+    public void createIndexIfAbsent() {
+
+        String[] fieldNames = { Account.INACTIVE_STR };
+        MCollection.createIndexIfAbsent(getDBName(), getCollName(), fieldNames, true);
+
+    }
+
     @Override
     public String getCollName() {
         return "accounts";

diff --git a/libs/dao/src/main/java/com/akto/dao/SingleTypeInfoDao.java b/libs/dao/src/main/java/com/akto/dao/SingleTypeInfoDao.java
@@ -52,8 +52,8 @@ public void createIndicesIfAbsent() {
         String[] fieldNames = {SingleTypeInfo._URL, SingleTypeInfo._METHOD, SingleTypeInfo._RESPONSE_CODE, SingleTypeInfo._IS_HEADER, SingleTypeInfo._PARAM, SingleTypeInfo.SUB_TYPE, SingleTypeInfo._API_COLLECTION_ID};
         MCollection.createIndexIfAbsent(getDBName(), getCollName(), fieldNames, true);
 
-        fieldNames = new String[]{SingleTypeInfo._API_COLLECTION_ID};
-        MCollection.createIndexIfAbsent(getDBName(), getCollName(), fieldNames, true);
+        fieldNames = new String[] { SingleTypeInfo._API_COLLECTION_ID, SingleTypeInfo._TIMESTAMP };
+        MCollection.createIndexIfAbsent(getDBName(), getCollName(), fieldNames, false);
 
         fieldNames = new String[]{SingleTypeInfo._PARAM, SingleTypeInfo._API_COLLECTION_ID};
         MCollection.createIndexIfAbsent(getDBName(), getCollName(), fieldNames, true);

diff --git a/libs/dao/src/main/java/com/akto/dao/UsersDao.java b/libs/dao/src/main/java/com/akto/dao/UsersDao.java
@@ -23,6 +23,13 @@
 
 public class UsersDao extends CommonContextDao<User> {
 
+    public void createIndicesIfAbsent() {
+
+        String[] fieldNames = { User.LOGIN };
+        MCollection.createIndexIfAbsent(getDBName(), getCollName(), fieldNames, true);
+
+    }
+
     public static User addUser(String login, String name, String password, boolean emailValidated) {
         // Checking if the user with same login exists or not
         if (UsersDao.instance.getMCollection().find(eq(User.LOGIN,login)).first() != null) {

diff --git a/libs/dao/src/main/java/com/akto/util/enums/GlobalEnums.java b/libs/dao/src/main/java/com/akto/util/enums/GlobalEnums.java
@@ -43,6 +43,7 @@ public enum TestCategory {
         SSTI("SSTI", Severity.HIGH, "Server Side Template Injection (SSTI)", "Server Side Template Injection"),
         LFI("LFI", Severity.HIGH, "Local File Inclusion (LFI)", "Local File Inclusion"),
         XSS("XSS", Severity.HIGH, "Cross-site scripting (XSS)", "Cross-site scripting"),
+        IIM("IIM", Severity.HIGH, "Improper Inventory Management (IIM)", "Improper Inventory Management"),
         LLM("LLM",Severity.HIGH,"LLM (Large Language Models) Top 10","LLM");
 
         private final String name;

diff --git a/libs/utils/pom.xml b/libs/utils/pom.xml
@@ -83,33 +83,17 @@
             <version>4.5.13</version>
             <scope>compile</scope>
         </dependency>
-        <dependency>
-            <groupId>com.fasterxml.jackson.core</groupId>
-            <artifactId>jackson-databind</artifactId>
-            <version>2.12.7.1</version>
-            <scope>compile</scope>
-        </dependency>
         <dependency>
             <groupId>de.sstoehr</groupId>
             <artifactId>har-reader</artifactId>
             <version>2.2.0</version>
         </dependency>
-        <dependency>
-            <groupId>com.fasterxml.jackson.core</groupId>
-            <artifactId>jackson-core</artifactId>
-            <version>2.12.2</version>
-        </dependency>
         <dependency>
             <groupId>org.junit.jupiter</groupId>
             <artifactId>junit-jupiter-api</artifactId>
             <version>5.4.2</version>
             <scope>test</scope>
         </dependency>
-        <dependency>
-            <groupId>com.fasterxml.jackson.core</groupId>
-            <artifactId>jackson-annotations</artifactId>
-            <version>2.12.2</version>
-        </dependency>
         <dependency>
             <groupId>org.apache.kafka</groupId>
             <artifactId>kafka-clients</artifactId>

diff --git a/libs/utils/src/main/java/com/akto/open_api/parser/parameter_parser/PathParamParser.java b/libs/utils/src/main/java/com/akto/open_api/parser/parameter_parser/PathParamParser.java
@@ -35,7 +35,11 @@ private static String replacePathParameterUtil(String path, Parameter parameter)
                 String type = schema.getType();
                 String format = schema.getFormat();
 
-                if ("integer".equalsIgnoreCase(type)) {
+                String example = schema.getExample() != null ? String.valueOf(schema.getExample()) : null;
+                if(example != null) {
+                    replacement = example;
+                }
+                else if ("integer".equalsIgnoreCase(type)) {
                     replacement = "INTEGER";
                 } else if ("string".equalsIgnoreCase(type)) {
                     if ("uuid".equalsIgnoreCase(format)) {

diff --git a/pom.xml b/pom.xml
@@ -108,6 +108,22 @@
             <version>4.13.1</version>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>com.fasterxml.jackson.core</groupId>
+            <artifactId>jackson-databind</artifactId>
+            <version>2.16.1</version>
+            <scope>compile</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.fasterxml.jackson.core</groupId>
+            <artifactId>jackson-core</artifactId>
+            <version>2.16.1</version>
+        </dependency>
+        <dependency>
+            <groupId>com.fasterxml.jackson.core</groupId>
+            <artifactId>jackson-annotations</artifactId>
+            <version>2.16.1</version>
+        </dependency>
     </dependencies>
 </project>