Merge pull request #1021 from akto-api-security/master

Recent hotfixes

.github/workflows/codeql.yml

@@ -14,16 +14,13 @@ name: "CodeQL"
 on:
   push:
     branches: [ "master" ]
-  pull_request:
-    # The branches below must be a subset of the branches above
-    branches: [ "master" ]
   schedule:
     - cron: '22 0 * * 1'
 
 jobs:
   analyze:
     name: Analyze
-    runs-on: ${{ (matrix.language == 'java' && 'macos-latest') || 'ubuntu-latest' }}
+    runs-on: 'ubuntu-latest'
     timeout-minutes: ${{ (matrix.language == 'java' && 120) || 360 }}
     permissions:
       actions: read

.github/workflows/staging.yml

@@ -13,8 +13,6 @@ jobs:
     outputs:
       IMAGE_TAG: ${{ steps.docker_tag.outputs.IMAGE_TAG }}
     steps:
-      - name: Setup Bolt
-        uses: koalalab-inc/bolt@v1
       - uses: actions/checkout@v2
       - uses: actions/setup-java@v2
         with:
@@ -112,8 +110,6 @@ jobs:
     outputs:
       lb_name: ${{ steps.deploy_cluster.outputs.lb_name }}
     steps:
-    - name: Setup Bolt
-      uses: koalalab-inc/bolt@v1
     #revert branch name
     - name: Setup mongo, akto-k8s-agent kube yaml's
       run: |
@@ -187,8 +183,6 @@ jobs:
     needs: deploy
     runs-on: ubuntu-latest
     steps:
-      - name: Setup Bolt
-        uses: koalalab-inc/bolt@v1
       - uses: actions/checkout@v2
       - uses: actions/setup-node@v2
         with:
@@ -227,8 +221,6 @@ jobs:
       FLASH_NLB_DNS: ${{ secrets.FLASH_NLB_DNS }}
       FLASH_MONGO_CONN: ${{ secrets.FLASH_MONGO_CONN }}
     steps:
-    - name: Setup Bolt
-      uses: koalalab-inc/bolt@v1
 
     - name: Fetch kube yaml
       #revert branch name

apps/api-runtime/src/main/java/com/akto/dependency/DependencyAnalyser.java

@@ -152,7 +152,10 @@ public void analyse(String message, int finalApiCollectionId) {
                 SingleTypeInfo.SuperType superType = urlTemplate.getTypes()[i];
                 if (superType == null) continue;
                 int idx = ogUrl.startsWith("http") ? i:i+1;
-                String s = ogUrlSplit[idx]; // because ogUrl=/api/books/123 while template url=api/books/INTEGER
+                Object s = ogUrlSplit[idx]; // because ogUrl=/api/books/123 while template url=api/books/INTEGER
+                if (superType.equals(SingleTypeInfo.SuperType.INTEGER)) {
+                    s = Integer.parseInt(ogUrlSplit[idx]);
+                }
                 Set<Object> val = new HashSet<>();
                 val.add(s);
                 processRequestParam(i+"", val, combinedUrl, true, false);

apps/api-runtime/src/test/java/com/akto/dependency/TestDependencyAnalyser.java

@@ -94,7 +94,7 @@ public void testAnalyse1()  {
         TreeHelper treeHelper = new TreeHelper();
         treeHelper.buildTree("1000", "/api/m7", "POST");
         Map<Integer, Node> result = treeHelper.result;
-        assertEquals(2, result.size()); // this is because /api/m6 gets best value from /api/m1
+        assertEquals(3, result.size()); // this is because /api/m6 gets best value from /api/m1
 
         Map<String, Connection> connections = result.get(Objects.hash("1000", "/api/m7", "POST")).getConnections();
         assertEquals(1, connections.size());
@@ -161,7 +161,7 @@ public void testAnalyse3() {
         TreeHelper treeHelper = new TreeHelper();
         treeHelper.buildTree("1000", "/api/m7", "POST");
         Map<Integer, Node> result = treeHelper.result;
-        assertEquals(6, result.size()); // this is because /api/m6 has 2 parameters getting data
+        assertEquals(7, result.size()); // this is because /api/m6 has 2 parameters getting data
 
         Map<String, Connection> connections = result.get(Objects.hash("1000", "/api/m7", "POST")).getConnections();
         assertEquals(1, connections.size());
@@ -197,7 +197,7 @@ public void testAnalyse4() {
         TreeHelper treeHelper = new TreeHelper();
         treeHelper.buildTree("1000", "api/cars/INTEGER", "POST");
         Map<Integer, Node> result = treeHelper.result;
-        assertEquals(1, result.size()); // this is because /api/m6 has 2 parameters getting data
+        assertEquals(2, result.size()); // this is because /api/m6 has 2 parameters getting data
 
         Map<String, Connection> connections = result.get(Objects.hash("1000", "api/cars/INTEGER", "POST")).getConnections();
         assertEquals(1, connections.size());

apps/dashboard/src/main/java/com/akto/action/DependencyAction.java

@@ -14,7 +14,7 @@
 import com.akto.dto.type.URLMethods.Method;
 import com.akto.log.LoggerMaker;
 import com.akto.runtime.RelationshipSync;
-import com.akto.utils.Build;
+import com.akto.test_editor.execution.Build;
 import com.akto.utils.Utils;
 import com.mongodb.BasicDBObject;
 import com.mongodb.ConnectionString;

apps/dashboard/src/main/java/com/akto/action/HarAction.java

@@ -80,7 +80,7 @@ public String execute() throws IOException {
                         return ERROR.toUpperCase();
                     }
                 } else {
-                    Collection<String> actionErrors = apiCollectionsAction.getActionErrors(); 
+                    Collection<String> actionErrors = apiCollectionsAction.getActionErrors();
                     if (actionErrors != null && actionErrors.size() > 0) {
                         for (String actionError: actionErrors) {
                             addActionError(actionError);
@@ -178,7 +178,7 @@ public void setTcpContent(byte[] tcpContent) {
     Awesome awesome = null;
 
     public String uploadTcp() {
-        
+
         File tmpDir = FileUtils.getTempDirectory();
         String filename = UUID.randomUUID().toString() + ".pcap";
         File tcpDump = new File(tmpDir, filename);
@@ -188,23 +188,23 @@ public String uploadTcp() {
             Awesome.GoString.ByValue str = new Awesome.GoString.ByValue();
             str.p = tcpDump.getAbsolutePath();
             str.n = str.p.length();
-    
+
             Awesome.GoString.ByValue str2 = new Awesome.GoString.ByValue();
             str2.p = System.getenv("AKTO_KAFKA_BROKER_URL");
             str2.n = str2.p.length();
-    
+
             awesome.readTcpDumpFile(str, str2 , apiCollectionId);
-    
-            return Action.SUCCESS.toUpperCase();            
+
+            return Action.SUCCESS.toUpperCase();
         } catch (IOException e) {
             // TODO Auto-generated catch block
             e.printStackTrace();
-            return Action.ERROR.toUpperCase();        
+            return Action.ERROR.toUpperCase();
         }
 
     }
 
-    interface Awesome extends Library {          
+    interface Awesome extends Library {
         public static class GoString extends Structure {
             /** C type : const char* */
             public String p;
@@ -224,8 +224,8 @@ public GoString(String p, long n) {
             public static class ByReference extends GoString implements Structure.ByReference {}
             public static class ByValue extends GoString implements Structure.ByValue {}
         }
-        
+
         public void readTcpDumpFile(GoString.ByValue filepath, GoString.ByValue kafkaURL, long apiCollectionId);
-        
+
     }
-}
+}

apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java

@@ -1795,6 +1795,8 @@ public static Organization fetchAndSaveFeatureWiseAllowed(Organization organizat
             hotjarSiteId = OrganizationUtils.fetchHotjarSiteId(metaData);
             boolean telemetryEnabled = OrganizationUtils.fetchTelemetryEnabled(metaData);
             setTelemetrySettings(organization, telemetryEnabled);
+            boolean testTelemetryEnabled = OrganizationUtils.fetchTestTelemetryEnabled(metaData);
+            organization.setTestTelemetryEnabled(testTelemetryEnabled);
 
             loggerMaker.infoAndAddToDb("Processed org metadata",LogDb.DASHBOARD);
 
@@ -1812,6 +1814,7 @@ public static Organization fetchAndSaveFeatureWiseAllowed(Organization organizat
                             Updates.set(Organization.FEATURE_WISE_ALLOWED, featureWiseAllowed),
                             Updates.set(Organization.GRACE_PERIOD, gracePeriod),
                             Updates.set(Organization.HOTJAR_SITE_ID, hotjarSiteId),
+                            Updates.set(Organization.TEST_TELEMETRY_ENABLED, testTelemetryEnabled),
                             Updates.set(Organization.LAST_FEATURE_MAP_UPDATE, lastFeatureMapUpdate)));
 
             loggerMaker.infoAndAddToDb("Updated org",LogDb.DASHBOARD);

apps/dashboard/src/main/java/com/akto/utils/Utils.java

@@ -158,6 +158,11 @@ public static boolean isValidURL(String url) {
             new URL(url).toURI();
             return true;
         } catch (MalformedURLException | URISyntaxException e) {
+            Pattern pattern = Pattern.compile("\\$\\{[^}]*\\}");
+                Matcher matcher = pattern.matcher(url);
+                if (matcher.find()) {
+                    return true;
+                }
             return false;
         }
     }
@@ -470,7 +475,7 @@ public static void pushDataToKafka(int apiCollectionId, String topic, List<Strin
                 info = new AccountHTTPCallParserAktoPolicyInfo();
                 HttpCallParser callParser = new HttpCallParser("userIdentifier", 1, 1, 1, false);
                 info.setHttpCallParser(callParser);
-//                info.setResourceAnalyser(new ResourceAnalyser(300_000, 0.01, 100_000, 0.01));
+                // info.setResourceAnalyser(new ResourceAnalyser(300_000, 0.01, 100_000, 0.01));
                 RuntimeListener.accountHTTPParserMap.put(accountId, info);
             }
 

apps/dashboard/src/main/java/com/akto/utils/billing/OrganizationUtils.java

@@ -286,4 +286,9 @@ public static boolean fetchTelemetryEnabled(BasicDBObject metadata) {
         BasicDBObject additionalMetaData = (BasicDBObject) metadata.getOrDefault("additionalMetaData", new BasicDBObject());
         return additionalMetaData.getString("ENABLE_TELEMETRY", "NA").equalsIgnoreCase("ENABLED");
     }
+
+    public static boolean fetchTestTelemetryEnabled(BasicDBObject metadata) {
+        BasicDBObject additionalMetaData = (BasicDBObject) metadata.getOrDefault("additionalMetaData", new BasicDBObject());
+        return additionalMetaData.getString("ENABLE_TEST_TELEMETRY", "NA").equalsIgnoreCase("ENABLED");
+    }
 }

apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/ApiDependency.jsx

@@ -97,7 +97,9 @@ function formatRawNodeData(nodes, currentApiCollectionId, currentEndpoint, curre
         let connections = node["connections"]
         let edgesMap = new Map()
         Object.values(connections).forEach(connection => {
-            let edge = connection["edges"][0] // todo: null check
+            let edge = connection["edges"][0]
+
+            if (!edge) return
 
             let source = calculateNodeId(edge["apiCollectionId"], edge["url"], edge["method"]);
             let edgeId = source + "-" + id;